From 5748f750ad1d64d344983fdb96082658c78df98e Mon Sep 17 00:00:00 2001 From: meridah7 Date: Thu, 23 Jul 2026 15:52:40 -0700 Subject: [PATCH] feat: add Infron AI chat provider Adds Infron (https://infron.ai) as an aggregator provider for the puter-chat-completion driver, following the OpenRouter provider as reference per doc/contributing-apis.md. - OpenAI-compatible gateway at llm.onerouter.pro/v1 - Dynamic model catalog (kv-cached 15 min, authenticated fetch), filtered to chat-capable LLM entries - Bills the gateway-reported authoritative cost when present (usage: {include: true}); falls back to catalog per-token pricing - Registered as an aggregator so its aliases never shadow first-party providers - Offline unit tests (mocked SDK/axios against a real test server) plus an env-gated integration test --- config.template.jsonc | 4 + .../drivers/ai-chat/ChatCompletionDriver.ts | 15 +- .../infron/InfronProvider.integration.test.ts | 59 +++ .../providers/infron/InfronProvider.test.ts | 483 ++++++++++++++++++ .../providers/infron/InfronProvider.ts | 291 +++++++++++ 5 files changed, 851 insertions(+), 1 deletion(-) create mode 100644 src/backend/drivers/ai-chat/providers/infron/InfronProvider.integration.test.ts create mode 100644 src/backend/drivers/ai-chat/providers/infron/InfronProvider.test.ts create mode 100644 src/backend/drivers/ai-chat/providers/infron/InfronProvider.ts diff --git a/config.template.jsonc b/config.template.jsonc index c1b7e49e63..e2cad0ce24 100644 --- a/config.template.jsonc +++ b/config.template.jsonc @@ -287,6 +287,10 @@ "apiKey": "", "apiBaseUrl": "https://openrouter.ai/api/v1" }, + "infron": { + "apiKey": "", + "apiBaseUrl": "https://llm.onerouter.pro/v1" + }, "together-ai": { "apiKey": "" }, // Local Ollama. `enabled: false` skips the auto-probe at startup // (otherwise Puter logs ECONNREFUSED on every boot when no Ollama diff --git a/src/backend/drivers/ai-chat/ChatCompletionDriver.ts b/src/backend/drivers/ai-chat/ChatCompletionDriver.ts index 4cdc418015..f6564764f4 100644 --- a/src/backend/drivers/ai-chat/ChatCompletionDriver.ts +++ b/src/backend/drivers/ai-chat/ChatCompletionDriver.ts @@ -37,6 +37,7 @@ import { DeepSeekProvider } from './providers/deepseek/DeepSeekProvider.js'; import { FakeChatProvider } from './providers/FakeChatProvider.js'; import { GeminiChatProvider } from './providers/gemini/GeminiChatProvider.js'; import { GroqAIProvider } from './providers/groq/GroqAIProvider.js'; +import { InfronProvider } from './providers/infron/InfronProvider.js'; import { MiniMaxProvider } from './providers/minimax/MiniMaxProvider.js'; import { MistralAIProvider } from './providers/mistral/MistralAiProvider.js'; import { MoonshotProvider } from './providers/moonshot/MoonshotProvider.js'; @@ -986,6 +987,18 @@ export class ChatCompletionDriver extends PuterDriver { ); } + const infron = providers['infron']; + const infronKey = readKey(infron); + if (infronKey) { + this.#providers['infron'] = new InfronProvider( + { + apiKey: infronKey, + apiBaseUrl: infron?.apiBaseUrl as string | undefined, + }, + metering, + ); + } + // Fake provider — always available for testing this.#providers['fake-chat'] = new FakeChatProvider(); } @@ -993,7 +1006,7 @@ export class ChatCompletionDriver extends PuterDriver { // -- Model map --------------------------------------------------- async #buildModelMap() { - const AGGREGATORS = new Set(['together-ai', 'openrouter']); + const AGGREGATORS = new Set(['together-ai', 'openrouter', 'infron']); for (const providerName in this.#providers) { const provider = this.#providers[providerName]; diff --git a/src/backend/drivers/ai-chat/providers/infron/InfronProvider.integration.test.ts b/src/backend/drivers/ai-chat/providers/infron/InfronProvider.integration.test.ts new file mode 100644 index 0000000000..83afbec24d --- /dev/null +++ b/src/backend/drivers/ai-chat/providers/infron/InfronProvider.integration.test.ts @@ -0,0 +1,59 @@ +/** + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +/** + * Integration test for the Infron aggregator. + * + * Routes through Infron to a tiny upstream model + * (`deepseek/deepseek-v4-flash`). Skipped when + * `PUTER_TEST_AI_INFRON_API_KEY` is unset. + */ + +import { describe, expect, it } from 'vitest'; +import { + INTEGRATION_TEST_TIMEOUT_MS, + makeMeteringStub, + optionalEnv, + skipUnlessEnv, + withTestActor, +} from '../../../integrationTestUtil.js'; +import { InfronProvider } from './InfronProvider.js'; + +const ENV_VAR = 'PUTER_TEST_AI_INFRON_API_KEY'; + +describe.skipIf(skipUnlessEnv(ENV_VAR))('InfronProvider (integration)', () => { + it('returns a non-empty completion via Infron', { timeout: INTEGRATION_TEST_TIMEOUT_MS }, async () => { + const provider = new InfronProvider( + { apiKey: optionalEnv(ENV_VAR)! }, + makeMeteringStub(), + ); + + const result = await withTestActor(() => + provider.complete({ + model: 'infron:deepseek/deepseek-v4-flash', + messages: [{ role: 'user', content: 'Say hi in one word.' }], + max_tokens: 16, + }), + ); + + const text = (result as { message?: { content?: string } }).message + ?.content; + expect(typeof text === 'string' && text.length > 0).toBe(true); + }); +}); diff --git a/src/backend/drivers/ai-chat/providers/infron/InfronProvider.test.ts b/src/backend/drivers/ai-chat/providers/infron/InfronProvider.test.ts new file mode 100644 index 0000000000..c8d69bcbd7 --- /dev/null +++ b/src/backend/drivers/ai-chat/providers/infron/InfronProvider.test.ts @@ -0,0 +1,483 @@ +/* + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +/** + * Offline unit tests for InfronProvider. + * + * Boots a real PuterServer (in-memory sqlite + dynamo + s3 + mock + * redis) and constructs InfronProvider directly against the live + * wired `MeteringService` so the recording side is exercised end-to- + * end. Infron is OpenAI-compatible, so the OpenAI SDK is mocked at + * the module boundary; the model catalog is fetched via `axios` + * which is mocked at its module boundary too. Both are the real + * network egress points. Each test clears the kv-cached model list. + * The companion integration test (InfronProvider.integration.test.ts) + * exercises the real Infron endpoint. + */ + +import { Writable } from 'node:stream'; +import { + afterAll, + afterEach, + beforeAll, + beforeEach, + describe, + expect, + it, + vi, + type MockInstance, +} from 'vitest'; + +import type { MeteringService } from '../../../../services/metering/MeteringService.js'; +import { PuterServer } from '../../../../server.js'; +import { setupTestServer } from '../../../../testUtil.js'; +import { kv } from '../../../../util/kvSingleton.js'; +import { withTestActor } from '../../../integrationTestUtil.js'; +import { AIChatStream } from '../../utils/Streaming.js'; +import { InfronProvider } from './InfronProvider.js'; + +// ── OpenAI SDK mock ───────────────────────────────────────────────── + +const { createMock, openAICtor } = vi.hoisted(() => ({ + createMock: vi.fn(), + openAICtor: vi.fn(), +})); + +vi.mock('openai', () => { + const OpenAICtor = vi.fn().mockImplementation(function ( + this: Record, + opts: unknown, + ) { + openAICtor(opts); + this.chat = { completions: { create: createMock } }; + }); + return { OpenAI: OpenAICtor, default: { OpenAI: OpenAICtor } }; +}); + +// ── axios mock (model catalog endpoint) ───────────────────────────── + +const { axiosRequestMock } = vi.hoisted(() => ({ + axiosRequestMock: vi.fn(), +})); + +vi.mock('axios', () => ({ + default: { request: axiosRequestMock }, + request: axiosRequestMock, +})); + +// ── Test harness ──────────────────────────────────────────────────── + +let server: PuterServer; +let recordSpy: MockInstance; + +const KV_KEY = 'infronChat:models'; + +// Prices are USD per million tokens (Infron catalog convention). +const SAMPLE_API_MODELS = [ + { + id: 'deepseek/deepseek-v4-flash', + display_name: 'DeepSeek: DeepSeek V4 Flash', + category_type: 'LLM', + supported_endpoint_types: ['openai'], + context_length: 1000000, + max_output_tokens: 384000, + min_prompt_price: 10, + min_completion_price: 30, + }, + { + id: 'anthropic/claude-haiku-4.5', + display_name: 'Anthropic: Claude Haiku 4.5', + category_type: 'LLM', + supported_endpoint_types: ['openai'], + context_length: 200000, + max_output_tokens: 8192, + min_prompt_price: 2, + min_completion_price: 10, + }, + { + // Non-chat modality — filtered out. + id: 'black-forest-labs/flux-2.1', + display_name: 'FLUX 2.1', + category_type: 'Text to Image', + supported_endpoint_types: ['openai'], + min_request_price: 0.04, + }, + { + // Display-only entries are not callable — filtered out. + id: 'example/display-only-model', + display_name: 'Display Only', + category_type: 'LLM', + is_display_only: true, + supported_endpoint_types: ['openai'], + }, +]; + +const seedModelsCache = () => + axiosRequestMock.mockResolvedValue({ data: { data: SAMPLE_API_MODELS } }); + +beforeAll(async () => { + server = await setupTestServer(); +}); + +afterAll(async () => { + await server?.shutdown(); +}); + +const makeProvider = () => { + const provider = new InfronProvider( + { apiKey: 'test-key' }, + server.services.metering, + ); + return { provider }; +}; + +const asAsyncIterable = (items: T[]): AsyncIterable => ({ + async *[Symbol.asyncIterator]() { + for (const item of items) { + yield item; + } + }, +}); + +const makeCapturingChatStream = () => { + const chunks: string[] = []; + const sink = new Writable({ + write(chunk, _enc, cb) { + chunks.push(chunk.toString('utf8')); + cb(); + }, + }); + const chatStream = new AIChatStream({ stream: sink }); + return { + chatStream, + events: () => + chunks + .join('') + .split('\n') + .filter(Boolean) + .map((line) => JSON.parse(line)), + }; +}; + +beforeEach(() => { + createMock.mockReset(); + openAICtor.mockReset(); + axiosRequestMock.mockReset(); + seedModelsCache(); + kv.del(KV_KEY); + recordSpy = vi.spyOn(server.services.metering, 'utilRecordUsageObject'); +}); + +afterEach(() => { + vi.restoreAllMocks(); + kv.del(KV_KEY); +}); + +// ── Construction ──────────────────────────────────────────────────── + +describe('InfronProvider construction', () => { + it('points the OpenAI SDK at the Infron base URL with the configured key', () => { + makeProvider(); + expect(openAICtor).toHaveBeenCalledTimes(1); + expect(openAICtor).toHaveBeenCalledWith({ + apiKey: 'test-key', + baseURL: 'https://llm.onerouter.pro/v1', + }); + }); + + it('honours an apiBaseUrl override', () => { + new InfronProvider( + { + apiKey: 'test-key', + apiBaseUrl: 'https://custom.infron.example/v1', + }, + server.services.metering, + ); + expect(openAICtor).toHaveBeenLastCalledWith({ + apiKey: 'test-key', + baseURL: 'https://custom.infron.example/v1', + }); + }); +}); + +// ── Model catalog ─────────────────────────────────────────────────── + +describe('InfronProvider model catalog', () => { + it('returns the infron-prefixed default model id', () => { + const { provider } = makeProvider(); + expect(provider.getDefaultModel()).toBe( + 'infron:deepseek/deepseek-v4-flash', + ); + }); + + it('sends the API key as a bearer token on the catalog fetch', async () => { + const { provider } = makeProvider(); + await provider.models(); + const [args] = axiosRequestMock.mock.calls[0]!; + expect(args.url).toBe('https://llm.onerouter.pro/v1/models'); + expect(args.headers).toMatchObject({ + Authorization: 'Bearer test-key', + }); + }); + + it('list() prefixes ids with infron: and filters non-chat and display-only entries', async () => { + const { provider } = makeProvider(); + const ids = await provider.list(); + expect(ids).toContain('infron:deepseek/deepseek-v4-flash'); + expect(ids).toContain('infron:anthropic/claude-haiku-4.5'); + expect(ids).not.toContain('infron:black-forest-labs/flux-2.1'); + expect(ids).not.toContain('infron:example/display-only-model'); + }); + + it('caches the model list in kv after the first axios round-trip', async () => { + const { provider } = makeProvider(); + await provider.models(); + await provider.models(); + // Second call should be a cache hit, not a second axios request. + expect(axiosRequestMock).toHaveBeenCalledTimes(1); + }); + + it('converts USD-per-million-token prices to microcents per token', async () => { + const { provider } = makeProvider(); + const models = await provider.models(); + // $10/M tokens → 10 * 100 = 1000 microcents per token. + expect(models).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + id: 'infron:deepseek/deepseek-v4-flash', + costs: expect.objectContaining({ + tokens: 1_000_000, + prompt: 1000, + completion: 3000, + }), + }), + ]), + ); + }); +}); + +// ── Request shape ────────────────────────────────────────────────── + +describe('InfronProvider.complete request shape', () => { + const baseCompletion = { + choices: [ + { + message: { content: 'hi', role: 'assistant' }, + finish_reason: 'stop', + }, + ], + usage: { prompt_tokens: 1, completion_tokens: 1 }, + cost: 0, + }; + + it('strips the infron: prefix from the wire model id', async () => { + const { provider } = makeProvider(); + createMock.mockResolvedValueOnce(baseCompletion); + + await withTestActor(() => + provider.complete({ + model: 'infron:deepseek/deepseek-v4-flash', + messages: [{ role: 'user', content: 'hello' }], + }), + ); + + const [args] = createMock.mock.calls[0]!; + // infron: prefix is dropped before the SDK call. + expect(args.model).toBe('deepseek/deepseek-v4-flash'); + // Infron requires `usage: { include: true }` to surface the + // cost field — the provider always sets this. + expect(args.usage).toEqual({ include: true }); + }); + + it('only sets stream_options.include_usage when streaming', async () => { + const { provider } = makeProvider(); + + createMock.mockResolvedValueOnce(baseCompletion); + await withTestActor(() => + provider.complete({ + model: 'infron:deepseek/deepseek-v4-flash', + messages: [{ role: 'user', content: 'hi' }], + stream: false, + }), + ); + expect(createMock.mock.calls[0]![0].stream).toBe(false); + expect('stream_options' in createMock.mock.calls[0]![0]).toBe(false); + + createMock.mockReturnValueOnce(asAsyncIterable([])); + await withTestActor(() => + provider.complete({ + model: 'infron:deepseek/deepseek-v4-flash', + messages: [{ role: 'user', content: 'hi' }], + stream: true, + }), + ); + expect(createMock.mock.calls[1]![0].stream_options).toEqual({ + include_usage: true, + }); + }); +}); + +// ── Non-stream completion: cost calculator branches ───────────────── + +describe('InfronProvider.complete non-stream output', () => { + it('uses the cost-bearing branch when the top-level cost is present', async () => { + const { provider } = makeProvider(); + createMock.mockResolvedValueOnce({ + choices: [ + { + message: { content: 'hi there', role: 'assistant' }, + finish_reason: 'stop', + }, + ], + usage: { + prompt_tokens: 100, + completion_tokens: 50, + prompt_tokens_details: { cached_tokens: 10 }, + }, + // Infron reports cost at the top level of the completion, + // not inside `usage`. + cost: 0.0001, + }); + + const result = (await withTestActor(() => + provider.complete({ + model: 'infron:deepseek/deepseek-v4-flash', + messages: [{ role: 'user', content: 'hi' }], + }), + )) as { usage: Record }; + + // The cost-bearing branch zeroes per-token costs and bills via a + // single `billedUsage` line item priced at cost * 1e8. + expect(recordSpy).toHaveBeenCalledTimes(1); + const [usage, , prefix, overrides] = recordSpy.mock.calls[0]!; + expect(prefix).toBe('infron:deepseek/deepseek-v4-flash'); + expect(usage).toMatchObject({ + prompt: 100 - 10, // prompt_tokens - cached + completion: 50, + input_cache_read: 10, + billedUsage: 1, + }); + // All per-token costs are zeroed so Infron's authoritative + // cost is the only thing that bills. + expect(overrides.prompt).toBe(0); + expect(overrides.completion).toBe(0); + expect(overrides.input_cache_read).toBe(0); + expect(overrides.billedUsage).toBe(0.0001 * 100_000_000); + // The returned usage exposes usd_cents derived from cost. + expect(result.usage.usd_cents).toBe(0.0001 * 100); + }); + + it('falls back to per-token pricing when cost is absent', async () => { + const { provider } = makeProvider(); + createMock.mockResolvedValueOnce({ + choices: [ + { + message: { content: 'ok', role: 'assistant' }, + finish_reason: 'stop', + }, + ], + // No top-level `cost` → fallback branch. + usage: { + prompt_tokens: 100, + completion_tokens: 50, + prompt_tokens_details: { cached_tokens: 10 }, + }, + }); + + await withTestActor(() => + provider.complete({ + model: 'infron:deepseek/deepseek-v4-flash', + messages: [{ role: 'user', content: 'hi' }], + }), + ); + + // deepseek-v4-flash catalog pricing converted to microcents per + // token: prompt=$10/M → 1000, completion=$30/M → 3000; cache + // reads fall back to the full prompt rate. + const [usage, , , overrides] = recordSpy.mock.calls[0]!; + expect(usage).toMatchObject({ + prompt: 90, + completion: 50, + input_cache_read: 10, + }); + expect(overrides.prompt).toBe(90 * 1000); + expect(overrides.completion).toBe(50 * 3000); + expect(overrides.input_cache_read).toBe(10 * 1000); + }); +}); + +// ── Streaming deltas ──────────────────────────────────────────────── + +describe('InfronProvider.complete streaming', () => { + it('streams text deltas through to text events and meters the final-chunk cost', async () => { + const { provider } = makeProvider(); + createMock.mockReturnValueOnce( + asAsyncIterable([ + { choices: [{ delta: { content: 'hel' } }] }, + { choices: [{ delta: { content: 'lo' } }] }, + { + choices: [{ delta: {} }], + usage: { + prompt_tokens: 4, + completion_tokens: 2, + }, + // Cost rides at the top level of the final chunk. + cost: 0.00005, + }, + ]), + ); + + const result = await withTestActor(() => + provider.complete({ + model: 'infron:deepseek/deepseek-v4-flash', + messages: [{ role: 'user', content: 'say hi' }], + stream: true, + }), + ); + expect((result as { stream: boolean }).stream).toBe(true); + + const harness = makeCapturingChatStream(); + await ( + result as { + init_chat_stream: (p: { chatStream: unknown }) => Promise; + } + ).init_chat_stream({ chatStream: harness.chatStream }); + + const events = harness.events(); + const textEvents = events.filter((e) => e.type === 'text'); + expect(textEvents.map((e) => e.text)).toEqual(['hel', 'lo']); + + // Cost-branch metering on the final chunk. + expect(recordSpy).toHaveBeenCalledTimes(1); + const [, , prefix, overrides] = recordSpy.mock.calls[0]!; + expect(prefix).toBe('infron:deepseek/deepseek-v4-flash'); + expect(overrides.billedUsage).toBe(0.00005 * 100_000_000); + }); +}); + +// ── Moderation ────────────────────────────────────────────────────── + +describe('InfronProvider.checkModeration', () => { + it('throws — Infron provider does not implement moderation', () => { + const { provider } = makeProvider(); + expect(() => provider.checkModeration('anything')).toThrow( + /not implemented/i, + ); + }); +}); diff --git a/src/backend/drivers/ai-chat/providers/infron/InfronProvider.ts b/src/backend/drivers/ai-chat/providers/infron/InfronProvider.ts new file mode 100644 index 0000000000..9b7e4eb3ce --- /dev/null +++ b/src/backend/drivers/ai-chat/providers/infron/InfronProvider.ts @@ -0,0 +1,291 @@ +/* + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import axios from 'axios'; +import { OpenAI } from 'openai'; +import { ChatCompletionCreateParams } from 'openai/resources'; +import { Context } from '../../../../core/context.js'; +import type { MeteringService } from '../../../../services/metering/MeteringService.js'; +import { kv } from '../../../../util/kvSingleton.js'; +import * as OpenAIUtil from '../../utils/OpenAIUtil.js'; +import type { + IChatModel, + IChatProvider, + IChatCompleteResult, + ICompleteArguments, +} from '../../types.js'; + +/** + * Shape of one entry in Infron's `GET /v1/models` catalog. Unlike OpenRouter + * there is no `pricing` object; prices are USD per million tokens in + * `min_prompt_price` / `min_completion_price`, and the catalog mixes non-chat + * modalities (image, video, embeddings) that this provider filters out via + * `category_type`. + */ +type InfronApiModel = { + id: string; + display_name?: string; + category_type?: string; + is_display_only?: boolean; + supported_endpoint_types?: string[]; + context_length?: number; + max_output_tokens?: number; + min_prompt_price?: number; + min_completion_price?: number; + min_request_price?: number; +}; + +type InfronUsage = OpenAI.Completions.CompletionUsage & { + cost?: number; +}; + +const KV_MODELS_KEY = 'infronChat:models'; + +export class InfronProvider implements IChatProvider { + #meteringService: MeteringService; + + #openai: OpenAI; + + #apiKey: string; + + #apiBaseUrl: string = 'https://llm.onerouter.pro/v1'; + + constructor( + config: { apiBaseUrl?: string; apiKey: string }, + meteringService: MeteringService, + ) { + this.#apiBaseUrl = config.apiBaseUrl || 'https://llm.onerouter.pro/v1'; + this.#apiKey = config.apiKey; + this.#openai = new OpenAI({ + apiKey: config.apiKey, + baseURL: this.#apiBaseUrl, + }); + this.#meteringService = meteringService; + } + + getDefaultModel() { + return 'infron:deepseek/deepseek-v4-flash'; + } + + /** + * Returns a list of available model names + * + * @returns {Promise} Array of model identifiers + */ + async list() { + const models = await this.models(); + const model_names: string[] = []; + for (const model of models) { + model_names.push(model.id); + } + return model_names; + } + + /** AI Chat completion method. See AIChatService for more details. */ + async complete({ + messages, + stream, + model, + tools, + max_tokens, + temperature, + }: ICompleteArguments): Promise { + const availableModels = await this.models(); + const modelUsed = + availableModels.find((m) => + [m.id, ...(m.aliases || [])].includes(model), + ) || availableModels.find((m) => m.id === this.getDefaultModel())!; + + const modelIdForParams = modelUsed.id.startsWith('infron:') + ? modelUsed.id.slice('infron:'.length) + : modelUsed.id; + + const actor = Context.get('actor'); + + messages = await OpenAIUtil.process_input_messages(messages); + + const completionParams = { + messages, + model: modelIdForParams, + ...(tools ? { tools } : {}), + max_tokens, + temperature, + stream, + ...(stream + ? { + stream_options: { include_usage: true }, + } + : {}), + // Surfaces the authoritative `cost` field (USD) on the + // response so metering doesn't depend on catalog prices. + usage: { include: true }, + } as ChatCompletionCreateParams; + + const completion = + await this.#openai.chat.completions.create(completionParams); + + const usage_calculator = ({ + usage, + cost, + }: { + usage: InfronUsage; + cost?: number; + }) => { + // Infron reports `cost` at the top level of the response, not + // inside `usage`. Non-streaming calls get it via the spread + // completion below; streaming injects it into `usage` via the + // `index_usage_from_stream_chunk` deviation. + const authoritativeCost = + typeof cost === 'number' ? cost : usage.cost; + const trackedUsage = { + prompt: + (usage.prompt_tokens ?? 0) - + (usage.prompt_tokens_details?.cached_tokens ?? 0), + completion: usage.completion_tokens ?? 0, + input_cache_read: + usage.prompt_tokens_details?.cached_tokens ?? 0, + request: 1, + }; + if (typeof authoritativeCost === 'number') { + // Bill the gateway-reported cost as a single line item and + // zero the per-token costs so nothing double-bills. + const billedTrackedUsage = { ...trackedUsage, billedUsage: 1 }; + const costOverwrites = Object.fromEntries( + Object.keys(billedTrackedUsage).map((k) => [k, 0]), + ); + costOverwrites.billedUsage = + authoritativeCost * 100_000_000 || 1; + this.#meteringService.utilRecordUsageObject( + billedTrackedUsage, + actor, + modelUsed.id, + costOverwrites, + ); + (billedTrackedUsage as Record).usd_cents = + authoritativeCost * 100; + return billedTrackedUsage; + } + // Fallback: per-token pricing from the model catalog. + const costOverwrites = Object.fromEntries( + Object.keys(trackedUsage).map((k) => { + return [ + k, + (modelUsed.costs[k] ?? 0) * + trackedUsage[k as keyof typeof trackedUsage], + ]; + }), + ); + this.#meteringService.utilRecordUsageObject( + trackedUsage, + actor, + modelUsed.id, + costOverwrites, + ); + return trackedUsage; + }; + + return OpenAIUtil.handle_completion_output({ + deviations: { + index_usage_from_stream_chunk: (chunk: { + usage?: InfronUsage; + cost?: number; + }) => + chunk.usage + ? { ...chunk.usage, cost: chunk.cost } + : chunk.usage, + }, + usage_calculator, + stream, + completion, + }); + } + + async models() { + let models = kv.get(KV_MODELS_KEY) as InfronApiModel[] | undefined; + if (!models) { + try { + const resp = await axios.request({ + method: 'GET', + url: `${this.#apiBaseUrl}/models`, + // Infron requires authentication on the catalog endpoint. + headers: { + Authorization: `Bearer ${this.#apiKey}`, + }, + }); + + models = resp.data.data; + kv.set(KV_MODELS_KEY, models, { EX: 15 * 60 }); // cache for 15 minutes + } catch (e) { + console.log(e); + } + } + if (!models) return []; + const coerced_models: IChatModel[] = []; + for (const model of models) { + // The catalog mixes chat with image/video/embedding/search + // models — only chat-completion-capable models belong here. + if (model.category_type !== 'LLM') continue; + if (model.is_display_only) continue; + if (!(model.supported_endpoint_types ?? []).includes('openai')) { + continue; + } + // Catalog prices are USD per million tokens; costs are + // microcents per token, so the conversion is ×100. + const promptCost = Math.round((model.min_prompt_price ?? 0) * 100); + coerced_models.push({ + id: `infron:${model.id}`, + name: `${model.display_name || model.id} (Infron)`, + aliases: [ + model.id, + ...(model.display_name ? [model.display_name] : []), + `infron/${model.id}`, + model.id.split('/').slice(1).join('/'), + ], + context: model.context_length, + max_tokens: model.max_output_tokens ?? 0, + costs_currency: 'usd-cents', + input_cost_key: 'prompt', + output_cost_key: 'completion', + costs: { + tokens: 1_000_000, + prompt: promptCost, + completion: Math.round( + (model.min_completion_price ?? 0) * 100, + ), + // The catalog carries no cache-read price; charge the + // full prompt rate in the fallback path so cached + // tokens are never billed below list. The normal path + // bills the gateway-reported cost instead. + input_cache_read: promptCost, + // USD per request → microcents per request. + request: Math.round( + (model.min_request_price ?? 0) * 1_000_000 * 100, + ), + }, + }); + } + return coerced_models; + } + + checkModeration( + _text: string, + ): ReturnType { + throw new Error('Method not implemented.'); + } +}