From baec6f2a63944c1bcae23d41dc7600bcdd57495b Mon Sep 17 00:00:00 2001 From: "rosetta-livekit-bot[bot]" <282703043+rosetta-livekit-bot[bot]@users.noreply.github.com> Date: Fri, 22 May 2026 18:24:33 +0000 Subject: [PATCH 1/5] Add OpenAI provider tools --- .changeset/openai-provider-tools.md | 5 ++ plugins/openai/src/index.ts | 1 + plugins/openai/src/responses/llm.ts | 22 +----- plugins/openai/src/tool_utils.ts | 43 ++++++++++++ plugins/openai/src/tools.ts | 104 ++++++++++++++++++++++++++++ plugins/openai/src/ws/llm.ts | 26 +------ 6 files changed, 157 insertions(+), 44 deletions(-) create mode 100644 .changeset/openai-provider-tools.md create mode 100644 plugins/openai/src/tool_utils.ts create mode 100644 plugins/openai/src/tools.ts diff --git a/.changeset/openai-provider-tools.md b/.changeset/openai-provider-tools.md new file mode 100644 index 000000000..8e793a935 --- /dev/null +++ b/.changeset/openai-provider-tools.md @@ -0,0 +1,5 @@ +--- +'@livekit/agents-plugin-openai': minor +--- + +Add OpenAI Responses provider tools for web search, file search, and code interpreter. diff --git a/plugins/openai/src/index.ts b/plugins/openai/src/index.ts index ccffdcb3f..6a5d9cb7c 100644 --- a/plugins/openai/src/index.ts +++ b/plugins/openai/src/index.ts @@ -5,6 +5,7 @@ import { Plugin } from '@livekit/agents'; export { LLM, LLMStream, type LLMOptions } from './llm.js'; export * from './models.js'; +export * from './tools.js'; export * as realtime from './realtime/index.js'; export * as responses from './responses/index.js'; export { STT, type STTOptions } from './stt.js'; diff --git a/plugins/openai/src/responses/llm.ts b/plugins/openai/src/responses/llm.ts index 58020da99..4a1dc9d92 100644 --- a/plugins/openai/src/responses/llm.ts +++ b/plugins/openai/src/responses/llm.ts @@ -13,6 +13,7 @@ import { } from '@livekit/agents'; import OpenAI from 'openai'; import type { ChatModels } from '../models.js'; +import { toResponsesTools } from '../tool_utils.js'; import { WSLLM } from '../ws/llm.js'; export interface LLMOptions { @@ -186,27 +187,8 @@ class ResponsesHttpLLMStream extends llm.LLMStream { 'openai.responses', )) as OpenAI.Responses.ResponseInputItem[]; - // TODO: support provider tools in the Responses schema. const tools = this.toolCtx - ? this.toolCtx - .flatten() - .filter(llm.isFunctionTool) - .map((t) => { - const oaiParams = { - type: 'function' as const, - name: t.name, - description: t.description, - parameters: llm.toJsonSchema( - t.parameters, - true, - this.strictToolSchema, - ) as unknown as OpenAI.Responses.FunctionTool['parameters'], - } as OpenAI.Responses.FunctionTool; - if (this.strictToolSchema) { - oaiParams.strict = true; - } - return oaiParams; - }) + ? toResponsesTools(this.toolCtx, this.strictToolSchema) : undefined; const requestOptions: Record = { ...this.modelOptions }; diff --git a/plugins/openai/src/tool_utils.ts b/plugins/openai/src/tool_utils.ts new file mode 100644 index 000000000..1e2e6c709 --- /dev/null +++ b/plugins/openai/src/tool_utils.ts @@ -0,0 +1,43 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { llm } from '@livekit/agents'; +import type OpenAI from 'openai'; +import { OpenAITool } from './tools.js'; + +export function toResponsesTools( + toolCtx: llm.ToolContext, + strictToolSchema: boolean, +): OpenAI.Responses.Tool[] | undefined { + const tools = toolCtx + .flatten() + .map((tool) => { + if (llm.isFunctionTool(tool)) { + const oaiParams = { + type: 'function' as const, + name: tool.name, + description: tool.description, + parameters: llm.toJsonSchema( + tool.parameters, + true, + strictToolSchema, + ) as unknown as OpenAI.Responses.FunctionTool['parameters'], + } as OpenAI.Responses.FunctionTool; + + if (strictToolSchema) { + oaiParams.strict = true; + } + + return oaiParams; + } + + if (tool instanceof OpenAITool) { + return tool.toToolConfig() as unknown as OpenAI.Responses.Tool; + } + + return undefined; + }) + .filter((tool): tool is OpenAI.Responses.Tool => tool !== undefined); + + return tools.length > 0 ? tools : undefined; +} diff --git a/plugins/openai/src/tools.ts b/plugins/openai/src/tools.ts new file mode 100644 index 000000000..850eb85b4 --- /dev/null +++ b/plugins/openai/src/tools.ts @@ -0,0 +1,104 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { llm } from '@livekit/agents'; + +export abstract class OpenAITool extends llm.ProviderTool { + abstract toToolConfig(): Record; +} + +export type WebSearchContextSize = 'low' | 'medium' | 'high'; + +export interface WebSearchOptions { + filters?: Record; + searchContextSize?: WebSearchContextSize | null; + userLocation?: Record; +} + +export class WebSearch extends OpenAITool { + readonly filters: Record | undefined; + readonly searchContextSize: WebSearchContextSize | null; + readonly userLocation: Record | undefined; + + constructor({ filters, searchContextSize = 'medium', userLocation }: WebSearchOptions = {}) { + super({ id: 'openai_web_search' }); + this.filters = filters; + this.searchContextSize = searchContextSize; + this.userLocation = userLocation; + } + + toToolConfig(): Record { + const result: Record = { + type: 'web_search', + search_context_size: this.searchContextSize, + }; + if (this.userLocation !== undefined) { + result.user_location = this.userLocation; + } + if (this.filters !== undefined) { + result.filters = this.filters; + } + return result; + } +} + +export interface FileSearchOptions { + vectorStoreIds?: string[]; + filters?: Record; + maxNumResults?: number; + rankingOptions?: Record; +} + +export class FileSearch extends OpenAITool { + readonly vectorStoreIds: string[]; + readonly filters: Record | undefined; + readonly maxNumResults: number | undefined; + readonly rankingOptions: Record | undefined; + + constructor({ + vectorStoreIds = [], + filters, + maxNumResults, + rankingOptions, + }: FileSearchOptions = {}) { + super({ id: 'openai_file_search' }); + this.vectorStoreIds = [...vectorStoreIds]; + this.filters = filters; + this.maxNumResults = maxNumResults; + this.rankingOptions = rankingOptions; + } + + toToolConfig(): Record { + const result: Record = { + type: 'file_search', + vector_store_ids: this.vectorStoreIds, + }; + if (this.filters !== undefined) { + result.filters = this.filters; + } + if (this.maxNumResults !== undefined) { + result.max_num_results = this.maxNumResults; + } + if (this.rankingOptions !== undefined) { + result.ranking_options = this.rankingOptions; + } + return result; + } +} + +export interface CodeInterpreterOptions { + container?: string | Record | null; +} + +export class CodeInterpreter extends OpenAITool { + readonly container: string | Record | null; + + constructor({ container = null }: CodeInterpreterOptions = {}) { + super({ id: 'openai_code_interpreter' }); + this.container = container; + } + + toToolConfig(): Record { + return { type: 'code_interpreter', container: this.container }; + } +} diff --git a/plugins/openai/src/ws/llm.ts b/plugins/openai/src/ws/llm.ts index 1dbea698d..f75054387 100644 --- a/plugins/openai/src/ws/llm.ts +++ b/plugins/openai/src/ws/llm.ts @@ -15,6 +15,7 @@ import { import type OpenAI from 'openai'; import { WebSocket } from 'ws'; import type { ChatModels } from '../models.js'; +import { toResponsesTools } from '../tool_utils.js'; import type { WsOutputItemDoneEvent, WsOutputTextDeltaEvent, @@ -429,30 +430,7 @@ export class WSLLMStream extends llm.LLMStream { 'openai.responses', )) as OpenAI.Responses.ResponseInputItem[]; - // TODO: support provider tools in the Responses schema. - const tools = this.toolCtx - ? this.toolCtx - .flatten() - .filter(llm.isFunctionTool) - .map((t) => { - const oaiParams = { - type: 'function' as const, - name: t.name, - description: t.description, - parameters: llm.toJsonSchema( - t.parameters, - true, - this.#strictToolSchema, - ) as unknown as OpenAI.Responses.FunctionTool['parameters'], - } as OpenAI.Responses.FunctionTool; - - if (this.#strictToolSchema) { - oaiParams.strict = true; - } - - return oaiParams; - }) - : undefined; + const tools = this.toolCtx ? toResponsesTools(this.toolCtx, this.#strictToolSchema) : undefined; const requestOptions: Record = { ...this.#modelOptions }; if (!tools) { From 0913c49b40c7127d6c82e035b088765c53887be5 Mon Sep 17 00:00:00 2001 From: "rosetta-livekit-bot[bot]" <282703043+rosetta-livekit-bot[bot]@users.noreply.github.com> Date: Tue, 26 May 2026 20:28:07 +0000 Subject: [PATCH 2/5] Test OpenAI provider tool serialization --- plugins/openai/src/tool_utils.test.ts | 78 +++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 plugins/openai/src/tool_utils.test.ts diff --git a/plugins/openai/src/tool_utils.test.ts b/plugins/openai/src/tool_utils.test.ts new file mode 100644 index 000000000..f5d19a866 --- /dev/null +++ b/plugins/openai/src/tool_utils.test.ts @@ -0,0 +1,78 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { llm } from '@livekit/agents'; +import { describe, expect, it } from 'vitest'; +import { z } from 'zod'; +import { toResponsesTools } from './tool_utils.js'; +import { CodeInterpreter, FileSearch, WebSearch } from './tools.js'; + +describe('toResponsesTools', () => { + it('serializes function tools', () => { + const fn = llm.tool({ + name: 'lookup_weather', + description: 'Look up weather', + parameters: z.object({ city: z.string() }), + execute: async () => 'sunny', + }); + + expect(toResponsesTools(new llm.ToolContext([fn]), true)).toEqual([ + { + type: 'function', + name: 'lookup_weather', + description: 'Look up weather', + parameters: { + $schema: 'http://json-schema.org/draft-07/schema#', + type: 'object', + properties: { city: { type: 'string' } }, + required: ['city'], + additionalProperties: false, + }, + strict: true, + }, + ]); + }); + + it('serializes OpenAI provider tools', () => { + const tools = toResponsesTools( + new llm.ToolContext([ + new WebSearch({ + filters: { allowed_domains: ['docs.livekit.io'] }, + searchContextSize: 'low', + userLocation: { type: 'approximate', country: 'US' }, + }), + new FileSearch({ + vectorStoreIds: ['vs_123'], + maxNumResults: 3, + rankingOptions: { ranker: 'auto' }, + }), + new CodeInterpreter({ container: { type: 'auto', file_ids: ['file_123'] } }), + ]), + false, + ); + + expect(tools).toEqual([ + { + type: 'web_search', + search_context_size: 'low', + filters: { allowed_domains: ['docs.livekit.io'] }, + user_location: { type: 'approximate', country: 'US' }, + }, + { + type: 'file_search', + vector_store_ids: ['vs_123'], + max_num_results: 3, + ranking_options: { ranker: 'auto' }, + }, + { type: 'code_interpreter', container: { type: 'auto', file_ids: ['file_123'] } }, + ]); + }); + + it('ignores non-OpenAI provider tools', () => { + class OtherProviderTool extends llm.ProviderTool {} + + expect( + toResponsesTools(new llm.ToolContext([new OtherProviderTool({ id: 'other' })]), false), + ).toBeUndefined(); + }); +}); From 70fbb20e7399417be928deeea3476375c03e4f01 Mon Sep 17 00:00:00 2001 From: "rosetta-livekit-bot[bot]" <282703043+rosetta-livekit-bot[bot]@users.noreply.github.com> Date: Tue, 26 May 2026 21:27:08 +0000 Subject: [PATCH 3/5] Document OpenAI provider tools --- plugins/openai/src/tools.ts | 78 ++++++++++++++++++++++++++++++++----- 1 file changed, 68 insertions(+), 10 deletions(-) diff --git a/plugins/openai/src/tools.ts b/plugins/openai/src/tools.ts index 850eb85b4..ff273803b 100644 --- a/plugins/openai/src/tools.ts +++ b/plugins/openai/src/tools.ts @@ -2,23 +2,50 @@ // // SPDX-License-Identifier: Apache-2.0 import { llm } from '@livekit/agents'; +import type OpenAI from 'openai'; +/** Base class for OpenAI Responses API provider tools. */ export abstract class OpenAITool extends llm.ProviderTool { + /** Convert this provider tool to the OpenAI Responses API tool configuration. */ abstract toToolConfig(): Record; } +/** + * High-level guidance for the amount of context window space to use for web search. + * OpenAI defaults this to `medium`. + */ export type WebSearchContextSize = 'low' | 'medium' | 'high'; +/** Options for the OpenAI web search tool. */ export interface WebSearchOptions { - filters?: Record; + /** + * Filters for the search, such as allowed domains. If not provided, all domains are allowed. + */ + filters?: OpenAI.Responses.WebSearchTool['filters']; + + /** + * Amount of context window space to use for the search. Defaults to `medium`. + */ searchContextSize?: WebSearchContextSize | null; - userLocation?: Record; + + /** Approximate location of the user, such as city, region, country, or timezone. */ + userLocation?: OpenAI.Responses.WebSearchTool['user_location']; } +/** + * Search the Internet for sources related to the prompt. + * + * @see https://platform.openai.com/docs/guides/tools-web-search + */ export class WebSearch extends OpenAITool { - readonly filters: Record | undefined; + /** Filters for the search, such as allowed domains. */ + readonly filters: OpenAI.Responses.WebSearchTool['filters'] | undefined; + + /** Amount of context window space to use for the search. */ readonly searchContextSize: WebSearchContextSize | null; - readonly userLocation: Record | undefined; + + /** Approximate location of the user. */ + readonly userLocation: OpenAI.Responses.WebSearchTool['user_location'] | undefined; constructor({ filters, searchContextSize = 'medium', userLocation }: WebSearchOptions = {}) { super({ id: 'openai_web_search' }); @@ -42,18 +69,38 @@ export class WebSearch extends OpenAITool { } } +/** Options for the OpenAI file search tool. */ export interface FileSearchOptions { + /** IDs of the vector stores to search. */ vectorStoreIds?: string[]; - filters?: Record; + + /** Filter to apply to file search results. */ + filters?: OpenAI.Responses.FileSearchTool['filters']; + + /** Maximum number of results to return. This should be between 1 and 50 inclusive. */ maxNumResults?: number; - rankingOptions?: Record; + + /** Ranking options for search, including ranker and score threshold. */ + rankingOptions?: OpenAI.Responses.FileSearchTool.RankingOptions; } +/** + * Search for relevant content from uploaded files. + * + * @see https://platform.openai.com/docs/guides/tools-file-search + */ export class FileSearch extends OpenAITool { + /** IDs of the vector stores to search. */ readonly vectorStoreIds: string[]; - readonly filters: Record | undefined; + + /** Filter to apply to file search results. */ + readonly filters: OpenAI.Responses.FileSearchTool['filters'] | undefined; + + /** Maximum number of results to return. */ readonly maxNumResults: number | undefined; - readonly rankingOptions: Record | undefined; + + /** Ranking options for search. */ + readonly rankingOptions: OpenAI.Responses.FileSearchTool.RankingOptions | undefined; constructor({ vectorStoreIds = [], @@ -86,12 +133,23 @@ export class FileSearch extends OpenAITool { } } +/** Options for the OpenAI code interpreter tool. */ export interface CodeInterpreterOptions { - container?: string | Record | null; + /** + * Code interpreter container. Can be a container ID or an object that specifies uploaded file IDs + * to make available to the code. + */ + container?: OpenAI.Responses.Tool.CodeInterpreter['container'] | null; } +/** + * Run Python code to help generate a response to a prompt. + * + * @see https://platform.openai.com/docs/guides/tools-code-interpreter + */ export class CodeInterpreter extends OpenAITool { - readonly container: string | Record | null; + /** Code interpreter container ID or configuration. */ + readonly container: OpenAI.Responses.Tool.CodeInterpreter['container'] | null; constructor({ container = null }: CodeInterpreterOptions = {}) { super({ id: 'openai_code_interpreter' }); From 7648d6b1e83801698125c45d422d73d2c4e3244c Mon Sep 17 00:00:00 2001 From: "rosetta-livekit-bot[bot]" <282703043+rosetta-livekit-bot[bot]@users.noreply.github.com> Date: Tue, 26 May 2026 21:32:25 +0000 Subject: [PATCH 4/5] Align OpenAI tool docs tone --- plugins/openai/src/tools.ts | 49 +++++++++++++++++++------------------ 1 file changed, 25 insertions(+), 24 deletions(-) diff --git a/plugins/openai/src/tools.ts b/plugins/openai/src/tools.ts index ff273803b..3b20d42e8 100644 --- a/plugins/openai/src/tools.ts +++ b/plugins/openai/src/tools.ts @@ -4,31 +4,32 @@ import { llm } from '@livekit/agents'; import type OpenAI from 'openai'; -/** Base class for OpenAI Responses API provider tools. */ +/** A provider tool for the OpenAI Responses API. */ export abstract class OpenAITool extends llm.ProviderTool { - /** Convert this provider tool to the OpenAI Responses API tool configuration. */ + /** Convert the tool to an OpenAI Responses API tool configuration. */ abstract toToolConfig(): Record; } /** - * High-level guidance for the amount of context window space to use for web search. - * OpenAI defaults this to `medium`. + * High level guidance for the amount of context window space to use for the search. + * One of `low`, `medium`, or `high`. `medium` is the default. */ export type WebSearchContextSize = 'low' | 'medium' | 'high'; -/** Options for the OpenAI web search tool. */ +/** Options for the web search tool. */ export interface WebSearchOptions { /** - * Filters for the search, such as allowed domains. If not provided, all domains are allowed. + * Filters for the search. If `allowed_domains` is not provided, all domains are allowed. */ filters?: OpenAI.Responses.WebSearchTool['filters']; /** - * Amount of context window space to use for the search. Defaults to `medium`. + * High level guidance for the amount of context window space to use for the search. + * One of `low`, `medium`, or `high`. `medium` is the default. */ searchContextSize?: WebSearchContextSize | null; - /** Approximate location of the user, such as city, region, country, or timezone. */ + /** The approximate location of the user. */ userLocation?: OpenAI.Responses.WebSearchTool['user_location']; } @@ -38,13 +39,13 @@ export interface WebSearchOptions { * @see https://platform.openai.com/docs/guides/tools-web-search */ export class WebSearch extends OpenAITool { - /** Filters for the search, such as allowed domains. */ + /** Filters for the search. */ readonly filters: OpenAI.Responses.WebSearchTool['filters'] | undefined; - /** Amount of context window space to use for the search. */ + /** High level guidance for the amount of context window space to use for the search. */ readonly searchContextSize: WebSearchContextSize | null; - /** Approximate location of the user. */ + /** The approximate location of the user. */ readonly userLocation: OpenAI.Responses.WebSearchTool['user_location'] | undefined; constructor({ filters, searchContextSize = 'medium', userLocation }: WebSearchOptions = {}) { @@ -69,34 +70,34 @@ export class WebSearch extends OpenAITool { } } -/** Options for the OpenAI file search tool. */ +/** Options for the file search tool. */ export interface FileSearchOptions { - /** IDs of the vector stores to search. */ + /** The IDs of the vector stores to search. */ vectorStoreIds?: string[]; - /** Filter to apply to file search results. */ + /** A filter to apply. */ filters?: OpenAI.Responses.FileSearchTool['filters']; - /** Maximum number of results to return. This should be between 1 and 50 inclusive. */ + /** The maximum number of results to return. This number should be between 1 and 50 inclusive. */ maxNumResults?: number; - /** Ranking options for search, including ranker and score threshold. */ + /** Ranking options for search. */ rankingOptions?: OpenAI.Responses.FileSearchTool.RankingOptions; } /** - * Search for relevant content from uploaded files. + * A tool that searches for relevant content from uploaded files. * * @see https://platform.openai.com/docs/guides/tools-file-search */ export class FileSearch extends OpenAITool { - /** IDs of the vector stores to search. */ + /** The IDs of the vector stores to search. */ readonly vectorStoreIds: string[]; - /** Filter to apply to file search results. */ + /** A filter to apply. */ readonly filters: OpenAI.Responses.FileSearchTool['filters'] | undefined; - /** Maximum number of results to return. */ + /** The maximum number of results to return. */ readonly maxNumResults: number | undefined; /** Ranking options for search. */ @@ -133,22 +134,22 @@ export class FileSearch extends OpenAITool { } } -/** Options for the OpenAI code interpreter tool. */ +/** Options for the code interpreter tool. */ export interface CodeInterpreterOptions { /** - * Code interpreter container. Can be a container ID or an object that specifies uploaded file IDs + * The code interpreter container. Can be a container ID or an object that specifies uploaded file IDs * to make available to the code. */ container?: OpenAI.Responses.Tool.CodeInterpreter['container'] | null; } /** - * Run Python code to help generate a response to a prompt. + * A tool that runs Python code to help generate a response to a prompt. * * @see https://platform.openai.com/docs/guides/tools-code-interpreter */ export class CodeInterpreter extends OpenAITool { - /** Code interpreter container ID or configuration. */ + /** The code interpreter container. */ readonly container: OpenAI.Responses.Tool.CodeInterpreter['container'] | null; constructor({ container = null }: CodeInterpreterOptions = {}) { From 812f4cd0e37e99530ee13f7a871202aa2424c53e Mon Sep 17 00:00:00 2001 From: "rosetta-livekit-bot[bot]" <282703043+rosetta-livekit-bot[bot]@users.noreply.github.com> Date: Tue, 26 May 2026 21:44:08 +0000 Subject: [PATCH 5/5] Omit unset code interpreter container --- plugins/openai/src/tool_utils.test.ts | 6 ++++++ plugins/openai/src/tools.ts | 6 +++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/plugins/openai/src/tool_utils.test.ts b/plugins/openai/src/tool_utils.test.ts index f5d19a866..ce1922f52 100644 --- a/plugins/openai/src/tool_utils.test.ts +++ b/plugins/openai/src/tool_utils.test.ts @@ -68,6 +68,12 @@ describe('toResponsesTools', () => { ]); }); + it('omits the code interpreter container when unset', () => { + expect(toResponsesTools(new llm.ToolContext([new CodeInterpreter()]), false)).toEqual([ + { type: 'code_interpreter' }, + ]); + }); + it('ignores non-OpenAI provider tools', () => { class OtherProviderTool extends llm.ProviderTool {} diff --git a/plugins/openai/src/tools.ts b/plugins/openai/src/tools.ts index 3b20d42e8..21bdad043 100644 --- a/plugins/openai/src/tools.ts +++ b/plugins/openai/src/tools.ts @@ -158,6 +158,10 @@ export class CodeInterpreter extends OpenAITool { } toToolConfig(): Record { - return { type: 'code_interpreter', container: this.container }; + const result: Record = { type: 'code_interpreter' }; + if (this.container !== null) { + result.container = this.container; + } + return result; } }