Add OpenAI provider tools

.changeset/openai-provider-tools.md

@@ -0,0 +1,5 @@
+---
+'@livekit/agents-plugin-openai': minor
+---
+
+Add OpenAI Responses provider tools for web search, file search, and code interpreter.

plugins/openai/src/index.ts

@@ -5,6 +5,7 @@ import { Plugin } from '@livekit/agents';
 
 export { LLM, LLMStream, type LLMOptions } from './llm.js';
 export * from './models.js';
+export * from './tools.js';
 export * as realtime from './realtime/index.js';
 export * as responses from './responses/index.js';
 export { STT, type STTOptions } from './stt.js';

plugins/openai/src/responses/llm.ts

@@ -13,6 +13,7 @@ import {
 } from '@livekit/agents';
 import OpenAI from 'openai';
 import type { ChatModels } from '../models.js';
+import { toResponsesTools } from '../tool_utils.js';
 import { WSLLM } from '../ws/llm.js';
 
 export interface LLMOptions {
@@ -186,27 +187,8 @@ class ResponsesHttpLLMStream extends llm.LLMStream {
         'openai.responses',
       )) as OpenAI.Responses.ResponseInputItem[];
 
-      // TODO: support provider tools in the Responses schema.
       const tools = this.toolCtx
-        ? this.toolCtx
-            .flatten()
-            .filter(llm.isFunctionTool)
-            .map((t) => {
-              const oaiParams = {
-                type: 'function' as const,
-                name: t.name,
-                description: t.description,
-                parameters: llm.toJsonSchema(
-                  t.parameters,
-                  true,
-                  this.strictToolSchema,
-                ) as unknown as OpenAI.Responses.FunctionTool['parameters'],
-              } as OpenAI.Responses.FunctionTool;
-              if (this.strictToolSchema) {
-                oaiParams.strict = true;
-              }
-              return oaiParams;
-            })
+        ? toResponsesTools(this.toolCtx, this.strictToolSchema)
         : undefined;
 
       const requestOptions: Record<string, unknown> = { ...this.modelOptions };

plugins/openai/src/tool_utils.ts

@@ -0,0 +1,43 @@
+// SPDX-FileCopyrightText: 2026 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { llm } from '@livekit/agents';
+import type OpenAI from 'openai';
+import { OpenAITool } from './tools.js';
+
+export function toResponsesTools(
+  toolCtx: llm.ToolContext,
+  strictToolSchema: boolean,
+): OpenAI.Responses.Tool[] | undefined {
+  const tools = toolCtx
+    .flatten()
+    .map((tool) => {
+      if (llm.isFunctionTool(tool)) {
+        const oaiParams = {
+          type: 'function' as const,
+          name: tool.name,
+          description: tool.description,
+          parameters: llm.toJsonSchema(
+            tool.parameters,
+            true,
+            strictToolSchema,
+          ) as unknown as OpenAI.Responses.FunctionTool['parameters'],
+        } as OpenAI.Responses.FunctionTool;
+
+        if (strictToolSchema) {
+          oaiParams.strict = true;
+        }
+
+        return oaiParams;
+      }
+
+      if (tool instanceof OpenAITool) {
+        return tool.toToolConfig() as unknown as OpenAI.Responses.Tool;
+      }
+
+      return undefined;
+    })
+    .filter((tool): tool is OpenAI.Responses.Tool => tool !== undefined);
+
+  return tools.length > 0 ? tools : undefined;
+}

plugins/openai/src/tools.ts

@@ -0,0 +1,104 @@
+// SPDX-FileCopyrightText: 2026 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { llm } from '@livekit/agents';
+
+export abstract class OpenAITool extends llm.ProviderTool {
+  abstract toToolConfig(): Record<string, unknown>;
+}
+
+export type WebSearchContextSize = 'low' | 'medium' | 'high';
+
+export interface WebSearchOptions {
+  filters?: Record<string, unknown>;
+  searchContextSize?: WebSearchContextSize | null;
+  userLocation?: Record<string, unknown>;
+}
+
+export class WebSearch extends OpenAITool {
+  readonly filters: Record<string, unknown> | undefined;
+  readonly searchContextSize: WebSearchContextSize | null;
+  readonly userLocation: Record<string, unknown> | undefined;
+
+  constructor({ filters, searchContextSize = 'medium', userLocation }: WebSearchOptions = {}) {
+    super({ id: 'openai_web_search' });
+    this.filters = filters;
+    this.searchContextSize = searchContextSize;
+    this.userLocation = userLocation;
+  }
+
+  toToolConfig(): Record<string, unknown> {
+    const result: Record<string, unknown> = {
+      type: 'web_search',
+      search_context_size: this.searchContextSize,
+    };
+    if (this.userLocation !== undefined) {
+      result.user_location = this.userLocation;
+    }
+    if (this.filters !== undefined) {
+      result.filters = this.filters;
+    }
+    return result;
+  }
+}
+
+export interface FileSearchOptions {
+  vectorStoreIds?: string[];
+  filters?: Record<string, unknown>;
+  maxNumResults?: number;
+  rankingOptions?: Record<string, unknown>;
+}
+
+export class FileSearch extends OpenAITool {
+  readonly vectorStoreIds: string[];
+  readonly filters: Record<string, unknown> | undefined;
+  readonly maxNumResults: number | undefined;
+  readonly rankingOptions: Record<string, unknown> | undefined;
+
+  constructor({
+    vectorStoreIds = [],
+    filters,
+    maxNumResults,
+    rankingOptions,
+  }: FileSearchOptions = {}) {
+    super({ id: 'openai_file_search' });
+    this.vectorStoreIds = [...vectorStoreIds];
+    this.filters = filters;
+    this.maxNumResults = maxNumResults;
+    this.rankingOptions = rankingOptions;
+  }
+
+  toToolConfig(): Record<string, unknown> {
+    const result: Record<string, unknown> = {
+      type: 'file_search',
+      vector_store_ids: this.vectorStoreIds,
+    };
+    if (this.filters !== undefined) {
+      result.filters = this.filters;
+    }
+    if (this.maxNumResults !== undefined) {
+      result.max_num_results = this.maxNumResults;
+    }
+    if (this.rankingOptions !== undefined) {
+      result.ranking_options = this.rankingOptions;
+    }
+    return result;
+  }
+}
+
+export interface CodeInterpreterOptions {
+  container?: string | Record<string, unknown> | null;
+}
+
+export class CodeInterpreter extends OpenAITool {
+  readonly container: string | Record<string, unknown> | null;
+
+  constructor({ container = null }: CodeInterpreterOptions = {}) {
+    super({ id: 'openai_code_interpreter' });
+    this.container = container;
+  }
+
+  toToolConfig(): Record<string, unknown> {
+    return { type: 'code_interpreter', container: this.container };
+  }
+}

plugins/openai/src/ws/llm.ts

@@ -15,6 +15,7 @@ import {
 import type OpenAI from 'openai';
 import { WebSocket } from 'ws';
 import type { ChatModels } from '../models.js';
+import { toResponsesTools } from '../tool_utils.js';
 import type {
   WsOutputItemDoneEvent,
   WsOutputTextDeltaEvent,
@@ -429,30 +430,7 @@ export class WSLLMStream extends llm.LLMStream {
       'openai.responses',
     )) as OpenAI.Responses.ResponseInputItem[];
 
-    // TODO: support provider tools in the Responses schema.
-    const tools = this.toolCtx
-      ? this.toolCtx
-          .flatten()
-          .filter(llm.isFunctionTool)
-          .map((t) => {
-            const oaiParams = {
-              type: 'function' as const,
-              name: t.name,
-              description: t.description,
-              parameters: llm.toJsonSchema(
-                t.parameters,
-                true,
-                this.#strictToolSchema,
-              ) as unknown as OpenAI.Responses.FunctionTool['parameters'],
-            } as OpenAI.Responses.FunctionTool;
-
-            if (this.#strictToolSchema) {
-              oaiParams.strict = true;
-            }
-
-            return oaiParams;
-          })
-      : undefined;
+    const tools = this.toolCtx ? toResponsesTools(this.toolCtx, this.#strictToolSchema) : undefined;
 
     const requestOptions: Record<string, unknown> = { ...this.#modelOptions };
     if (!tools) {