Add OpenAI provider tools

.changeset/openai-provider-tools.md

@@ -0,0 +1,5 @@
+---
+'@livekit/agents-plugin-openai': minor
+---
+
+Add OpenAI Responses provider tools for web search, file search, and code interpreter.

plugins/openai/src/index.ts

@@ -5,6 +5,7 @@ import { Plugin } from '@livekit/agents';
 
 export { LLM, LLMStream, type LLMOptions } from './llm.js';
 export * from './models.js';
+export * from './tools.js';
 export * as realtime from './realtime/index.js';
 export * as responses from './responses/index.js';
 export { STT, type STTOptions } from './stt.js';

plugins/openai/src/responses/llm.ts

@@ -13,6 +13,7 @@ import {
 } from '@livekit/agents';
 import OpenAI from 'openai';
 import type { ChatModels } from '../models.js';
+import { toResponsesTools } from '../tool_utils.js';
 import { WSLLM } from '../ws/llm.js';
 
 export interface LLMOptions {
@@ -186,27 +187,8 @@ class ResponsesHttpLLMStream extends llm.LLMStream {
         'openai.responses',
       )) as OpenAI.Responses.ResponseInputItem[];
 
-      // TODO: support provider tools in the Responses schema.
       const tools = this.toolCtx
-        ? this.toolCtx
-            .flatten()
-            .filter(llm.isFunctionTool)
-            .map((t) => {
-              const oaiParams = {
-                type: 'function' as const,
-                name: t.name,
-                description: t.description,
-                parameters: llm.toJsonSchema(
-                  t.parameters,
-                  true,
-                  this.strictToolSchema,
-                ) as unknown as OpenAI.Responses.FunctionTool['parameters'],
-              } as OpenAI.Responses.FunctionTool;
-              if (this.strictToolSchema) {
-                oaiParams.strict = true;
-              }
-              return oaiParams;
-            })
+        ? toResponsesTools(this.toolCtx, this.strictToolSchema)
         : undefined;
 
       const requestOptions: Record<string, unknown> = { ...this.modelOptions };

plugins/openai/src/tool_utils.test.ts

@@ -0,0 +1,84 @@
+// SPDX-FileCopyrightText: 2026 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { llm } from '@livekit/agents';
+import { describe, expect, it } from 'vitest';
+import { z } from 'zod';
+import { toResponsesTools } from './tool_utils.js';
+import { CodeInterpreter, FileSearch, WebSearch } from './tools.js';
+
+describe('toResponsesTools', () => {
+  it('serializes function tools', () => {
+    const fn = llm.tool({
+      name: 'lookup_weather',
+      description: 'Look up weather',
+      parameters: z.object({ city: z.string() }),
+      execute: async () => 'sunny',
+    });
+
+    expect(toResponsesTools(new llm.ToolContext([fn]), true)).toEqual([
+      {
+        type: 'function',
+        name: 'lookup_weather',
+        description: 'Look up weather',
+        parameters: {
+          $schema: 'http://json-schema.org/draft-07/schema#',
+          type: 'object',
+          properties: { city: { type: 'string' } },
+          required: ['city'],
+          additionalProperties: false,
+        },
+        strict: true,
+      },
+    ]);
+  });
+
+  it('serializes OpenAI provider tools', () => {
+    const tools = toResponsesTools(
+      new llm.ToolContext([
+        new WebSearch({
+          filters: { allowed_domains: ['docs.livekit.io'] },
+          searchContextSize: 'low',
+          userLocation: { type: 'approximate', country: 'US' },
+        }),
+        new FileSearch({
+          vectorStoreIds: ['vs_123'],
+          maxNumResults: 3,
+          rankingOptions: { ranker: 'auto' },
+        }),
+        new CodeInterpreter({ container: { type: 'auto', file_ids: ['file_123'] } }),
+      ]),
+      false,
+    );
+
+    expect(tools).toEqual([
+      {
+        type: 'web_search',
+        search_context_size: 'low',
+        filters: { allowed_domains: ['docs.livekit.io'] },
+        user_location: { type: 'approximate', country: 'US' },
+      },
+      {
+        type: 'file_search',
+        vector_store_ids: ['vs_123'],
+        max_num_results: 3,
+        ranking_options: { ranker: 'auto' },
+      },
+      { type: 'code_interpreter', container: { type: 'auto', file_ids: ['file_123'] } },
+    ]);
+  });
+
+  it('omits the code interpreter container when unset', () => {
+    expect(toResponsesTools(new llm.ToolContext([new CodeInterpreter()]), false)).toEqual([
+      { type: 'code_interpreter' },
+    ]);
+  });
+
+  it('ignores non-OpenAI provider tools', () => {
+    class OtherProviderTool extends llm.ProviderTool {}
+
+    expect(
+      toResponsesTools(new llm.ToolContext([new OtherProviderTool({ id: 'other' })]), false),
+    ).toBeUndefined();
+  });
+});

plugins/openai/src/tool_utils.ts

@@ -0,0 +1,43 @@
+// SPDX-FileCopyrightText: 2026 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { llm } from '@livekit/agents';
+import type OpenAI from 'openai';
+import { OpenAITool } from './tools.js';
+
+export function toResponsesTools(
+  toolCtx: llm.ToolContext,
+  strictToolSchema: boolean,
+): OpenAI.Responses.Tool[] | undefined {
+  const tools = toolCtx
+    .flatten()
+    .map((tool) => {
+      if (llm.isFunctionTool(tool)) {
+        const oaiParams = {
+          type: 'function' as const,
+          name: tool.name,
+          description: tool.description,
+          parameters: llm.toJsonSchema(
+            tool.parameters,
+            true,
+            strictToolSchema,
+          ) as unknown as OpenAI.Responses.FunctionTool['parameters'],
+        } as OpenAI.Responses.FunctionTool;
+
+        if (strictToolSchema) {
+          oaiParams.strict = true;
+        }
+
+        return oaiParams;
+      }
+
+      if (tool instanceof OpenAITool) {
+        return tool.toToolConfig() as unknown as OpenAI.Responses.Tool;
+      }
+
+      return undefined;
+    })
+    .filter((tool): tool is OpenAI.Responses.Tool => tool !== undefined);
+
+  return tools.length > 0 ? tools : undefined;
+}

plugins/openai/src/tools.ts

@@ -0,0 +1,167 @@
+// SPDX-FileCopyrightText: 2026 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { llm } from '@livekit/agents';
+import type OpenAI from 'openai';
+
+/** A provider tool for the OpenAI Responses API. */
+export abstract class OpenAITool extends llm.ProviderTool {
+  /** Convert the tool to an OpenAI Responses API tool configuration. */
+  abstract toToolConfig(): Record<string, unknown>;
+}
+
+/**
+ * High level guidance for the amount of context window space to use for the search.
+ * One of `low`, `medium`, or `high`. `medium` is the default.
+ */
+export type WebSearchContextSize = 'low' | 'medium' | 'high';
+
+/** Options for the web search tool. */
+export interface WebSearchOptions {
+  /**
+   * Filters for the search. If `allowed_domains` is not provided, all domains are allowed.
+   */
+  filters?: OpenAI.Responses.WebSearchTool['filters'];
+
+  /**
+   * High level guidance for the amount of context window space to use for the search.
+   * One of `low`, `medium`, or `high`. `medium` is the default.
+   */
+  searchContextSize?: WebSearchContextSize | null;
+
+  /** The approximate location of the user. */
+  userLocation?: OpenAI.Responses.WebSearchTool['user_location'];
+}
+
+/**
+ * Search the Internet for sources related to the prompt.
+ *
+ * @see https://platform.openai.com/docs/guides/tools-web-search
+ */
+export class WebSearch extends OpenAITool {
+  /** Filters for the search. */
+  readonly filters: OpenAI.Responses.WebSearchTool['filters'] | undefined;
+
+  /** High level guidance for the amount of context window space to use for the search. */
+  readonly searchContextSize: WebSearchContextSize | null;
+
+  /** The approximate location of the user. */
+  readonly userLocation: OpenAI.Responses.WebSearchTool['user_location'] | undefined;
+
+  constructor({ filters, searchContextSize = 'medium', userLocation }: WebSearchOptions = {}) {
+    super({ id: 'openai_web_search' });
+    this.filters = filters;
+    this.searchContextSize = searchContextSize;
+    this.userLocation = userLocation;
+  }
+
+  toToolConfig(): Record<string, unknown> {
+    const result: Record<string, unknown> = {
+      type: 'web_search',
+      search_context_size: this.searchContextSize,
+    };
+    if (this.userLocation !== undefined) {
+      result.user_location = this.userLocation;
+    }
+    if (this.filters !== undefined) {
+      result.filters = this.filters;
+    }
+    return result;
+  }
+}
+
+/** Options for the file search tool. */
+export interface FileSearchOptions {
+  /** The IDs of the vector stores to search. */
+  vectorStoreIds?: string[];
+
+  /** A filter to apply. */
+  filters?: OpenAI.Responses.FileSearchTool['filters'];
+
+  /** The maximum number of results to return. This number should be between 1 and 50 inclusive. */
+  maxNumResults?: number;
+
+  /** Ranking options for search. */
+  rankingOptions?: OpenAI.Responses.FileSearchTool.RankingOptions;
+}
+
+/**
+ * A tool that searches for relevant content from uploaded files.
+ *
+ * @see https://platform.openai.com/docs/guides/tools-file-search
+ */
+export class FileSearch extends OpenAITool {
+  /** The IDs of the vector stores to search. */
+  readonly vectorStoreIds: string[];
+
+  /** A filter to apply. */
+  readonly filters: OpenAI.Responses.FileSearchTool['filters'] | undefined;
+
+  /** The maximum number of results to return. */
+  readonly maxNumResults: number | undefined;
+
+  /** Ranking options for search. */
+  readonly rankingOptions: OpenAI.Responses.FileSearchTool.RankingOptions | undefined;
+
+  constructor({
+    vectorStoreIds = [],
+    filters,
+    maxNumResults,
+    rankingOptions,
+  }: FileSearchOptions = {}) {
+    super({ id: 'openai_file_search' });
+    this.vectorStoreIds = [...vectorStoreIds];
+    this.filters = filters;
+    this.maxNumResults = maxNumResults;
+    this.rankingOptions = rankingOptions;
+  }
+
+  toToolConfig(): Record<string, unknown> {
+    const result: Record<string, unknown> = {
+      type: 'file_search',
+      vector_store_ids: this.vectorStoreIds,
+    };
+    if (this.filters !== undefined) {
+      result.filters = this.filters;
+    }
+    if (this.maxNumResults !== undefined) {
+      result.max_num_results = this.maxNumResults;
+    }
+    if (this.rankingOptions !== undefined) {
+      result.ranking_options = this.rankingOptions;
+    }
+    return result;
+  }
+}
+
+/** Options for the code interpreter tool. */
+export interface CodeInterpreterOptions {
+  /**
+   * The code interpreter container. Can be a container ID or an object that specifies uploaded file IDs
+   * to make available to the code.
+   */
+  container?: OpenAI.Responses.Tool.CodeInterpreter['container'] | null;
+}
+
+/**
+ * A tool that runs Python code to help generate a response to a prompt.
+ *
+ * @see https://platform.openai.com/docs/guides/tools-code-interpreter
+ */
+export class CodeInterpreter extends OpenAITool {
+  /** The code interpreter container. */
+  readonly container: OpenAI.Responses.Tool.CodeInterpreter['container'] | null;
+
+  constructor({ container = null }: CodeInterpreterOptions = {}) {
+    super({ id: 'openai_code_interpreter' });
+    this.container = container;
+  }
+
+  toToolConfig(): Record<string, unknown> {
+    const result: Record<string, unknown> = { type: 'code_interpreter' };
+    if (this.container !== null) {
+      result.container = this.container;
+    }
+    return result;
+  }
+}

plugins/openai/src/ws/llm.ts

@@ -15,6 +15,7 @@ import {
 import type OpenAI from 'openai';
 import { WebSocket } from 'ws';
 import type { ChatModels } from '../models.js';
+import { toResponsesTools } from '../tool_utils.js';
 import type {
   WsOutputItemDoneEvent,
   WsOutputTextDeltaEvent,
@@ -429,30 +430,7 @@ export class WSLLMStream extends llm.LLMStream {
       'openai.responses',
     )) as OpenAI.Responses.ResponseInputItem[];
 
-    // TODO: support provider tools in the Responses schema.
-    const tools = this.toolCtx
-      ? this.toolCtx
-          .flatten()
-          .filter(llm.isFunctionTool)
-          .map((t) => {
-            const oaiParams = {
-              type: 'function' as const,
-              name: t.name,
-              description: t.description,
-              parameters: llm.toJsonSchema(
-                t.parameters,
-                true,
-                this.#strictToolSchema,
-              ) as unknown as OpenAI.Responses.FunctionTool['parameters'],
-            } as OpenAI.Responses.FunctionTool;
-
-            if (this.#strictToolSchema) {
-              oaiParams.strict = true;
-            }
-
-            return oaiParams;
-          })
-      : undefined;
+    const tools = this.toolCtx ? toResponsesTools(this.toolCtx, this.#strictToolSchema) : undefined;
 
     const requestOptions: Record<string, unknown> = { ...this.#modelOptions };
     if (!tools) {