Add xAI realtime provider tools

.changeset/xai-provider-tools.md

@@ -0,0 +1,6 @@
+---
+'@livekit/agents-plugin-openai': patch
+'@livekit/agents-plugin-xai': minor
+---
+
+Add xAI Realtime provider tools for web search, X search, and file search.

plugins/openai/src/realtime/realtime_model.ts

@@ -697,7 +697,9 @@ export class RealtimeSession extends llm.RealtimeSession {
 
     // TODO(brian): these logics below are noops I think, leaving it here to keep
     // parity with the python but we should remove them later
-    const retainedToolNames = new Set(ev.session.tools.map((tool) => tool.name));
+    const retainedToolNames = new Set(
+      ev.session.tools.filter((tool) => tool.type === 'function').map((tool) => tool.name),
+    );
     // Keep provider tools and Toolsets as-is; only drop function tools the server didn't accept.
     const retainedEntries = _tools.tools.filter(
       (entry) => !llm.isFunctionTool(entry) || retainedToolNames.has(entry.name),
@@ -708,7 +710,7 @@ export class RealtimeSession extends llm.RealtimeSession {
     unlock();
   }
 
-  private createToolsUpdateEvent(_tools: llm.ToolContext): api_proto.SessionUpdateEvent {
+  protected createToolsUpdateEvent(_tools: llm.ToolContext): api_proto.SessionUpdateEvent {
     const oaiTools: api_proto.Tool[] = [];
 
     for (const t of _tools.flatten()) {
@@ -1637,6 +1639,10 @@ export class RealtimeSession extends llm.RealtimeSession {
       if (!item.call_id || !item.name || !item.arguments) {
         throw new Error('item is not a function call');
       }
+      if (!this._tools.getFunctionTool(item.name)) {
+        this.#logger.warn({ name: item.name }, 'unknown function tool, ignoring');
+        return;
+      }
       this.currentGeneration.functionChannel.write(
         llm.FunctionCall.create({
           callId: item.call_id,

plugins/xai/src/index.ts

@@ -5,6 +5,7 @@ import { Plugin } from '@livekit/agents';
 
 export * as realtime from './realtime/index.js';
 export { STT, type STTOptions, type STTLanguages } from './stt.js';
+export * from './tools.js';
 
 class XAIPlugin extends Plugin {
   constructor() {

plugins/xai/src/realtime/realtime_model.ts

@@ -1,9 +1,11 @@
 // SPDX-FileCopyrightText: 2024 LiveKit, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
+import { llm } from '@livekit/agents';
 import { realtime } from '@livekit/agents-plugin-openai';
+import { XAITool } from '../tools.js';
 
-const { RealtimeModel: OpenAIRealtimeModel } = realtime;
+const { RealtimeModel: OpenAIRealtimeModel, RealtimeSession: OpenAIRealtimeSession } = realtime;
 type OpenAIRealtimeModelOptions = ConstructorParameters<typeof OpenAIRealtimeModel>[0];
 
 const XAI_BASE_URL = 'wss://api.x.ai/v1';
@@ -46,5 +48,23 @@ export class RealtimeModel extends OpenAIRealtimeModel {
       turnDetection: XAI_DEFAULT_TURN_DETECTION,
       ...options,
     });
+    this.capabilities.perResponseToolChoice = false;
+  }
+
+  override session() {
+    return new RealtimeSession(this);
+  }
+}
+
+export class RealtimeSession extends OpenAIRealtimeSession {
+  protected override createToolsUpdateEvent(tools: llm.ToolContext): realtime.SessionUpdateEvent {
+    const event = super.createToolsUpdateEvent(tools);
+    const xaiTools = tools
+      .flatten()
+      .filter((tool): tool is XAITool => tool instanceof XAITool)
+      .map((tool) => tool.toToolConfig());
+
+    event.session.tools = [...(event.session.tools ?? []), ...xaiTools] as realtime.Tool[];
+    return event;
   }
 }

plugins/xai/src/tools.ts

@@ -0,0 +1,69 @@
+// SPDX-FileCopyrightText: 2026 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { llm } from '@livekit/agents';
+
+export abstract class XAITool extends llm.ProviderTool {
+  abstract toToolConfig(): Record<string, unknown>;
+}
+
+/** Enable web search tool for real-time internet searches. */
+export class WebSearch extends XAITool {
+  constructor() {
+    super({ id: 'xai_web_search' });
+  }
+
+  toToolConfig(): Record<string, unknown> {
+    return { type: 'web_search' };
+  }
+}
+
+export interface XSearchOptions {
+  allowedXHandles?: string[];
+}
+
+/** Enable X search tool for searching posts. */
+export class XSearch extends XAITool {
+  readonly allowedXHandles: string[] | undefined;
+
+  constructor({ allowedXHandles }: XSearchOptions = {}) {
+    super({ id: 'xai_x_search' });
+    this.allowedXHandles = allowedXHandles ? [...allowedXHandles] : undefined;
+  }
+
+  toToolConfig(): Record<string, unknown> {
+    const result: Record<string, unknown> = { type: 'x_search' };
+    if (this.allowedXHandles !== undefined) {
+      result.allowed_x_handles = this.allowedXHandles;
+    }
+    return result;
+  }
+}
+
+export interface FileSearchOptions {
+  vectorStoreIds?: string[];
+  maxNumResults?: number;
+}
+
+/** Enable file search tool for searching uploaded document collections. */
+export class FileSearch extends XAITool {
+  readonly vectorStoreIds: string[];
+  readonly maxNumResults: number | undefined;
+
+  constructor({ vectorStoreIds = [], maxNumResults }: FileSearchOptions = {}) {
+    super({ id: 'xai_file_search' });
+    this.vectorStoreIds = [...vectorStoreIds];
+    this.maxNumResults = maxNumResults;
+  }
+
+  toToolConfig(): Record<string, unknown> {
+    const result: Record<string, unknown> = {
+      type: 'file_search',
+      vector_store_ids: this.vectorStoreIds,
+    };
+    if (this.maxNumResults !== undefined) {
+      result.max_num_results = this.maxNumResults;
+    }
+    return result;
+  }
+}