Add Gemini provider tools

.changeset/gemini-provider-tools.md

@@ -0,0 +1,5 @@
+---
+'@livekit/agents-plugin-google': minor
+---
+
+Add Gemini provider tools for Google Search, Google Maps, URL context, File Search, code execution, and Vertex RAG retrieval, and serialize them from `ToolContext` for Google LLM and realtime sessions.

plugins/google/src/beta/realtime/realtime_api.ts

@@ -33,7 +33,7 @@ import {
 import { Mutex } from '@livekit/mutex';
 import { AudioFrame, AudioResampler, type VideoFrame } from '@livekit/rtc-node';
 import { type LLMTools } from '../../tools.js';
-import { toFunctionDeclarations } from '../../utils.js';
+import { toToolsConfig } from '../../utils.js';
 import type * as api_proto from './api_proto.js';
 import type { LiveAPIModels, Voice } from './api_proto.js';
 
@@ -70,13 +70,6 @@ export interface InputTranscription {
   transcript: string;
 }
 
-/**
- * Helper function to check if two sets are equal
- */
-function setsEqual<T>(a: Set<T>, b: Set<T>): boolean {
-  return a.size === b.size && [...a].every((x) => b.has(x));
-}
-
 /**
  * Internal realtime options for Google Realtime API
  */
@@ -455,7 +448,6 @@ export class RealtimeSession extends llm.RealtimeSession {
   private _chatCtx = llm.ChatContext.empty();
 
   private options: RealtimeOptions;
-  private geminiDeclarations: types.FunctionDeclaration[] = [];
   private messageChannel = new Queue<api_proto.ClientEvents>();
   private inputResampler?: AudioResampler;
   private inputResamplerInputRate?: number;
@@ -764,15 +756,12 @@ export class RealtimeSession extends llm.RealtimeSession {
   }
 
   async updateTools(tools: llm.ToolContext): Promise<void> {
-    const newDeclarations = toFunctionDeclarations(tools);
-    const currentToolNames = new Set(this.geminiDeclarations.map((f) => f.name));
-    const newToolNames = new Set(newDeclarations.map((f) => f.name));
-
-    if (!setsEqual(currentToolNames, newToolNames)) {
-      this.geminiDeclarations = newDeclarations;
-      this._tools = tools;
-      this.markRestartNeeded();
+    if (this._tools.equals(tools)) {
+      return;
     }
+
+    this._tools = tools;
+    this.markRestartNeeded();
   }
 
   get chatCtx(): llm.ChatContext {
@@ -1424,21 +1413,11 @@ export class RealtimeSession extends llm.RealtimeSession {
         },
         languageCode: opts.language,
       },
-      tools:
-        this.geminiDeclarations.length > 0 || this.options.geminiTools
-          ? [
-              {
-                functionDeclarations:
-                  this.options.toolBehavior !== undefined
-                    ? this.geminiDeclarations.map((d) => ({
-                        ...d,
-                        behavior: this.options.toolBehavior,
-                      }))
-                    : this.geminiDeclarations,
-                ...this.options.geminiTools,
-              },
-            ]
-          : undefined,
+      tools: toToolsConfig({
+        toolCtx: this._tools,
+        geminiTools: this.options.geminiTools,
+        toolBehavior: this.options.toolBehavior,
+      }),
       inputAudioTranscription: opts.inputAudioTranscription,
       outputAudioTranscription: opts.outputAudioTranscription,
       sessionResumption: this.sessionResumptionHandle

plugins/google/src/index.ts

@@ -6,6 +6,7 @@ import { Plugin } from '@livekit/agents';
 export * as beta from './beta/index.js';
 export { LLM, LLMStream, type LLMOptions } from './llm.js';
 export * from './models.js';
+export * from './tools.js';
 
 class GooglePlugin extends Plugin {
   constructor() {

plugins/google/src/llm.ts

@@ -13,7 +13,7 @@ import {
 } from '@livekit/agents';
 import type { ChatModels } from './models.js';
 import type { LLMTools } from './tools.js';
-import { toFunctionDeclarations } from './utils.js';
+import { toToolsConfig } from './utils.js';
 
 interface GoogleFormatData {
   systemMessages: string[] | null;
@@ -355,11 +355,11 @@ export class LLMStream extends llm.LLMStream {
         parts: turn.parts as types.Part[],
       }));
 
-      const functionDeclarations = this.toolCtx ? toFunctionDeclarations(this.toolCtx) : undefined;
-      const tools =
-        functionDeclarations && functionDeclarations.length > 0
-          ? [{ functionDeclarations }]
-          : undefined;
+      const tools = toToolsConfig({
+        toolCtx: this.toolCtx,
+        geminiTools: this.#geminiTools,
+        onlySingleType: true,
+      });
 
       let systemInstruction: types.Content | undefined = undefined;
       if (extraData.systemMessages && extraData.systemMessages.length > 0) {

plugins/google/src/tools.ts

@@ -1,6 +1,100 @@
 // SPDX-FileCopyrightText: 2025 LiveKit, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
-import type { Tool } from '@google/genai';
+import type * as types from '@google/genai';
+import { llm } from '@livekit/agents';
 
-export type LLMTools = Omit<Tool, 'functionDeclarations'>;
+export type LLMTools = Omit<types.Tool, 'functionDeclarations'>;
+
+export abstract class GeminiTool extends llm.ProviderTool {
+  abstract toToolConfig(): types.Tool;
+}
+
+export class GoogleSearch extends GeminiTool {
+  constructor(public readonly options: types.GoogleSearch = {}) {
+    super({ id: 'gemini_google_search' });
+  }
+
+  toToolConfig(): types.Tool {
+    return { googleSearch: this.options };
+  }
+}
+
+export class GoogleMaps extends GeminiTool {
+  constructor(public readonly options: types.GoogleMaps = {}) {
+    super({ id: 'gemini_google_maps' });
+  }
+
+  toToolConfig(): types.Tool {
+    return { googleMaps: this.options };
+  }
+}
+
+export class URLContext extends GeminiTool {
+  constructor() {
+    super({ id: 'gemini_url_context' });
+  }
+
+  toToolConfig(): types.Tool {
+    return { urlContext: {} };
+  }
+}
+
+export interface FileSearchOptions extends types.FileSearch {
+  fileSearchStoreNames: string[];
+}
+
+export class FileSearch extends GeminiTool {
+  constructor(public readonly options: FileSearchOptions) {
+    super({ id: 'gemini_file_search' });
+  }
+
+  toToolConfig(): types.Tool {
+    return { fileSearch: this.options };
+  }
+}
+
+export class ToolCodeExecution extends GeminiTool {
+  constructor() {
+    super({ id: 'gemini_code_execution' });
+  }
+
+  toToolConfig(): types.Tool {
+    return { codeExecution: {} };
+  }
+}
+
+export interface VertexRAGRetrievalOptions {
+  ragResources: string[];
+  similarityTopK?: number;
+  vectorDistanceThreshold?: number;
+}
+
+export class VertexRAGRetrieval extends GeminiTool {
+  readonly ragResources: string[];
+  readonly similarityTopK: number;
+  readonly vectorDistanceThreshold?: number;
+
+  constructor({
+    ragResources,
+    similarityTopK = 3,
+    vectorDistanceThreshold,
+  }: VertexRAGRetrievalOptions) {
+    super({ id: 'gemini_vertex_rag_retrieval' });
+    this.ragResources = ragResources;
+    this.similarityTopK = similarityTopK;
+    this.vectorDistanceThreshold = vectorDistanceThreshold;
+  }
+
+  toToolConfig(): types.Tool {
+    return {
+      retrieval: {
+        vertexRagStore: {
+          ragResources: this.ragResources.map((ragCorpus) => ({ ragCorpus })),
+          similarityTopK: this.similarityTopK,
+          vectorDistanceThreshold: this.vectorDistanceThreshold,
+        },
+      },
+    };
+  }
+}

plugins/google/src/utils.ts

@@ -1,9 +1,11 @@
 // SPDX-FileCopyrightText: 2025 LiveKit, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
+import type * as types from '@google/genai';
 import type { FunctionDeclaration, Schema } from '@google/genai';
 import { llm } from '@livekit/agents';
 import type { JSONSchema7 } from 'json-schema';
+import { GeminiTool, type LLMTools } from './tools.js';
 
 /**
  * JSON Schema v7
@@ -157,3 +159,57 @@ export function toFunctionDeclarations(toolCtx: llm.ToolContext): FunctionDeclar
 
   return functionDeclarations;
 }
+
+export function toToolsConfig({
+  toolCtx,
+  geminiTools,
+  toolBehavior,
+  onlySingleType = false,
+}: {
+  toolCtx?: llm.ToolContext;
+  geminiTools?: LLMTools;
+  toolBehavior?: types.Behavior;
+  onlySingleType?: boolean;
+}): types.Tool[] | undefined {
+  const tools: types.Tool[] = [];
+  const providerTools: types.Tool[] = [];
+
+  if (toolCtx) {
+    const functionDeclarations = toFunctionDeclarations(toolCtx);
+    if (functionDeclarations.length > 0) {
+      tools.push({
+        functionDeclarations:
+          toolBehavior !== undefined
+            ? functionDeclarations.map((declaration) => ({
+                ...declaration,
+                behavior: toolBehavior,
+              }))
+            : functionDeclarations,
+      });
+    }
+  }
+
+  if (geminiTools !== undefined) {
+    providerTools.push(geminiTools);
+  }
+
+  if (toolCtx) {
+    for (const tool of toolCtx.providerTools) {
+      if (tool instanceof GeminiTool) {
+        providerTools.push(tool.toToolConfig());
+      }
+    }
+  }
+
+  if (tools.length > 0 && providerTools.length > 0) {
+    throw new Error('Gemini does not support mixing function tools and provider tools');
+  }
+
+  if (onlySingleType && tools.length > 0) {
+    return tools;
+  }
+
+  tools.push(...providerTools);
+
+  return tools.length > 0 ? tools : undefined;
+}