From 881477949982614fe1c068a7e0a3ccec901ba04e Mon Sep 17 00:00:00 2001 From: Krishna Shukla Date: Wed, 20 May 2026 05:37:39 +0530 Subject: [PATCH 01/12] feat(plugin-anthropic): add Anthropic LLM provider with Claude 3.5 tool support --- plugins/anthropic/README.md | 22 ++ plugins/anthropic/api-extractor.json | 20 ++ plugins/anthropic/package.json | 51 ++++ plugins/anthropic/src/index.ts | 19 ++ plugins/anthropic/src/llm.test.ts | 109 +++++++ plugins/anthropic/src/llm.ts | 405 +++++++++++++++++++++++++++ plugins/anthropic/src/models.ts | 21 ++ plugins/anthropic/tsconfig.json | 19 ++ plugins/anthropic/tsup.config.ts | 7 + pnpm-lock.yaml | 122 +++++++- 10 files changed, 794 insertions(+), 1 deletion(-) create mode 100644 plugins/anthropic/README.md create mode 100644 plugins/anthropic/api-extractor.json create mode 100644 plugins/anthropic/package.json create mode 100644 plugins/anthropic/src/index.ts create mode 100644 plugins/anthropic/src/llm.test.ts create mode 100644 plugins/anthropic/src/llm.ts create mode 100644 plugins/anthropic/src/models.ts create mode 100644 plugins/anthropic/tsconfig.json create mode 100644 plugins/anthropic/tsup.config.ts diff --git a/plugins/anthropic/README.md b/plugins/anthropic/README.md new file mode 100644 index 000000000..3d88b605e --- /dev/null +++ b/plugins/anthropic/README.md @@ -0,0 +1,22 @@ +# @livekit/agents-plugin-anthropic + +Anthropic plugin for LiveKit Node Agents. + +## Installation + +```bash +npm install @livekit/agents-plugin-anthropic +``` + +## Usage + +```typescript +import { anthropic } from '@livekit/agents-plugin-anthropic'; + +const agent = new Agent({ + llm: new anthropic.LLM({ + model: 'claude-3-5-sonnet-20241022', + // caching: 'ephemeral' // uncomment to enable prompt caching + }), +}); +``` diff --git a/plugins/anthropic/api-extractor.json b/plugins/anthropic/api-extractor.json new file mode 100644 index 000000000..1f75e0708 --- /dev/null +++ b/plugins/anthropic/api-extractor.json @@ -0,0 +1,20 @@ +/** + * Config file for API Extractor. For more info, please visit: https://api-extractor.com + */ +{ + "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", + + /** + * Optionally specifies another JSON config file that this file extends from. This provides a way for + * standard settings to be shared across multiple projects. + * + * If the path starts with "./" or "../", the path is resolved relative to the folder of the file that contains + * the "extends" field. Otherwise, the first path segment is interpreted as an NPM package name, and will be + * resolved using NodeJS require(). + * + * SUPPORTED TOKENS: none + * DEFAULT VALUE: "" + */ + "extends": "../../api-extractor-shared.json", + "mainEntryPointFilePath": "./dist/index.d.ts" +} diff --git a/plugins/anthropic/package.json b/plugins/anthropic/package.json new file mode 100644 index 000000000..5f6d401de --- /dev/null +++ b/plugins/anthropic/package.json @@ -0,0 +1,51 @@ +{ + "name": "@livekit/agents-plugin-anthropic", + "version": "1.0.0", + "description": "Anthropic plugin for LiveKit Node Agents", + "main": "dist/index.js", + "require": "dist/index.cjs", + "types": "dist/index.d.ts", + "exports": { + "import": { + "types": "./dist/index.d.ts", + "default": "./dist/index.js" + }, + "require": { + "types": "./dist/index.d.cts", + "default": "./dist/index.cjs" + } + }, + "author": "LiveKit", + "type": "module", + "repository": "git@github.com:livekit/agents-js.git", + "license": "Apache-2.0", + "files": [ + "dist", + "src", + "README.md" + ], + "scripts": { + "build": "tsup --onSuccess \"pnpm build:types\"", + "build:types": "tsc --declaration --emitDeclarationOnly && node ../../scripts/copyDeclarationOutput.js", + "clean": "rm -rf dist", + "clean:build": "pnpm clean && pnpm build", + "lint": "eslint -f unix \"src/**/*.{ts,js}\"", + "api:check": "api-extractor run --typescript-compiler-folder ../../node_modules/typescript", + "api:update": "api-extractor run --local --typescript-compiler-folder ../../node_modules/typescript --verbose" + }, + "devDependencies": { + "@livekit/agents": "workspace:*", + "@livekit/agents-plugins-test": "workspace:*", + "@livekit/rtc-node": "catalog:", + "@microsoft/api-extractor": "^7.35.0", + "tsup": "^8.3.5", + "typescript": "^5.0.0" + }, + "dependencies": { + "@anthropic-ai/sdk": "^0.33.1" + }, + "peerDependencies": { + "@livekit/agents": "workspace:*", + "@livekit/rtc-node": "catalog:" + } +} \ No newline at end of file diff --git a/plugins/anthropic/src/index.ts b/plugins/anthropic/src/index.ts new file mode 100644 index 000000000..4bb3fa436 --- /dev/null +++ b/plugins/anthropic/src/index.ts @@ -0,0 +1,19 @@ +// SPDX-FileCopyrightText: 2025 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { Plugin } from '@livekit/agents'; + +export { LLM, LLMStream, type LLMOptions } from './llm.js'; +export * from './models.js'; + +class AnthropicPlugin extends Plugin { + constructor() { + super({ + title: 'anthropic', + version: __PACKAGE_VERSION__, + package: __PACKAGE_NAME__, + }); + } +} + +Plugin.registerPlugin(new AnthropicPlugin()); diff --git a/plugins/anthropic/src/llm.test.ts b/plugins/anthropic/src/llm.test.ts new file mode 100644 index 000000000..a898b8dff --- /dev/null +++ b/plugins/anthropic/src/llm.test.ts @@ -0,0 +1,109 @@ +// SPDX-FileCopyrightText: 2025 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { llm } from '@livekit/agents'; +import { describe, expect, it } from 'vitest'; +import { LLM } from './llm.js'; + +describe('Anthropic LLM', () => { + it('correctly maps ChatContext to Anthropic system and messages arrays', () => { + const anthropicLlm = new LLM({ apiKey: 'dummy', model: 'claude-3-5-sonnet-20241022' }); + + const chatCtx = new llm.ChatContext(); + chatCtx.addMessage({ + role: 'system', + content: 'You are a mock agent.', + }); + chatCtx.addMessage({ + role: 'user', + content: 'Hello, world!', + }); + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const { system, messages } = (anthropicLlm as any)._buildAnthropicContext(chatCtx); + + // Assert that system prompts were correctly isolated + expect(system).toHaveLength(1); + expect(system[0].text).toBe('You are a mock agent.'); + + // Assert that strictly user messages ended up in the messages payload + expect(messages).toHaveLength(1); + expect(messages[0].role).toBe('user'); + expect(messages[0].content).toBe('Hello, world!'); + }); + + it('merges consecutive same-role messages', () => { + const anthropicLlm = new LLM({ apiKey: 'dummy', model: 'claude-3-5-sonnet-20241022' }); + + const chatCtx = new llm.ChatContext(); + chatCtx.addMessage({ role: 'user', content: 'First message' }); + chatCtx.addMessage({ role: 'user', content: 'Second message' }); + chatCtx.addMessage({ role: 'assistant', content: 'Reply' }); + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const { messages } = (anthropicLlm as any)._buildAnthropicContext(chatCtx); + + // Two user messages should be merged into one with array content + expect(messages).toHaveLength(2); + expect(messages[0].role).toBe('user'); + expect(Array.isArray(messages[0].content)).toBe(true); + expect(messages[0].content).toHaveLength(2); + expect(messages[1].role).toBe('assistant'); + }); + + it('injects a dummy user message if conversation starts with assistant', () => { + const anthropicLlm = new LLM({ apiKey: 'dummy', model: 'claude-3-5-sonnet-20241022' }); + + const chatCtx = new llm.ChatContext(); + chatCtx.addMessage({ role: 'system', content: 'System prompt' }); + chatCtx.addMessage({ role: 'assistant', content: 'I start talking' }); + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const { messages } = (anthropicLlm as any)._buildAnthropicContext(chatCtx); + + // Should have injected a dummy user message at the start + expect(messages[0].role).toBe('user'); + expect(messages[0].content).toBe('(empty)'); + expect(messages[1].role).toBe('assistant'); + }); + + it('handles function_call and function_call_output items', () => { + const anthropicLlm = new LLM({ apiKey: 'dummy', model: 'claude-3-5-sonnet-20241022' }); + + const chatCtx = new llm.ChatContext(); + chatCtx.addMessage({ role: 'user', content: 'What is the weather?' }); + + // Simulate a function call from the assistant + chatCtx.items.push( + new llm.FunctionCall({ + callId: 'call_123', + name: 'get_weather', + args: '{"city":"London"}', + }), + ); + + // Simulate the tool result + chatCtx.items.push( + new llm.FunctionCallOutput({ + callId: 'call_123', + name: 'get_weather', + output: '{"temp": 20}', + isError: false, + }), + ); + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const { messages } = (anthropicLlm as any)._buildAnthropicContext(chatCtx); + + // user message, then assistant tool_use, then user tool_result + expect(messages).toHaveLength(3); + expect(messages[0].role).toBe('user'); + expect(messages[1].role).toBe('assistant'); + expect(Array.isArray(messages[1].content)).toBe(true); + expect(messages[1].content[0].type).toBe('tool_use'); + expect(messages[1].content[0].id).toBe('call_123'); + expect(messages[2].role).toBe('user'); + expect(Array.isArray(messages[2].content)).toBe(true); + expect(messages[2].content[0].type).toBe('tool_result'); + }); +}); diff --git a/plugins/anthropic/src/llm.ts b/plugins/anthropic/src/llm.ts new file mode 100644 index 000000000..eccdcf1a2 --- /dev/null +++ b/plugins/anthropic/src/llm.ts @@ -0,0 +1,405 @@ +// SPDX-FileCopyrightText: 2025 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import Anthropic from '@anthropic-ai/sdk'; +import type { APIConnectOptions } from '@livekit/agents'; +import { + APIConnectionError, + APIStatusError, + APITimeoutError, + DEFAULT_API_CONNECT_OPTIONS, + llm, +} from '@livekit/agents'; +import type { ChatModels } from './models.js'; + +/** Configuration options for the Anthropic LLM plugin. */ +export interface LLMOptions { + /** The model identifier to use. */ + model: string | ChatModels; + /** Anthropic API key. Falls back to the `ANTHROPIC_API_KEY` environment variable. */ + apiKey?: string; + /** Custom base URL for the Anthropic API. */ + baseURL?: string; + /** Sampling temperature. */ + temperature?: number; + /** Pre-configured Anthropic client instance. */ + client?: Anthropic; + /** Tool selection strategy. */ + toolChoice?: llm.ToolChoice; + /** Whether to allow parallel tool calls. */ + parallelToolCalls?: boolean; + /** Maximum number of tokens in the response. Defaults to 4096. */ + maxTokens?: number; +} + +const defaultLLMOptions: LLMOptions = { + model: 'claude-sonnet-4-6', + /* eslint-disable-next-line turbo/no-undeclared-env-vars */ + apiKey: process.env.ANTHROPIC_API_KEY, + parallelToolCalls: true, +}; + +/** + * Anthropic LLM provider for LiveKit Agents. + * + * @remarks + * Implements the {@link llm.LLM} interface using the Anthropic Messages API. + * Supports streaming, tool calling, and system prompt isolation required by + * Claude 3.5+ models. + */ +export class LLM extends llm.LLM { + #opts: LLMOptions; + #client: Anthropic; + + constructor(opts: Partial = defaultLLMOptions) { + super(); + + this.#opts = { ...defaultLLMOptions, ...opts }; + if (!this.#opts.apiKey && !this.#opts.client) { + throw new Error( + 'Anthropic API key is required, whether as an argument or as $ANTHROPIC_API_KEY', + ); + } + + this.#client = + this.#opts.client || + new Anthropic({ + baseURL: this.#opts.baseURL, + apiKey: this.#opts.apiKey, + }); + } + + /** @returns Human-readable label for logging. */ + label(): string { + return 'anthropic.LLM'; + } + + /** @returns The model identifier being used. */ + get model(): string { + return this.#opts.model; + } + + /** @returns The API provider host. */ + get provider(): string { + try { + const url = new URL(this.#client.baseURL); + return url.host; + } catch { + return 'api.anthropic.com'; + } + } + + /** + * Converts a framework ChatContext into Anthropic's message format. + * + * @remarks + * - System prompts are isolated into a separate `TextBlockParam[]` array + * (required by Claude 3.5+). + * - `function_call` items are mapped to Anthropic `tool_use` content blocks. + * - `function_call_output` items are mapped to `tool_result` content blocks. + * - Consecutive same-role messages are merged to satisfy Anthropic's + * strict alternating-turn requirement. + * - A dummy `(empty)` user message is injected if the conversation doesn't + * start with a user turn. + */ + protected _buildAnthropicContext(chatCtx: llm.ChatContext): { + system: Anthropic.TextBlockParam[]; + messages: Anthropic.MessageParam[]; + } { + const system: Anthropic.TextBlockParam[] = []; + const rawMessages: Anthropic.MessageParam[] = []; + + for (const msg of chatCtx.items) { + if (msg.type === 'message') { + const textContent = msg.textContent || ''; + if (msg.role === 'system') { + system.push({ type: 'text', text: textContent }); + } else if (msg.role === 'user' || msg.role === 'assistant') { + rawMessages.push({ + role: msg.role, + content: textContent, + }); + } + } else if (msg.type === 'function_call') { + // Map to Anthropic's tool_use content block (assistant role) + rawMessages.push({ + role: 'assistant', + content: [ + { + type: 'tool_use', + id: msg.callId, + name: msg.name, + input: JSON.parse(msg.args || '{}'), + }, + ], + }); + } else if (msg.type === 'function_call_output') { + // Map to Anthropic's tool_result content block (user role) + rawMessages.push({ + role: 'user', + content: [ + { + type: 'tool_result', + tool_use_id: msg.callId, + content: msg.output, + is_error: msg.isError, + }, + ], + }); + } + } + + // Merge consecutive same-role messages (Anthropic requires alternating turns) + const messages: Anthropic.MessageParam[] = []; + for (const msg of rawMessages) { + const prev = messages[messages.length - 1]; + if (prev && prev.role === msg.role) { + // Merge content into a single content array + const prevContent = Array.isArray(prev.content) + ? prev.content + : [{ type: 'text' as const, text: prev.content as string }]; + const curContent = Array.isArray(msg.content) + ? msg.content + : [{ type: 'text' as const, text: msg.content as string }]; + prev.content = [...prevContent, ...curContent] as Anthropic.ContentBlockParam[]; + } else { + messages.push({ ...msg }); + } + } + + // Anthropic requires conversations to start with a user turn + if (messages.length === 0 || messages[0]!.role !== 'user') { + messages.unshift({ role: 'user', content: '(empty)' }); + } + + return { system, messages }; + } + + /** + * Creates a streaming chat completion. + * + * @remarks + * Maps `toolChoice` to Anthropic's format: + * - `"required"` → `{ type: "any" }` + * - `"none"` → clears tools + * - `{ type: "function", function: { name } }` → `{ type: "tool", name }` + */ + chat({ + chatCtx, + toolCtx, + connOptions = DEFAULT_API_CONNECT_OPTIONS, + parallelToolCalls, + toolChoice, + extraKwargs, + }: { + chatCtx: llm.ChatContext; + toolCtx?: llm.ToolContext; + connOptions?: APIConnectOptions; + parallelToolCalls?: boolean; + toolChoice?: llm.ToolChoice; + extraKwargs?: Record; + }): LLMStream { + const extras: Record = { ...extraKwargs }; + + if (this.#opts.temperature) extras.temperature = this.#opts.temperature; + + const { system, messages } = this._buildAnthropicContext(chatCtx); + + // Build Anthropic tool schemas + const anthropicTools: Anthropic.Tool[] = []; + if (toolCtx) { + for (const [name, tool] of Object.entries(toolCtx)) { + anthropicTools.push({ + name: name, + description: tool.description || '', + input_schema: tool.parameters as Anthropic.Tool.InputSchema, + }); + } + } + + // Map toolChoice to Anthropic format + const resolvedToolChoice = toolChoice ?? this.#opts.toolChoice; + if (resolvedToolChoice && anthropicTools.length > 0) { + let anthropicToolChoice: Record | undefined = { type: 'auto' }; + + if (typeof resolvedToolChoice === 'string') { + if (resolvedToolChoice === 'required') { + anthropicToolChoice = { type: 'any' }; + } else if (resolvedToolChoice === 'none') { + // Clear tools entirely when none is requested + anthropicTools.length = 0; + anthropicToolChoice = undefined; + } + } else if ( + typeof resolvedToolChoice === 'object' && + 'type' in resolvedToolChoice && + resolvedToolChoice.type === 'function' + ) { + const fn = (resolvedToolChoice as { function: { name: string } }).function; + anthropicToolChoice = { type: 'tool', name: fn.name }; + } + + if (anthropicToolChoice) { + // Map parallelToolCalls + const resolvedParallel = parallelToolCalls ?? this.#opts.parallelToolCalls; + if (resolvedParallel !== undefined) { + anthropicToolChoice.disable_parallel_tool_use = !resolvedParallel; + } + extras.tool_choice = anthropicToolChoice; + } + } + + const streamPromise = this.#client.messages.create( + { + model: this.#opts.model, + messages: messages, + system: system.length > 0 ? system : undefined, + tools: anthropicTools.length > 0 ? anthropicTools : undefined, + stream: true, + max_tokens: this.#opts.maxTokens || 4096, + ...extras, + }, + { timeout: connOptions.timeoutMs }, + ); + + return new LLMStream(this, streamPromise, chatCtx, toolCtx, connOptions); + } +} + +/** + * Streaming response handler for Anthropic Messages API. + * + * @remarks + * Parses SSE events including: + * - `content_block_start` / `content_block_delta` / `content_block_stop` for text and tool use + * - `message_start` / `message_delta` for token usage tracking + * - Chain-of-thought `` block filtering when tools are active + */ +export class LLMStream extends llm.LLMStream { + #streamPromise: Promise>; + #toolCallId?: string; + #fncName?: string; + #fncRawArgs?: string; + #requestId = ''; + #ignoringCoT = false; + #inputTokens = 0; + #outputTokens = 0; + #toolCtx?: llm.ToolContext; + + constructor( + llmInst: LLM, + streamPromise: Promise>, + chatCtx: llm.ChatContext, + toolCtx: llm.ToolContext | undefined, + connOptions: APIConnectOptions, + ) { + super(llmInst, { chatCtx, connOptions }); + this.#streamPromise = streamPromise; + this.#toolCtx = toolCtx; + } + + protected async run(): Promise { + let retryable = true; + try { + const stream = await this.#streamPromise; + for await (const event of stream) { + if (event.type === 'message_start') { + this.#requestId = event.message.id; + this.#inputTokens = event.message.usage.input_tokens; + this.#outputTokens = event.message.usage.output_tokens; + } else if (event.type === 'message_delta') { + this.#outputTokens += event.usage.output_tokens; + } else if ( + event.type === 'content_block_start' && + event.content_block.type === 'tool_use' + ) { + this.#toolCallId = event.content_block.id; + this.#fncName = event.content_block.name; + this.#fncRawArgs = ''; + } else if (event.type === 'content_block_delta' && event.delta.type === 'text_delta') { + let text = event.delta.text; + + // Filter chain-of-thought blocks when tools are active + if (this.#toolCtx) { + if (text.startsWith('')) { + this.#ignoringCoT = true; + } else if (this.#ignoringCoT && text.includes('')) { + text = text.split('').pop() || ''; + this.#ignoringCoT = false; + } + } + + if (this.#ignoringCoT) { + continue; + } + + this.queue.put({ + id: this.#requestId, + delta: { role: 'assistant', content: text }, + }); + retryable = false; + } else if ( + event.type === 'content_block_delta' && + event.delta.type === 'input_json_delta' + ) { + this.#fncRawArgs += event.delta.partial_json; + } else if (event.type === 'content_block_stop' && this.#toolCallId) { + this.queue.put({ + id: this.#requestId, + delta: { + role: 'assistant', + toolCalls: [ + { + id: '', + type: 'function_call', + callId: this.#toolCallId, + name: this.#fncName || '', + args: this.#fncRawArgs || '', + createdAt: Date.now(), + extra: {}, + toJSON: () => ({}), + }, + ], + }, + }); + this.#toolCallId = undefined; + this.#fncName = undefined; + this.#fncRawArgs = undefined; + retryable = false; + } + } + + // Emit final usage chunk + this.queue.put({ + id: this.#requestId, + usage: { + completionTokens: this.#outputTokens, + promptTokens: this.#inputTokens, + totalTokens: this.#inputTokens + this.#outputTokens, + promptCachedTokens: 0, + }, + }); + } catch (e: unknown) { + if (e instanceof Anthropic.APIError) { + if (e.status === 408 || e.status === 409) { + throw new APITimeoutError({ + message: e.message, + options: { retryable }, + }); + } + throw new APIStatusError({ + message: e.message, + options: { + statusCode: e.status, + body: e.error as object, + retryable: retryable && (e.status === 429 || e.status >= 500), + }, + }); + } + throw new APIConnectionError({ + message: e instanceof Error ? e.message : String(e), + options: { retryable }, + }); + } + } +} diff --git a/plugins/anthropic/src/models.ts b/plugins/anthropic/src/models.ts new file mode 100644 index 000000000..0a98c2a1a --- /dev/null +++ b/plugins/anthropic/src/models.ts @@ -0,0 +1,21 @@ +// SPDX-FileCopyrightText: 2025 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 + +/** + * Supported Anthropic Chat Models. + * + * @remarks + * Based on https://docs.anthropic.com/en/docs/about-claude/model-deprecations + */ +export type ChatModels = + | 'claude-3-5-sonnet-20241022' + | 'claude-3-5-haiku-20241022' + | 'claude-3-haiku-20240307' + | 'claude-3-7-sonnet-20250219' + | 'claude-sonnet-4-20250514' + | 'claude-sonnet-4-6' + | 'claude-opus-4-20250514' + | 'claude-opus-4-1-20250805' + | 'claude-opus-4-6' + | (string & Record); diff --git a/plugins/anthropic/tsconfig.json b/plugins/anthropic/tsconfig.json new file mode 100644 index 000000000..5aa30e995 --- /dev/null +++ b/plugins/anthropic/tsconfig.json @@ -0,0 +1,19 @@ +{ + "extends": "../../tsconfig.json", + "include": [ + "./src" + ], + "compilerOptions": { + // match output dir to input dir. e.g. dist/index instead of dist/src/index + "rootDir": "./src", + "declarationDir": "./dist", + "outDir": "./dist" + }, + "typedocOptions": { + "name": "plugins/agents-plugin-openai", + "entryPointStrategy": "resolve", + "entryPoints": [ + "src/index.ts" + ] + } +} \ No newline at end of file diff --git a/plugins/anthropic/tsup.config.ts b/plugins/anthropic/tsup.config.ts new file mode 100644 index 000000000..8ca20961f --- /dev/null +++ b/plugins/anthropic/tsup.config.ts @@ -0,0 +1,7 @@ +import { defineConfig } from 'tsup'; + +import defaults from '../../tsup.config'; + +export default defineConfig({ + ...defaults, +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0dba29006..d2f735ccd 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -12,6 +12,9 @@ catalogs: '@types/ws': specifier: ^8.5.10 version: 8.18.1 + ws: + specifier: ^8.18.0 + version: 8.20.1 patchedDependencies: '@changesets/assemble-release-plan': @@ -233,12 +236,18 @@ importers: examples: dependencies: + '@anthropic-ai/sdk': + specifier: ^0.20.9 + version: 0.20.9 '@livekit/agents': specifier: workspace:* version: link:../agents '@livekit/agents-plugin-anam': specifier: workspace:* version: link:../plugins/anam + '@livekit/agents-plugin-anthropic': + specifier: workspace:* + version: link:../plugins/anthropic '@livekit/agents-plugin-baseten': specifier: workspace:* version: link:../plugins/baseten @@ -401,6 +410,31 @@ importers: specifier: ^5.0.0 version: 5.9.3 + plugins/anthropic: + dependencies: + '@anthropic-ai/sdk': + specifier: ^0.33.1 + version: 0.33.1 + devDependencies: + '@livekit/agents': + specifier: workspace:* + version: link:../../agents + '@livekit/agents-plugins-test': + specifier: workspace:* + version: link:../test + '@livekit/rtc-node': + specifier: 'catalog:' + version: 0.13.27 + '@microsoft/api-extractor': + specifier: ^7.35.0 + version: 7.43.7(@types/node@25.6.0) + tsup: + specifier: ^8.3.5 + version: 8.4.0(@microsoft/api-extractor@7.43.7(@types/node@25.6.0))(postcss@8.5.9)(tsx@4.21.0)(typescript@5.9.3) + typescript: + specifier: ^5.0.0 + version: 5.9.3 + plugins/assemblyai: dependencies: ws: @@ -1289,6 +1323,12 @@ importers: packages: + '@anthropic-ai/sdk@0.20.9': + resolution: {integrity: sha512-Lq74+DhiEQO6F9/gdVOLmHx57pX45ebK2Q/zH14xYe1157a7QeUVknRqIp0Jz5gQI01o7NKbuv9Dag2uQsLjDg==} + + '@anthropic-ai/sdk@0.33.1': + resolution: {integrity: sha512-VrlbxiAdVRGuKP2UQlCnsShDHJKWepzvfRCkZMpU+oaUdKLpOfmylLMRojGrAgebV+kDtPjewCVP0laHXg+vsA==} + '@babel/code-frame@7.24.2': resolution: {integrity: sha512-y5+tLQyV8pg3fsiln67BVLD1P13Eg4lh5RW9mF0zUuvLrv9uIQ4MCL+CRT+FTsBlBjcIan6PGsLcBN0m3ClUyQ==} engines: {node: '>=6.9.0'} @@ -2706,9 +2746,15 @@ packages: '@types/json5@0.0.29': resolution: {integrity: sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ==} + '@types/node-fetch@2.6.13': + resolution: {integrity: sha512-QGpRVpzSaUs30JBSGPjOg4Uveu384erbHBoT1zeONvyCfwQxIkUshLAOqN/k9EjGviPRmWTTe6aH2qySWKTVSw==} + '@types/node@12.20.55': resolution: {integrity: sha512-J8xLz7q2OFulZ2cyGTLE1TbbZcjpno7FaN6zdJNrgAdrJ+DZzh/uFR6YrTb4C+nXakvud8Q4+rbhoIWlYQbUFQ==} + '@types/node@18.19.130': + resolution: {integrity: sha512-GRaXQx6jGfL8sKfaIDD6OupbIHBr9jv7Jnaml9tB7l4v068PAOXqfcujMMo5PhbIs6ggR1XODELqahT2R8v0fg==} + '@types/node@22.19.1': resolution: {integrity: sha512-LCCV0HdSZZZb34qifBsyWlUmok6W7ouER+oQIGBScS8EsZsQbrtFTUrDX4hOl+CS6p7cnNC4td+qrSVGSCTUfQ==} @@ -2908,6 +2954,10 @@ packages: resolution: {integrity: sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==} engines: {node: '>= 14'} + agentkeepalive@4.6.0: + resolution: {integrity: sha512-kja8j7PjmncONqaTsB8fQ+wE2mSU2DJ9D4XKoJ5PFWIdRMa6SLSN1ff4mOr4jCbfRSsxR4keIiySJU0N9T5hIQ==} + engines: {node: '>= 8.0.0'} + ajv@6.12.6: resolution: {integrity: sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==} @@ -3682,10 +3732,17 @@ packages: resolution: {integrity: sha512-TMKDUnIte6bfb5nWv7V/caI169OHgvwjb7V4WkeUvbQQdjr5rWKqHFiKWb/fcOwB+CzBT+qbWjvj+DVwRskpIg==} engines: {node: '>=14'} + form-data-encoder@1.7.2: + resolution: {integrity: sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A==} + form-data@4.0.5: resolution: {integrity: sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w==} engines: {node: '>= 6'} + formdata-node@4.4.1: + resolution: {integrity: sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==} + engines: {node: '>= 12.20'} + formdata-polyfill@4.0.10: resolution: {integrity: sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==} engines: {node: '>=12.20.0'} @@ -3873,6 +3930,9 @@ packages: resolution: {integrity: sha512-AXcZb6vzzrFAUE61HnN4mpLqd/cSIwNQjtNWR0euPm6y0iqx3G4gOXaIDdtdDwZmhwe82LA6+zinmW4UBWVePQ==} engines: {node: '>=16.17.0'} + humanize-ms@1.2.1: + resolution: {integrity: sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==} + iconv-lite@0.7.2: resolution: {integrity: sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw==} engines: {node: '>=0.10.0'} @@ -5182,6 +5242,9 @@ packages: unbox-primitive@1.0.2: resolution: {integrity: sha512-61pPlCD9h51VoreyJ0BReideM3MDKMKnh6+V9L08331ipq6Q8OFXZYiqP6n/tbHx4s5I9uRhcye6BrbkizkBDw==} + undici-types@5.26.5: + resolution: {integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==} + undici-types@6.21.0: resolution: {integrity: sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==} @@ -5381,6 +5444,10 @@ packages: resolution: {integrity: sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==} engines: {node: '>= 8'} + web-streams-polyfill@4.0.0-beta.3: + resolution: {integrity: sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==} + engines: {node: '>= 14'} + webidl-conversions@3.0.1: resolution: {integrity: sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==} @@ -5483,6 +5550,31 @@ packages: snapshots: + '@anthropic-ai/sdk@0.20.9': + dependencies: + '@types/node': 18.19.130 + '@types/node-fetch': 2.6.13 + abort-controller: 3.0.0 + agentkeepalive: 4.6.0 + form-data-encoder: 1.7.2 + formdata-node: 4.4.1 + node-fetch: 2.7.0 + web-streams-polyfill: 3.3.3 + transitivePeerDependencies: + - encoding + + '@anthropic-ai/sdk@0.33.1': + dependencies: + '@types/node': 18.19.130 + '@types/node-fetch': 2.6.13 + abort-controller: 3.0.0 + agentkeepalive: 4.6.0 + form-data-encoder: 1.7.2 + formdata-node: 4.4.1 + node-fetch: 2.7.0 + transitivePeerDependencies: + - encoding + '@babel/code-frame@7.24.2': dependencies: '@babel/highlight': 7.24.5 @@ -6819,8 +6911,17 @@ snapshots: '@types/json5@0.0.29': {} + '@types/node-fetch@2.6.13': + dependencies: + '@types/node': 22.19.1 + form-data: 4.0.5 + '@types/node@12.20.55': {} + '@types/node@18.19.130': + dependencies: + undici-types: 5.26.5 + '@types/node@22.19.1': dependencies: undici-types: 6.21.0 @@ -7083,6 +7184,10 @@ snapshots: agent-base@7.1.4: {} + agentkeepalive@4.6.0: + dependencies: + humanize-ms: 1.2.1 + ajv@6.12.6: dependencies: fast-deep-equal: 3.1.3 @@ -8055,6 +8160,8 @@ snapshots: cross-spawn: 7.0.6 signal-exit: 4.1.0 + form-data-encoder@1.7.2: {} + form-data@4.0.5: dependencies: asynckit: 0.4.0 @@ -8063,6 +8170,11 @@ snapshots: hasown: 2.0.2 mime-types: 2.1.35 + formdata-node@4.4.1: + dependencies: + node-domexception: 1.0.0 + web-streams-polyfill: 4.0.0-beta.3 + formdata-polyfill@4.0.10: dependencies: fetch-blob: 3.2.0 @@ -8276,6 +8388,10 @@ snapshots: human-signals@5.0.0: {} + humanize-ms@1.2.1: + dependencies: + ms: 2.1.3 + iconv-lite@0.7.2: dependencies: safer-buffer: 2.1.2 @@ -9118,7 +9234,7 @@ snapshots: require-in-the-middle@7.5.2: dependencies: - debug: 4.4.1 + debug: 4.4.3 module-details-from-path: 1.0.4 resolve: 1.22.8 transitivePeerDependencies: @@ -9674,6 +9790,8 @@ snapshots: has-symbols: 1.1.0 which-boxed-primitive: 1.0.2 + undici-types@5.26.5: {} + undici-types@6.21.0: {} undici-types@7.19.2: @@ -9923,6 +10041,8 @@ snapshots: web-streams-polyfill@3.3.3: {} + web-streams-polyfill@4.0.0-beta.3: {} + webidl-conversions@3.0.1: {} webidl-conversions@4.0.2: {} From f9edad153470420998e38d3b8eac8deb185370cf Mon Sep 17 00:00:00 2001 From: Krishna Shukla Date: Wed, 20 May 2026 05:46:11 +0530 Subject: [PATCH 02/12] added changeset --- .changeset/orange-squids-cover.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/orange-squids-cover.md diff --git a/.changeset/orange-squids-cover.md b/.changeset/orange-squids-cover.md new file mode 100644 index 000000000..7b30e4f3b --- /dev/null +++ b/.changeset/orange-squids-cover.md @@ -0,0 +1,5 @@ +--- +'@livekit/agents-plugin-anthropic': patch +--- + +added anthropic plugin From 8b9d883c3b8ba3a0ca758b007f1cab6c271d3114 Mon Sep 17 00:00:00 2001 From: Krishna Shukla Date: Thu, 21 May 2026 23:43:34 +0530 Subject: [PATCH 03/12] minor fixes --- plugins/anthropic/src/llm.ts | 4 ++-- plugins/anthropic/tsconfig.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/anthropic/src/llm.ts b/plugins/anthropic/src/llm.ts index eccdcf1a2..e546c9888 100644 --- a/plugins/anthropic/src/llm.ts +++ b/plugins/anthropic/src/llm.ts @@ -201,7 +201,7 @@ export class LLM extends llm.LLM { }): LLMStream { const extras: Record = { ...extraKwargs }; - if (this.#opts.temperature) extras.temperature = this.#opts.temperature; + if (this.#opts.temperature !== undefined) extras.temperature = this.#opts.temperature; const { system, messages } = this._buildAnthropicContext(chatCtx); @@ -293,7 +293,7 @@ export class LLMStream extends llm.LLMStream { toolCtx: llm.ToolContext | undefined, connOptions: APIConnectOptions, ) { - super(llmInst, { chatCtx, connOptions }); + super(llmInst, { chatCtx, toolCtx, connOptions }); this.#streamPromise = streamPromise; this.#toolCtx = toolCtx; } diff --git a/plugins/anthropic/tsconfig.json b/plugins/anthropic/tsconfig.json index 5aa30e995..8293360d3 100644 --- a/plugins/anthropic/tsconfig.json +++ b/plugins/anthropic/tsconfig.json @@ -10,7 +10,7 @@ "outDir": "./dist" }, "typedocOptions": { - "name": "plugins/agents-plugin-openai", + "name": "plugins/agents-plugin-anthropic", "entryPointStrategy": "resolve", "entryPoints": [ "src/index.ts" From eeae148dd22b42d3fc8e5d2e19ad297eafb1cb50 Mon Sep 17 00:00:00 2001 From: Krishna Shukla Date: Fri, 22 May 2026 00:07:34 +0530 Subject: [PATCH 04/12] minor fixes --- plugins/anthropic/src/llm.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/plugins/anthropic/src/llm.ts b/plugins/anthropic/src/llm.ts index e546c9888..fe4ca6f0d 100644 --- a/plugins/anthropic/src/llm.ts +++ b/plugins/anthropic/src/llm.ts @@ -212,7 +212,9 @@ export class LLM extends llm.LLM { anthropicTools.push({ name: name, description: tool.description || '', - input_schema: tool.parameters as Anthropic.Tool.InputSchema, + input_schema: (tool.parameters + ? llm.toJsonSchema(tool.parameters, false) + : { type: 'object', properties: {} }) as Anthropic.Tool.InputSchema, }); } } From 230c6aa6e5ece1b3f889799485296c4c413c8ba7 Mon Sep 17 00:00:00 2001 From: Krishna Shukla Date: Fri, 22 May 2026 00:20:59 +0530 Subject: [PATCH 05/12] correct output token accumulation and parallel tool options --- plugins/anthropic/src/llm.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/plugins/anthropic/src/llm.ts b/plugins/anthropic/src/llm.ts index fe4ca6f0d..2b3d122de 100644 --- a/plugins/anthropic/src/llm.ts +++ b/plugins/anthropic/src/llm.ts @@ -219,9 +219,11 @@ export class LLM extends llm.LLM { } } - // Map toolChoice to Anthropic format + // Map toolChoice and parallelToolCalls to Anthropic format const resolvedToolChoice = toolChoice ?? this.#opts.toolChoice; - if (resolvedToolChoice && anthropicTools.length > 0) { + const resolvedParallel = parallelToolCalls ?? this.#opts.parallelToolCalls; + + if ((resolvedToolChoice || resolvedParallel !== undefined) && anthropicTools.length > 0) { let anthropicToolChoice: Record | undefined = { type: 'auto' }; if (typeof resolvedToolChoice === 'string') { @@ -243,7 +245,6 @@ export class LLM extends llm.LLM { if (anthropicToolChoice) { // Map parallelToolCalls - const resolvedParallel = parallelToolCalls ?? this.#opts.parallelToolCalls; if (resolvedParallel !== undefined) { anthropicToolChoice.disable_parallel_tool_use = !resolvedParallel; } @@ -310,7 +311,7 @@ export class LLMStream extends llm.LLMStream { this.#inputTokens = event.message.usage.input_tokens; this.#outputTokens = event.message.usage.output_tokens; } else if (event.type === 'message_delta') { - this.#outputTokens += event.usage.output_tokens; + this.#outputTokens = event.usage.output_tokens; } else if ( event.type === 'content_block_start' && event.content_block.type === 'tool_use' From da2e10f28f78f153f5d928fe34e40a47851eaa2e Mon Sep 17 00:00:00 2001 From: Krishna Shukla Date: Fri, 22 May 2026 00:47:22 +0530 Subject: [PATCH 06/12] move API call into run() for proper retry support --- plugins/anthropic/src/llm.ts | 36 +++++++++++++++++++----------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/plugins/anthropic/src/llm.ts b/plugins/anthropic/src/llm.ts index 2b3d122de..f7bf160a5 100644 --- a/plugins/anthropic/src/llm.ts +++ b/plugins/anthropic/src/llm.ts @@ -252,20 +252,17 @@ export class LLM extends llm.LLM { } } - const streamPromise = this.#client.messages.create( - { - model: this.#opts.model, - messages: messages, - system: system.length > 0 ? system : undefined, - tools: anthropicTools.length > 0 ? anthropicTools : undefined, - stream: true, - max_tokens: this.#opts.maxTokens || 4096, - ...extras, - }, - { timeout: connOptions.timeoutMs }, - ); + const requestParams: Anthropic.MessageCreateParamsStreaming = { + model: this.#opts.model, + messages: messages, + system: system.length > 0 ? system : undefined, + tools: anthropicTools.length > 0 ? anthropicTools : undefined, + stream: true, + max_tokens: this.#opts.maxTokens || 4096, + ...extras, + }; - return new LLMStream(this, streamPromise, chatCtx, toolCtx, connOptions); + return new LLMStream(this, this.#client, requestParams, chatCtx, toolCtx, connOptions); } } @@ -279,7 +276,8 @@ export class LLM extends llm.LLM { * - Chain-of-thought `` block filtering when tools are active */ export class LLMStream extends llm.LLMStream { - #streamPromise: Promise>; + #client: Anthropic; + #requestParams: Anthropic.MessageCreateParamsStreaming; #toolCallId?: string; #fncName?: string; #fncRawArgs?: string; @@ -291,20 +289,24 @@ export class LLMStream extends llm.LLMStream { constructor( llmInst: LLM, - streamPromise: Promise>, + client: Anthropic, + requestParams: Anthropic.MessageCreateParamsStreaming, chatCtx: llm.ChatContext, toolCtx: llm.ToolContext | undefined, connOptions: APIConnectOptions, ) { super(llmInst, { chatCtx, toolCtx, connOptions }); - this.#streamPromise = streamPromise; + this.#client = client; + this.#requestParams = requestParams; this.#toolCtx = toolCtx; } protected async run(): Promise { let retryable = true; try { - const stream = await this.#streamPromise; + const stream = await this.#client.messages.create(this.#requestParams, { + timeout: this.connOptions.timeoutMs, + }); for await (const event of stream) { if (event.type === 'message_start') { this.#requestId = event.message.id; From c64e171315f083ef7519e259465df0bf94cd8194 Mon Sep 17 00:00:00 2001 From: "rosetta-livekit-bot[bot]" <282703043+rosetta-livekit-bot[bot]@users.noreply.github.com> Date: Sun, 24 May 2026 20:04:50 +0000 Subject: [PATCH 07/12] test(anthropic): cover retry stream recreation --- plugins/anthropic/src/llm.test.ts | 41 +++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/plugins/anthropic/src/llm.test.ts b/plugins/anthropic/src/llm.test.ts index a898b8dff..df746dc39 100644 --- a/plugins/anthropic/src/llm.test.ts +++ b/plugins/anthropic/src/llm.test.ts @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: 2025 LiveKit, Inc. // // SPDX-License-Identifier: Apache-2.0 +import type Anthropic from '@anthropic-ai/sdk'; import { llm } from '@livekit/agents'; import { describe, expect, it } from 'vitest'; import { LLM } from './llm.js'; @@ -106,4 +107,44 @@ describe('Anthropic LLM', () => { expect(Array.isArray(messages[2].content)).toBe(true); expect(messages[2].content[0].type).toBe('tool_result'); }); + + it('creates a fresh stream on retry', async () => { + let calls = 0; + const client = { + messages: { + create: async () => { + calls += 1; + if (calls === 1) { + throw new Error('transient connect failure'); + } + return (async function* (): AsyncGenerator {})(); + }, + }, + } as unknown as Anthropic; + const anthropicLlm = new LLM({ + apiKey: 'dummy', + client, + model: 'claude-3-5-sonnet-20241022', + }); + anthropicLlm.on('error', () => {}); + const chatCtx = new llm.ChatContext(); + chatCtx.addMessage({ role: 'user', content: 'Hello, world!' }); + + const stream = anthropicLlm.chat({ + chatCtx, + connOptions: { maxRetry: 1, retryIntervalMs: 0, timeoutMs: 1000 }, + }); + const chunks: llm.ChatChunk[] = []; + for await (const chunk of stream) { + chunks.push(chunk); + } + + expect(calls).toBe(2); + expect(chunks.at(-1)?.usage).toEqual({ + completionTokens: 0, + promptTokens: 0, + promptCachedTokens: 0, + totalTokens: 0, + }); + }); }); From df9cd9a26e852b0b4682d2cda31e5691fd2a070c Mon Sep 17 00:00:00 2001 From: "rosetta-livekit-bot[bot]" <282703043+rosetta-livekit-bot[bot]@users.noreply.github.com> Date: Sun, 24 May 2026 20:07:52 +0000 Subject: [PATCH 08/12] chore: sync lockfile --- pnpm-lock.yaml | 42 ++++++------------------------------------ 1 file changed, 6 insertions(+), 36 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d2f735ccd..c21ee9f14 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -60,7 +60,7 @@ importers: version: 8.10.0(eslint@8.57.0) eslint-config-standard: specifier: ^17.1.0 - version: 17.1.0(eslint-plugin-import@2.29.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.9.3))(eslint@8.57.0))(eslint-plugin-n@16.6.2(eslint@8.57.0))(eslint-plugin-promise@6.1.1(eslint@8.57.0))(eslint@8.57.0) + version: 17.1.0(eslint-plugin-import@2.29.1)(eslint-plugin-n@16.6.2(eslint@8.57.0))(eslint-plugin-promise@6.1.1(eslint@8.57.0))(eslint@8.57.0) eslint-config-turbo: specifier: ^1.12.2 version: 1.13.3(eslint@8.57.0) @@ -236,18 +236,12 @@ importers: examples: dependencies: - '@anthropic-ai/sdk': - specifier: ^0.20.9 - version: 0.20.9 '@livekit/agents': specifier: workspace:* version: link:../agents '@livekit/agents-plugin-anam': specifier: workspace:* version: link:../plugins/anam - '@livekit/agents-plugin-anthropic': - specifier: workspace:* - version: link:../plugins/anthropic '@livekit/agents-plugin-baseten': specifier: workspace:* version: link:../plugins/baseten @@ -1323,9 +1317,6 @@ importers: packages: - '@anthropic-ai/sdk@0.20.9': - resolution: {integrity: sha512-Lq74+DhiEQO6F9/gdVOLmHx57pX45ebK2Q/zH14xYe1157a7QeUVknRqIp0Jz5gQI01o7NKbuv9Dag2uQsLjDg==} - '@anthropic-ai/sdk@0.33.1': resolution: {integrity: sha512-VrlbxiAdVRGuKP2UQlCnsShDHJKWepzvfRCkZMpU+oaUdKLpOfmylLMRojGrAgebV+kDtPjewCVP0laHXg+vsA==} @@ -5550,19 +5541,6 @@ packages: snapshots: - '@anthropic-ai/sdk@0.20.9': - dependencies: - '@types/node': 18.19.130 - '@types/node-fetch': 2.6.13 - abort-controller: 3.0.0 - agentkeepalive: 4.6.0 - form-data-encoder: 1.7.2 - formdata-node: 4.4.1 - node-fetch: 2.7.0 - web-streams-polyfill: 3.3.3 - transitivePeerDependencies: - - encoding - '@anthropic-ai/sdk@0.33.1': dependencies: '@types/node': 18.19.130 @@ -7086,14 +7064,6 @@ snapshots: optionalDependencies: vite: 7.3.2(@types/node@22.19.1)(tsx@4.21.0) - '@vitest/mocker@4.0.17(vite@7.3.2(@types/node@25.6.0)(tsx@4.21.0))': - dependencies: - '@vitest/spy': 4.0.17 - estree-walker: 3.0.3 - magic-string: 0.30.21 - optionalDependencies: - vite: 7.3.2(@types/node@25.6.0)(tsx@4.21.0) - '@vitest/pretty-format@3.2.2': dependencies: tinyrainbow: 2.0.0 @@ -7815,7 +7785,7 @@ snapshots: dependencies: eslint: 8.57.0 - eslint-config-standard@17.1.0(eslint-plugin-import@2.29.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.9.3))(eslint@8.57.0))(eslint-plugin-n@16.6.2(eslint@8.57.0))(eslint-plugin-promise@6.1.1(eslint@8.57.0))(eslint@8.57.0): + eslint-config-standard@17.1.0(eslint-plugin-import@2.29.1)(eslint-plugin-n@16.6.2(eslint@8.57.0))(eslint-plugin-promise@6.1.1(eslint@8.57.0))(eslint@8.57.0): dependencies: eslint: 8.57.0 eslint-plugin-import: 2.29.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.9.3))(eslint-import-resolver-typescript@3.6.1)(eslint@8.57.0) @@ -7840,7 +7810,7 @@ snapshots: debug: 4.4.1 enhanced-resolve: 5.16.1 eslint: 8.57.0 - eslint-module-utils: 2.8.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.9.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.9.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1)(eslint@8.57.0))(eslint@8.57.0) + eslint-module-utils: 2.8.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.9.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1)(eslint@8.57.0) eslint-plugin-import: 2.29.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.9.3))(eslint-import-resolver-typescript@3.6.1)(eslint@8.57.0) fast-glob: 3.3.2 get-tsconfig: 4.7.5 @@ -7852,7 +7822,7 @@ snapshots: - eslint-import-resolver-webpack - supports-color - eslint-module-utils@2.8.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.9.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.9.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1)(eslint@8.57.0))(eslint@8.57.0): + eslint-module-utils@2.8.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.9.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1)(eslint@8.57.0): dependencies: debug: 3.2.7 optionalDependencies: @@ -7880,7 +7850,7 @@ snapshots: doctrine: 2.1.0 eslint: 8.57.0 eslint-import-resolver-node: 0.3.9 - eslint-module-utils: 2.8.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.9.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.9.3))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1)(eslint@8.57.0))(eslint@8.57.0) + eslint-module-utils: 2.8.1(@typescript-eslint/parser@6.21.0(eslint@8.57.0)(typescript@5.9.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1)(eslint@8.57.0) hasown: 2.0.2 is-core-module: 2.13.1 is-glob: 4.0.3 @@ -9995,7 +9965,7 @@ snapshots: vitest@4.0.17(@opentelemetry/api@1.9.0)(@types/node@25.6.0)(tsx@4.21.0): dependencies: '@vitest/expect': 4.0.17 - '@vitest/mocker': 4.0.17(vite@7.3.2(@types/node@25.6.0)(tsx@4.21.0)) + '@vitest/mocker': 4.0.17(vite@7.3.2(@types/node@22.19.1)(tsx@4.21.0)) '@vitest/pretty-format': 4.0.17 '@vitest/runner': 4.0.17 '@vitest/snapshot': 4.0.17 From 1f568300d7e5e69340034c205ead9addba626ca0 Mon Sep 17 00:00:00 2001 From: "rosetta-livekit-bot[bot]" <282703043+rosetta-livekit-bot[bot]@users.noreply.github.com> Date: Sun, 24 May 2026 20:20:20 +0000 Subject: [PATCH 09/12] fix(anthropic): reset retry stream state --- plugins/anthropic/src/llm.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/plugins/anthropic/src/llm.ts b/plugins/anthropic/src/llm.ts index f7bf160a5..a83581562 100644 --- a/plugins/anthropic/src/llm.ts +++ b/plugins/anthropic/src/llm.ts @@ -112,7 +112,7 @@ export class LLM extends llm.LLM { for (const msg of chatCtx.items) { if (msg.type === 'message') { const textContent = msg.textContent || ''; - if (msg.role === 'system') { + if (msg.role === 'system' || msg.role === 'developer') { system.push({ type: 'text', text: textContent }); } else if (msg.role === 'user' || msg.role === 'assistant') { rawMessages.push({ @@ -303,6 +303,14 @@ export class LLMStream extends llm.LLMStream { protected async run(): Promise { let retryable = true; + this.#toolCallId = undefined; + this.#fncName = undefined; + this.#fncRawArgs = undefined; + this.#requestId = ''; + this.#ignoringCoT = false; + this.#inputTokens = 0; + this.#outputTokens = 0; + try { const stream = await this.#client.messages.create(this.#requestParams, { timeout: this.connOptions.timeoutMs, From 0ff706aa2e454d4e2ae9f3aa5917810514f6e802 Mon Sep 17 00:00:00 2001 From: David Zhao Date: Mon, 25 May 2026 00:17:18 -0700 Subject: [PATCH 10/12] Update plugins/anthropic/src/llm.ts Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- plugins/anthropic/src/llm.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/anthropic/src/llm.ts b/plugins/anthropic/src/llm.ts index a83581562..efc17fbd6 100644 --- a/plugins/anthropic/src/llm.ts +++ b/plugins/anthropic/src/llm.ts @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: 2025 LiveKit, Inc. +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. // // SPDX-License-Identifier: Apache-2.0 import Anthropic from '@anthropic-ai/sdk'; From c8d2e5b8cb1d33a883e839e13188d11b7fbed2cf Mon Sep 17 00:00:00 2001 From: "rosetta-livekit-bot[bot]" <282703043+rosetta-livekit-bot[bot]@users.noreply.github.com> Date: Mon, 25 May 2026 21:47:38 +0000 Subject: [PATCH 11/12] fix(anthropic): address stream review feedback --- plugins/anthropic/src/llm.ts | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/plugins/anthropic/src/llm.ts b/plugins/anthropic/src/llm.ts index efc17fbd6..31f6f83b0 100644 --- a/plugins/anthropic/src/llm.ts +++ b/plugins/anthropic/src/llm.ts @@ -334,9 +334,10 @@ export class LLMStream extends llm.LLMStream { // Filter chain-of-thought blocks when tools are active if (this.#toolCtx) { - if (text.startsWith('')) { + if (text.includes('')) { this.#ignoringCoT = true; - } else if (this.#ignoringCoT && text.includes('')) { + } + if (this.#ignoringCoT && text.includes('')) { text = text.split('').pop() || ''; this.#ignoringCoT = false; } @@ -362,16 +363,11 @@ export class LLMStream extends llm.LLMStream { delta: { role: 'assistant', toolCalls: [ - { - id: '', - type: 'function_call', + llm.FunctionCall.create({ callId: this.#toolCallId, name: this.#fncName || '', args: this.#fncRawArgs || '', - createdAt: Date.now(), - extra: {}, - toJSON: () => ({}), - }, + }), ], }, }); @@ -394,12 +390,22 @@ export class LLMStream extends llm.LLMStream { }); } catch (e: unknown) { if (e instanceof Anthropic.APIError) { - if (e.status === 408 || e.status === 409) { + if (e.status === 408) { throw new APITimeoutError({ message: e.message, options: { retryable }, }); } + if (e.status === 409) { + throw new APIStatusError({ + message: e.message, + options: { + statusCode: e.status, + body: e.error as object, + retryable, + }, + }); + } throw new APIStatusError({ message: e.message, options: { From 9f8f651dd03fbd48a0b6a04f6671f2827523fa22 Mon Sep 17 00:00:00 2001 From: "rosetta-livekit-bot[bot]" <282703043+rosetta-livekit-bot[bot]@users.noreply.github.com> Date: Mon, 25 May 2026 22:42:11 +0000 Subject: [PATCH 12/12] fix(anthropic): preserve text around thinking blocks --- plugins/anthropic/src/llm.test.ts | 91 +++++++++++++++++++++++++++++++ plugins/anthropic/src/llm.ts | 21 +++++-- 2 files changed, 108 insertions(+), 4 deletions(-) diff --git a/plugins/anthropic/src/llm.test.ts b/plugins/anthropic/src/llm.test.ts index df746dc39..c476967f7 100644 --- a/plugins/anthropic/src/llm.test.ts +++ b/plugins/anthropic/src/llm.test.ts @@ -6,6 +6,53 @@ import { llm } from '@livekit/agents'; import { describe, expect, it } from 'vitest'; import { LLM } from './llm.js'; +function messageStartEvent(): Anthropic.MessageStreamEvent { + return { + type: 'message_start', + message: { + id: 'msg_123', + usage: { input_tokens: 0, output_tokens: 0 }, + }, + } as Anthropic.MessageStreamEvent; +} + +function textDeltaEvent(text: string): Anthropic.MessageStreamEvent { + return { + type: 'content_block_delta', + delta: { type: 'text_delta', text }, + } as Anthropic.MessageStreamEvent; +} + +async function collectTextFromEvents( + events: Anthropic.MessageStreamEvent[], + toolCtx?: llm.ToolContext, +): Promise { + const client = { + messages: { + create: async () => + (async function* (): AsyncGenerator { + yield* events; + })(), + }, + } as unknown as Anthropic; + const anthropicLlm = new LLM({ + apiKey: 'dummy', + client, + model: 'claude-3-5-sonnet-20241022', + }); + const chatCtx = new llm.ChatContext(); + chatCtx.addMessage({ role: 'user', content: 'Hello, world!' }); + + const stream = anthropicLlm.chat({ chatCtx, toolCtx }); + const textChunks: string[] = []; + for await (const chunk of stream) { + if (chunk.delta?.content) { + textChunks.push(chunk.delta.content); + } + } + return textChunks.join(''); +} + describe('Anthropic LLM', () => { it('correctly maps ChatContext to Anthropic system and messages arrays', () => { const anthropicLlm = new LLM({ apiKey: 'dummy', model: 'claude-3-5-sonnet-20241022' }); @@ -147,4 +194,48 @@ describe('Anthropic LLM', () => { totalTokens: 0, }); }); + + it('filters thinking blocks when tools are active', async () => { + const text = await collectTextFromEvents( + [ + messageStartEvent(), + textDeltaEvent('hidden'), + textDeltaEvent('still hiddenvisible'), + ], + {}, + ); + + expect(text).toBe('visible'); + }); + + it('does not filter thinking blocks without tools', async () => { + const text = await collectTextFromEvents([ + messageStartEvent(), + textDeltaEvent('visible without tools'), + ]); + + expect(text).toBe('visible without tools'); + }); + + it('preserves text around same-delta thinking blocks', async () => { + const text = await collectTextFromEvents( + [messageStartEvent(), textDeltaEvent('before hidden after')], + {}, + ); + + expect(text).toBe('before after'); + }); + + it('preserves text before split thinking blocks', async () => { + const text = await collectTextFromEvents( + [ + messageStartEvent(), + textDeltaEvent('before hidden'), + textDeltaEvent('still hidden after'), + ], + {}, + ); + + expect(text).toBe('before after'); + }); }); diff --git a/plugins/anthropic/src/llm.ts b/plugins/anthropic/src/llm.ts index 31f6f83b0..4f963d462 100644 --- a/plugins/anthropic/src/llm.ts +++ b/plugins/anthropic/src/llm.ts @@ -334,12 +334,25 @@ export class LLMStream extends llm.LLMStream { // Filter chain-of-thought blocks when tools are active if (this.#toolCtx) { - if (text.includes('')) { + const thinkingStart = text.indexOf(''); + if (thinkingStart >= 0) { + const preThinking = text.slice(0, thinkingStart); + if (preThinking) { + this.queue.put({ + id: this.#requestId, + delta: { role: 'assistant', content: preThinking }, + }); + retryable = false; + } + text = text.slice(thinkingStart + ''.length); this.#ignoringCoT = true; } - if (this.#ignoringCoT && text.includes('')) { - text = text.split('').pop() || ''; - this.#ignoringCoT = false; + if (this.#ignoringCoT) { + const thinkingEnd = text.indexOf(''); + if (thinkingEnd >= 0) { + text = text.slice(thinkingEnd + ''.length); + this.#ignoringCoT = false; + } } }