diff --git a/.changeset/tool-model-output-content-parts.md b/.changeset/tool-model-output-content-parts.md new file mode 100644 index 000000000..39d251eb1 --- /dev/null +++ b/.changeset/tool-model-output-content-parts.md @@ -0,0 +1,5 @@ +--- +"eve": patch +--- + +`toModelOutput` can now return `{ type: "content", value }` with text and file parts, so a tool can send images (screenshots, rendered charts) to vision-capable models as actual pixels instead of descriptions. Build outputs with the new `toolOutput.text` / `toolOutput.json` / `toolOutput.content` helpers and parts with `toolOutputPart.text` / `toolOutputPart.file`, all from `eve/tools`; file payloads must be base64 strings. diff --git a/docs/tools/overview.mdx b/docs/tools/overview.mdx index b0fa3fbea..20b8d8a58 100644 --- a/docs/tools/overview.mdx +++ b/docs/tools/overview.mdx @@ -82,6 +82,33 @@ toModelOutput(output) { Tool outputs must be JSON-serializable. Return plain objects, arrays, strings, numbers, booleans, or `null`; convert values like `Date`, `Map`, `Set`, `NaN`, and cyclic objects before returning them from `execute` or from a `{ type: "json" }` `toModelOutput`. +### Send images to the model with content parts + +A tool that produces an image — a screenshot, a rendered chart — can hand the pixels to a vision-capable model by returning a `content` output from `toModelOutput`. Build outputs with the `toolOutput` helpers and parts with the `toolOutputPart` helpers, both from `eve/tools`: + +```ts +import { defineTool, toolOutput, toolOutputPart } from "eve/tools"; + +export default defineTool({ + description: "Capture a screenshot of the current page", + inputSchema: z.object({ url: z.string() }), + async execute(input) { + const png = await captureScreenshot(input.url); + return { path: png.path, screenshotBase64: png.base64 }; + }, + toModelOutput(output) { + return toolOutput.content([ + toolOutputPart.text(`Screenshot of ${output.path}:`), + toolOutputPart.file(output.screenshotBase64, { mediaType: "image/png" }), + ]); + }, +}); +``` + +The `toolOutput.text` and `toolOutput.json` builders construct the other two output shapes; hand-written literals remain valid everywhere. + +File payloads must be base64 strings — raw bytes (`Uint8Array`, `Buffer`) are rejected because they do not survive eve's durable JSON boundary. Keep payloads small: a content-part image is persisted in session history and re-sent on every subsequent model call, and eve warns above 3 MiB. Sending image parts to a model without vision support fails with that provider's error, the same as image parts in user messages. + Do not return secrets, credentials, unnecessary personal data, or unbounded sensitive content from tools. Filter, minimize, and redact tool outputs before returning them. ## What to read next diff --git a/e2e/fixtures/agent-tools/agent/tools/render-stripes.ts b/e2e/fixtures/agent-tools/agent/tools/render-stripes.ts new file mode 100644 index 000000000..7f564b9fc --- /dev/null +++ b/e2e/fixtures/agent-tools/agent/tools/render-stripes.ts @@ -0,0 +1,79 @@ +import { crc32, deflateSync } from "node:zlib"; +import { defineTool, toolOutput, toolOutputPart } from "eve/tools"; +import { z } from "zod"; + +// Single-token names no model paraphrases (unlike cyan/teal or purple/violet), +// so the eval can match the reply against the rendered sequence verbatim. +const PALETTE = { + black: [0, 0, 0], + blue: [0, 0, 255], + green: [0, 160, 0], + orange: [255, 140, 0], + red: [255, 0, 0], + yellow: [255, 220, 0], +} as const; + +type ColorName = keyof typeof PALETTE; + +const WIDTH = 240; +const HEIGHT = 120; +const STRIPE_COUNT = 3; + +function pngChunk(type: string, data: Buffer): Buffer { + const length = Buffer.alloc(4); + length.writeUInt32BE(data.length); + const typeBuffer = Buffer.from(type, "latin1"); + const crc = Buffer.alloc(4); + crc.writeUInt32BE(crc32(Buffer.concat([typeBuffer, data])) >>> 0); + return Buffer.concat([length, typeBuffer, data, crc]); +} + +function renderStripesPng(stripes: readonly ColorName[]): Buffer { + const ihdr = Buffer.alloc(13); + ihdr.writeUInt32BE(WIDTH, 0); + ihdr.writeUInt32BE(HEIGHT, 4); + ihdr[8] = 8; // bit depth + ihdr[9] = 2; // truecolor + + const stripeWidth = WIDTH / stripes.length; + const row = Buffer.alloc(1 + WIDTH * 3); // leading scanline filter byte 0 + for (let x = 0; x < WIDTH; x += 1) { + const stripe = Math.min(Math.floor(x / stripeWidth), stripes.length - 1); + const color = PALETTE[stripes[stripe]!]; + row.set(color, 1 + x * 3); + } + const idat = deflateSync(Buffer.concat(Array.from({ length: HEIGHT }, () => row)), { level: 9 }); + + return Buffer.concat([ + Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]), + pngChunk("IHDR", ihdr), + pngChunk("IDAT", idat), + pngChunk("IEND", Buffer.alloc(0)), + ]); +} + +export default defineTool({ + description: + "Smoke-test fixture: renders an image of colored vertical stripes chosen at random. " + + "Only call when the user explicitly asks to use `render-stripes`. The colors appear " + + "ONLY in the returned image — inspect it visually; do not guess.", + inputSchema: z.object({}), + async execute() { + const names = Object.keys(PALETTE) as ColorName[]; + const colors = [...names].sort(() => Math.random() - 0.5).slice(0, STRIPE_COUNT); + const png = renderStripesPng(colors); + // `colors` is the eval's answer key. It reaches action.result (and the + // eval's event stream) but never the model: the projection below sends + // only the pixels. + return { colors, imageBase64: png.toString("base64") }; + }, + toModelOutput(output) { + return toolOutput.content([ + toolOutputPart.text("Rendered stripes:"), + toolOutputPart.file(output.imageBase64, { + filename: "stripes.png", + mediaType: "image/png", + }), + ]); + }, +}); diff --git a/e2e/fixtures/agent-tools/evals/static-tools/to-model-output-content-parts.eval.ts b/e2e/fixtures/agent-tools/evals/static-tools/to-model-output-content-parts.eval.ts new file mode 100644 index 000000000..cab2a09d0 --- /dev/null +++ b/e2e/fixtures/agent-tools/evals/static-tools/to-model-output-content-parts.eval.ts @@ -0,0 +1,82 @@ +import type { HandleMessageStreamEvent } from "eve/client"; +import { defineEval } from "eve/evals"; + +const TOOL_NAME = "render-stripes"; + +// The stripe colors are randomized per run, so a blind model cannot pass by +// guessing; the eval stays self-contained by validating the reply against +// the answer key the tool records on action.result. The pixels reach the +// model exclusively through `toModelOutput` content parts. +export default defineEval({ + description: "Static tools smoke: toModelOutput content parts deliver an image to the model.", + async test(t) { + await t.send( + `Call \`${TOOL_NAME}\` exactly once, look at the rendered image, and reply with only ` + + "the stripe colors left to right, comma-separated.", + ); + + t.succeeded(); + t.noFailedActions(); + t.calledTool(TOOL_NAME, { count: 1, output: isRenderStripesOutput }); + t.eventsSatisfy("a reply names the rendered colors in order", (events) => { + const answer = assistantAnswers(events)[0]; + return answer !== undefined && namesColorsInOrder(events, answer); + }); + + // The content part is baked into persisted history, so a follow-up turn + // must answer from replay without re-running the tool. + await t.send( + "Without calling any tool, repeat the stripe colors left to right, comma-separated.", + ); + + t.succeeded(); + t.calledTool(TOOL_NAME, { count: 1 }); + t.eventsSatisfy("the replayed image still answers the follow-up", (events) => { + const answer = assistantAnswers(events).at(-1); + return answer !== undefined && namesColorsInOrder(events, answer); + }); + }, +}); + +function isRenderStripesOutput(value: unknown): boolean { + if (typeof value !== "object" || value === null) return false; + const output = value as { readonly colors?: unknown; readonly imageBase64?: unknown }; + return ( + Array.isArray(output.colors) && + output.colors.length > 0 && + output.colors.every((color) => typeof color === "string") && + typeof output.imageBase64 === "string" && + output.imageBase64.startsWith("iVBOR") // PNG magic bytes, base64-encoded + ); +} + +function renderedColors(events: readonly HandleMessageStreamEvent[]): readonly string[] { + for (const event of events) { + if (event.type !== "action.result" || event.data.result.kind !== "tool-result") continue; + if (event.data.result.toolName !== TOOL_NAME) continue; + const output = event.data.result.output as { readonly colors?: unknown }; + if (Array.isArray(output?.colors) && output.colors.every((c) => typeof c === "string")) { + return output.colors as string[]; + } + } + return []; +} + +/** Final (non-tool-call) assistant messages, in turn order. */ +function assistantAnswers(events: readonly HandleMessageStreamEvent[]): readonly string[] { + return events.flatMap((event) => + event.type === "message.completed" && + event.data.finishReason !== "tool-calls" && + event.data.message !== null && + event.data.message.trim().length > 0 + ? [event.data.message] + : [], + ); +} + +function namesColorsInOrder(events: readonly HandleMessageStreamEvent[], answer: string): boolean { + const colors = renderedColors(events); + if (colors.length === 0) return false; + const pattern = new RegExp(colors.map((color) => `\\b${color}\\b`).join("[\\s\\S]*"), "iu"); + return pattern.test(answer); +} diff --git a/packages/eve/extension-contracts/compatibility/dynamicTool/v3.ts b/packages/eve/extension-contracts/compatibility/dynamicTool/v3.ts new file mode 100644 index 000000000..675d31a89 --- /dev/null +++ b/packages/eve/extension-contracts/compatibility/dynamicTool/v3.ts @@ -0,0 +1,26 @@ +import { defineDynamic, defineTool } from "#public/tools/index.js"; + +export default defineDynamic({ + events: { + "session.started": (_event, ctx) => ({ + summarize_report: defineTool({ + description: "Summarize a report for the model", + inputSchema: { type: "object", properties: {} }, + async execute(_input, toolContext) { + return { + internal: "details", + resolverSessionId: ctx.session.id, + sessionId: toolContext.session.id, + summary: "Report generated", + }; + }, + toModelOutput(output) { + return { + type: "text", + value: (output as { summary: string }).summary, + }; + }, + }), + }), + }, +}); diff --git a/packages/eve/extension-contracts/compatibility/tool/v2.ts b/packages/eve/extension-contracts/compatibility/tool/v2.ts new file mode 100644 index 000000000..801d59638 --- /dev/null +++ b/packages/eve/extension-contracts/compatibility/tool/v2.ts @@ -0,0 +1,19 @@ +import { defineTool } from "#public/tools/index.js"; + +export default defineTool({ + description: "Summarize a report for the model", + inputSchema: { type: "object", properties: {} }, + async execute(_input, ctx) { + return { + internal: "details", + sessionId: ctx.session.id, + summary: "Report generated", + }; + }, + toModelOutput(output) { + return { + type: "text", + value: (output as { summary: string }).summary, + }; + }, +}); diff --git a/packages/eve/extension-contracts/entrypoints/tool.ts b/packages/eve/extension-contracts/entrypoints/tool.ts index b03622f7f..f28a73a37 100644 --- a/packages/eve/extension-contracts/entrypoints/tool.ts +++ b/packages/eve/extension-contracts/entrypoints/tool.ts @@ -9,5 +9,7 @@ export { experimental_workflow, isDisabledToolSentinel, isExperimentalWorkflowToolDefinition, + toolOutput, + toolOutputPart, toolResultFrom, } from "../../src/public/tools/index.ts"; diff --git a/packages/eve/extension-contracts/reports/dynamicTool/v4.json b/packages/eve/extension-contracts/reports/dynamicTool/v4.json new file mode 100644 index 000000000..dcaf3e5e8 --- /dev/null +++ b/packages/eve/extension-contracts/reports/dynamicTool/v4.json @@ -0,0 +1,13 @@ +{ + "kind": "eve-extension-capability-contract", + "capability": "dynamicTool", + "epoch": 4, + "sha256": "d5ed3d2576c48abe76044aaf8a9aa0e8ab734d3fc6d4d3bc7be91c3a6648a198", + "exports": [ + "DynamicToolEntry", + "DynamicToolEvents", + "DynamicToolResult", + "DynamicToolSet", + "defineDynamic" + ] +} diff --git a/packages/eve/extension-contracts/reports/tool/v3.json b/packages/eve/extension-contracts/reports/tool/v3.json new file mode 100644 index 000000000..fb1b02c70 --- /dev/null +++ b/packages/eve/extension-contracts/reports/tool/v3.json @@ -0,0 +1,21 @@ +{ + "kind": "eve-extension-capability-contract", + "capability": "tool", + "epoch": 3, + "sha256": "64e6354ca546aaa2bbd07a05b5fb66596d34d14633222a41e5c647898211e14f", + "exports": [ + "defineBashTool", + "defineGlobTool", + "defineGrepTool", + "defineReadFileTool", + "defineTool", + "defineWriteFileTool", + "disableTool", + "experimental_workflow", + "isDisabledToolSentinel", + "isExperimentalWorkflowToolDefinition", + "toolOutput", + "toolOutputPart", + "toolResultFrom" + ] +} diff --git a/packages/eve/src/compiler/extension-compatibility.ts b/packages/eve/src/compiler/extension-compatibility.ts index 13a8d2d47..fd3151903 100644 --- a/packages/eve/src/compiler/extension-compatibility.ts +++ b/packages/eve/src/compiler/extension-compatibility.ts @@ -21,8 +21,8 @@ interface ExtensionCapabilityContract { const EXTENSION_CAPABILITY_CONTRACTS = { extension: { current: 1, supported: [1], dropped: {} }, - tool: { current: 2, supported: [1, 2], dropped: {} }, - dynamicTool: { current: 3, supported: [1, 2, 3], dropped: {} }, + tool: { current: 3, supported: [1, 2, 3], dropped: {} }, + dynamicTool: { current: 4, supported: [1, 2, 3, 4], dropped: {} }, connection: { current: 2, supported: [1, 2], dropped: {} }, hook: { current: 2, supported: [1, 2], dropped: {} }, skill: { current: 1, supported: [1], dropped: {} }, diff --git a/packages/eve/src/discover/agent.integration.test.ts b/packages/eve/src/discover/agent.integration.test.ts index ad08e405a..90cd2bb19 100644 --- a/packages/eve/src/discover/agent.integration.test.ts +++ b/packages/eve/src/discover/agent.integration.test.ts @@ -3,6 +3,10 @@ import { resolve } from "node:path"; import { describe, expect, it } from "vitest"; import { buildMemoryAgentProject } from "#internal/testing/memory-agent-source.js"; +import { + EXTENSION_CAPABILITY_SUPPORT, + EXTENSION_CAPABILITY_VERSIONS, +} from "#compiler/extension-compatibility.js"; import { discoverAgent } from "#discover/discover-agent.js"; import { DISCOVER_EXTENSION_CAPABILITY_INCOMPATIBLE, @@ -983,6 +987,9 @@ describe("discoverAgent (memory)", () => { }); it("rejects a mounted extension that requires an unsupported capability version", async () => { + // One past the current epoch is unsupported by construction, so the + // fixture keeps rejecting after future capability bumps. + const unsupportedToolVersion = EXTENSION_CAPABILITY_VERSIONS.tool + 1; const project = buildMemoryAgentProject({ appFiles: { "node_modules/@acme/crm/package.json": JSON.stringify({ @@ -994,7 +1001,7 @@ describe("discoverAgent (memory)", () => { kind: "eve-extension", formatVersion: 1, builtWithEve: "9.0.0", - requires: { extension: 1, tool: 3 }, + requires: { extension: 1, tool: unsupportedToolVersion }, }), "node_modules/@acme/crm/extension/extension.ts": "export default {};\n", "node_modules/@acme/crm/extension/tools/search.ts": "export default {};\n", @@ -1015,8 +1022,10 @@ describe("discoverAgent (memory)", () => { (diagnostic) => diagnostic.code === DISCOVER_EXTENSION_CAPABILITY_INCOMPATIBLE, ); expect(incompatible).toBeDefined(); - expect(incompatible?.message).toContain("tool contract v3"); - expect(incompatible?.message).toContain("versions: v1, v2"); + expect(incompatible?.message).toContain(`tool contract v${unsupportedToolVersion}`); + expect(incompatible?.message).toContain( + `versions: ${EXTENSION_CAPABILITY_SUPPORT.tool.map((version) => `v${version}`).join(", ")}`, + ); expect(result.manifest.resolvedExtensions).toEqual([]); }); diff --git a/packages/eve/src/harness/tool-model-output.test.ts b/packages/eve/src/harness/tool-model-output.test.ts new file mode 100644 index 000000000..e5d2fbe9a --- /dev/null +++ b/packages/eve/src/harness/tool-model-output.test.ts @@ -0,0 +1,132 @@ +import { describe, expect, it } from "vitest"; + +import { toolOutput, toolOutputPart } from "#public/tools/output-builders.js"; +import { normalizeToolJsonOutput, normalizeToolModelOutput } from "#harness/tool-model-output.js"; + +function normalize(output: unknown): unknown { + return normalizeToolModelOutput({ output, toolCallId: "call_1", toolName: "screenshot" }); +} + +describe("normalizeToolJsonOutput", () => { + it("normalizes top-level undefined to null", () => { + expect( + normalizeToolJsonOutput({ boundary: "execute", output: undefined, toolName: "echo" }), + ).toBeNull(); + }); + + it("rejects non-JSON-serializable values with the boundary in the error", () => { + expect(() => + normalizeToolJsonOutput({ + boundary: "execute", + output: { now: new Date("2026-01-02T03:04:05.000Z") }, + toolCallId: "call_timestamp", + toolName: "timestamp", + }), + ).toThrow( + 'Tool "timestamp" call "call_timestamp" returned a non-JSON-serializable result. ' + + "Expected a JSON-serializable value.", + ); + }); +}); + +describe("normalizeToolModelOutput", () => { + it("normalizes a content output of text and file parts into the AI SDK shape", () => { + const pixel = + "iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAIAAAAlC+aJAAAAS0lEQVR42u3PQQkAAAgAsetfWiP4FgYrsKZeS0BAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEDgsqnc8OJg6Ln3AAAAAElFTkSuQmCC"; + + expect( + normalize({ + type: "content", + value: [ + { type: "text", text: "Screenshot:" }, + { + type: "file", + data: { type: "data", data: pixel }, + mediaType: "image/png", + filename: "pixel.png", + }, + ], + }), + ).toEqual({ + type: "content", + value: [ + { type: "text", text: "Screenshot:" }, + { + type: "file", + data: { type: "data", data: pixel }, + mediaType: "image/png", + filename: "pixel.png", + }, + ], + }); + }); + + it("accepts outputs built with the toolOutput builders", () => { + expect(normalize(toolOutput.text("visible"))).toEqual({ type: "text", value: "visible" }); + expect(normalize(toolOutput.json({ summary: "ok" }))).toEqual({ + type: "json", + value: { summary: "ok" }, + }); + expect( + normalize( + toolOutput.content([ + toolOutputPart.text("Screenshot:"), + toolOutputPart.file("aGVsbG8=", { mediaType: "image/png" }), + ]), + ), + ).toEqual({ + type: "content", + value: [ + { type: "text", text: "Screenshot:" }, + { type: "file", data: { type: "data", data: "aGVsbG8=" }, mediaType: "image/png" }, + ], + }); + }); + + it.each([ + [ + "raw Uint8Array payload", + { type: "data", data: new Uint8Array([1, 2, 3]) }, + /base64 string, received raw bytes/u, + ], + ["raw untagged bytes", new Uint8Array([1, 2, 3]), /base64 string, received raw bytes/u], + [ + "url file-data tag", + { type: "url", url: "https://example.com/a.png" }, + /"url" is not supported yet/u, + ], + [ + "reference file-data tag", + { type: "reference", reference: "file_abc" }, + /"reference" is not supported yet/u, + ], + ["text file-data tag", { type: "text", text: "inline" }, /"text" is not supported yet/u], + ["untagged string data", "aGVsbG8=", /expected object, received string at "value\[0\]\.data"/u], + ] as const)("rejects content file parts with %s", (_label, data, expected) => { + expect(() => + normalize({ type: "content", value: [{ type: "file", data, mediaType: "image/png" }] }), + ).toThrow(expected); + }); + + it("rejects empty content part arrays with the toModelOutput error identity", () => { + expect(() => normalize({ type: "content", value: [] })).toThrow( + 'Tool "screenshot" call "call_1" returned a non-JSON-serializable model output. ' + + 'Too small: expected array to have >=1 items at "value"', + ); + }); + + it("rejects unknown content part types", () => { + expect(() => + normalize({ + type: "content", + value: [{ type: "media", data: "aGVsbG8=", mediaType: "image/png" }], + }), + ).toThrow(/Invalid discriminator value. Expected 'text' \| 'file' at "value\[0\]\.type"/u); + }); + + it("rejects unknown output types", () => { + expect(() => normalize({ type: "markdown", value: "# nope" })).toThrow( + /Invalid discriminator value. Expected 'text' \| 'json' \| 'content' at "type"/u, + ); + }); +}); diff --git a/packages/eve/src/harness/tool-model-output.ts b/packages/eve/src/harness/tool-model-output.ts new file mode 100644 index 000000000..9ad4bb66e --- /dev/null +++ b/packages/eve/src/harness/tool-model-output.ts @@ -0,0 +1,181 @@ +import type { JSONValue } from "ai"; + +import { z } from "#compiled/zod/index.js"; +import { createLogger } from "#internal/logging.js"; +import { parseJsonValue, type JsonValue } from "#shared/json.js"; +import type { ToolModelOutputPart } from "#shared/tool-definition.js"; +import { formatValidationError } from "#runtime/validation.js"; +import { withToolOutputSerializationError } from "#harness/tool-output-serialization.js"; + +/** + * A validated {@link ToolModelOutput} in the AI SDK's expected shape: + * `json` values are proven JSON-serializable and `content` arrays are + * mutable so the value is assignable to the SDK's `ToolResultOutput`. + */ +export type ToolModelOutputValue = + | { readonly type: "json"; readonly value: JSONValue } + | { readonly type: "text"; readonly value: string } + | { readonly type: "content"; readonly value: ToolModelOutputPart[] }; + +const log = createLogger("harness.tool-model-output"); + +/** + * Content-part file payloads above this decoded size warn. Matches the + * attachment pipeline's inline-image hydration cap: unlike sandbox-ref + * attachments, a content part is baked into persisted history and + * re-sent on every subsequent model call, so large payloads compound. + */ +const CONTENT_FILE_WARN_BYTES = 3 * 1024 * 1024; + +/** File-data tags the AI SDK accepts but eve does not support yet. */ +const UNSUPPORTED_FILE_DATA_TAGS = new Set(["url", "reference", "text"]); + +const toolModelOutputPartSchema = z.discriminatedUnion("type", [ + z.object({ type: z.literal("text"), text: z.string() }), + z.object({ + type: z.literal("file"), + data: z.object({ type: z.literal("data"), data: z.string() }), + mediaType: z.string().min(1), + filename: z.string().optional(), + }), +]); + +const toolModelOutputSchema = z.discriminatedUnion("type", [ + z.object({ type: z.literal("text"), value: z.string() }), + z.object({ type: z.literal("json"), value: z.unknown() }), + z.object({ type: z.literal("content"), value: z.array(toolModelOutputPartSchema).min(1) }), +]); + +/** + * Validates a tool output as JSON at one of the serialization boundaries, + * normalizing top-level `undefined` to `null`. Throws + * `ToolOutputSerializationError` on non-JSON-serializable values. + */ +export function normalizeToolJsonOutput(input: { + readonly boundary: "execute" | "toModelOutput"; + readonly output: unknown; + readonly toolCallId?: string; + readonly toolName: string; +}): JsonValue { + const candidate = input.output === undefined ? null : input.output; + + return withToolOutputSerializationError(input, () => { + parseJsonValue(candidate); + return candidate as JsonValue; + }); +} + +/** + * Single funnel for authored `toModelOutput` results. Validates the + * eve-owned {@link ToolModelOutput} union and returns it in the AI SDK's + * expected shape; every rejection throws `ToolOutputSerializationError` + * at the `toModelOutput` boundary. + */ +export function normalizeToolModelOutput(input: { + readonly output: unknown; + readonly toolCallId?: string; + readonly toolName: string; +}): ToolModelOutputValue { + return withToolOutputSerializationError( + { + boundary: "toModelOutput", + toolCallId: input.toolCallId, + toolName: input.toolName, + }, + () => { + assertSupportedFilePartData(input.output); + + const parsed = toolModelOutputSchema.safeParse(input.output); + if (!parsed.success) { + throw new TypeError(formatValidationError(parsed.error)); + } + + const output = parsed.data; + if (output.type === "json") { + return { + type: "json", + value: normalizeToolJsonOutput({ + boundary: "toModelOutput", + output: output.value, + toolCallId: input.toolCallId, + toolName: input.toolName, + }) as JSONValue, + }; + } + if (output.type === "content") { + for (const part of output.value) { + if (part.type === "file") { + warnOversizedFilePayload(part, input.toolName); + } + } + } + return output; + }, + ); +} + +/** + * Asserts every file part's `data` is the supported form, covering the two + * author mistakes the schema cannot explain well: raw bytes (which + * `JSON.stringify` would corrupt silently at the durable boundary — the + * #497 failure class), and AI SDK `FileData` tags eve does not support + * yet. On a `Uint8Array` the schema would report a missing `data.type` + * key; the real fix is to base64-encode. Everything structural is left to + * the schema. + */ +function assertSupportedFilePartData(output: unknown): void { + if (output === null || typeof output !== "object") return; + const candidate = output as { readonly type?: unknown; readonly value?: unknown }; + if (candidate.type !== "content" || !Array.isArray(candidate.value)) return; + + for (const part of candidate.value) { + if (part === null || typeof part !== "object") continue; + const filePart = part as { readonly type?: unknown; readonly data?: unknown }; + if (filePart.type !== "file") continue; + + const data = filePart.data; + const taggedPayload = + data !== null && typeof data === "object" + ? (data as { readonly data?: unknown }).data + : undefined; + if (isBinaryPayload(data) || isBinaryPayload(taggedPayload)) { + throw new TypeError( + "Expected file content part data to be a base64 string, received raw bytes. " + + "Base64-encode binary payloads before returning them from toModelOutput.", + ); + } + + const tag = + data !== null && typeof data === "object" + ? (data as { readonly type?: unknown }).type + : undefined; + if (typeof tag === "string" && UNSUPPORTED_FILE_DATA_TAGS.has(tag)) { + throw new TypeError( + `File content part data tag "${tag}" is not supported yet; ` + + 'only { type: "data" } with a base64 string is accepted.', + ); + } + } +} + +function warnOversizedFilePayload( + part: { readonly data: { readonly data: string }; readonly mediaType: string }, + toolName: string, +): void { + const estimatedBytes = Math.floor((part.data.data.length * 3) / 4); + if (estimatedBytes <= CONTENT_FILE_WARN_BYTES) return; + log.warn( + "content-part file payload exceeds the inline size guideline — it is persisted in " + + "history and re-sent on every subsequent model call", + { + estimatedBytes, + mediaType: part.mediaType, + toolName, + warnBytes: CONTENT_FILE_WARN_BYTES, + }, + ); +} + +function isBinaryPayload(value: unknown): value is Uint8Array | ArrayBuffer { + return value instanceof Uint8Array || value instanceof ArrayBuffer; +} diff --git a/packages/eve/src/harness/tools.test.ts b/packages/eve/src/harness/tools.test.ts index e2d665d42..0cba032bd 100644 --- a/packages/eve/src/harness/tools.test.ts +++ b/packages/eve/src/harness/tools.test.ts @@ -5,6 +5,7 @@ import { ContextContainer, contextStorage } from "#context/container.js"; import { SessionKey, type Session } from "#context/keys.js"; import { SCHEDULE_APP_AUTH } from "#channel/schedule-auth.js"; import { always, never, once } from "#public/tools/approval/approval-helpers.js"; + import type { RuntimeModelReference } from "#runtime/agent/bootstrap.js"; import { WEB_SEARCH_ANTHROPIC_OUTPUT_SCHEMA, @@ -635,6 +636,51 @@ describe("buildToolSet", () => { ); }); + it("passes valid content toModelOutput values through in the AI SDK shape", async () => { + const pixel = + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/q842iQAAAABJRU5ErkJggg=="; + const tools: HarnessToolMap = new Map([ + [ + "screenshot", + { + description: "Capture a screenshot.", + execute: async () => ({ ok: true }), + inputSchema: jsonSchema({}), + name: "screenshot", + toModelOutput: () => ({ + type: "content" as const, + value: [ + { type: "text" as const, text: "Screenshot:" }, + { + type: "file" as const, + data: { type: "data" as const, data: pixel }, + mediaType: "image/png", + filename: "pixel.png", + }, + ], + }), + }, + ], + ]); + + const result = buildToolSet({ tools }); + + await expect( + projectSdkToolOutput({ output: { ok: true }, tool: result.screenshot }), + ).resolves.toEqual({ + type: "content", + value: [ + { type: "text", text: "Screenshot:" }, + { + type: "file", + data: { type: "data", data: pixel }, + mediaType: "image/png", + filename: "pixel.png", + }, + ], + }); + }); + it("passes valid text toModelOutput values through", async () => { const tools: HarnessToolMap = new Map([ [ diff --git a/packages/eve/src/harness/tools.ts b/packages/eve/src/harness/tools.ts index 4431365e9..46cbf6267 100644 --- a/packages/eve/src/harness/tools.ts +++ b/packages/eve/src/harness/tools.ts @@ -1,17 +1,10 @@ -import { - type JSONValue, - type ToolApprovalConfiguration, - type ToolApprovalStatus, - type ToolSet, - tool, -} from "ai"; +import { type ToolApprovalConfiguration, type ToolApprovalStatus, type ToolSet, tool } from "ai"; import type { SessionCapabilities } from "#channel/types.js"; import type { RuntimeModelReference } from "#runtime/agent/bootstrap.js"; import { ASK_QUESTION_TOOL_NAME } from "#runtime/framework-tools/ask-question.js"; import { WEB_SEARCH_TOOL_DEFINITION } from "#runtime/framework-tools/web-search.js"; import { isObject } from "#shared/guards.js"; -import { parseJsonValue, type JsonValue } from "#shared/json.js"; import type { HarnessToolDefinition } from "#harness/execute-tool.js"; import type { ApprovalStatus } from "#public/definitions/approval.js"; import { resolveWebSearchBackend, resolveWebSearchProviderTool } from "#harness/provider-tools.js"; @@ -25,13 +18,9 @@ import { modelFacingAuthorizationOutput, } from "#harness/authorization.js"; import { stashToolInterrupt } from "#harness/tool-interrupts.js"; -import { withToolOutputSerializationError } from "#harness/tool-output-serialization.js"; +import { normalizeToolJsonOutput, normalizeToolModelOutput } from "#harness/tool-model-output.js"; import type { ToolExecuteOptions } from "#shared/tool-definition.js"; -type ToolModelOutputValue = - | { readonly type: "json"; readonly value: JSONValue } - | { readonly type: "text"; readonly value: string }; - type NativeApprovalStatus = Exclude; const toolApprovals = new WeakMap< @@ -192,63 +181,6 @@ export function wrapToolExecute( }; } -function normalizeToolJsonOutput(input: { - readonly boundary: "execute" | "toModelOutput"; - readonly output: unknown; - readonly toolCallId?: string; - readonly toolName: string; -}): JsonValue { - const candidate = input.output === undefined ? null : input.output; - - return withToolOutputSerializationError(input, () => { - parseJsonValue(candidate); - return candidate as JsonValue; - }); -} - -function normalizeToolModelOutput(input: { - readonly output: unknown; - readonly toolCallId?: string; - readonly toolName: string; -}): ToolModelOutputValue { - return withToolOutputSerializationError( - { - boundary: "toModelOutput", - toolCallId: input.toolCallId, - toolName: input.toolName, - }, - () => { - if (input.output === null || typeof input.output !== "object") { - throw new TypeError("Expected a tool model output object."); - } - - const output = input.output as { readonly type?: unknown; readonly value?: unknown }; - - if (output.type === "text") { - if (typeof output.value !== "string") { - throw new TypeError('Expected text model output to include a string "value".'); - } - - return { type: "text", value: output.value }; - } - - if (output.type === "json") { - return { - type: "json", - value: normalizeToolJsonOutput({ - boundary: "toModelOutput", - output: output.value, - toolCallId: input.toolCallId, - toolName: input.toolName, - }) as JSONValue, - }; - } - - throw new TypeError('Expected tool model output type to be "text" or "json".'); - }, - ); -} - /** * Builds the AI SDK ToolSet for one harness step. * diff --git a/packages/eve/src/internal/nitro/host/build-extension.scenario.test.ts b/packages/eve/src/internal/nitro/host/build-extension.scenario.test.ts index 1e150589e..847df3a66 100644 --- a/packages/eve/src/internal/nitro/host/build-extension.scenario.test.ts +++ b/packages/eve/src/internal/nitro/host/build-extension.scenario.test.ts @@ -6,7 +6,10 @@ import { pathToFileURL } from "node:url"; import { describe, expect, it } from "vitest"; -import { parseExtensionCompatibilityManifest } from "#compiler/extension-compatibility.js"; +import { + EXTENSION_CAPABILITY_VERSIONS, + parseExtensionCompatibilityManifest, +} from "#compiler/extension-compatibility.js"; import { buildExtensionPackage, tryReadExtensionBuildConfig, @@ -223,13 +226,15 @@ describe("extension build output", () => { ); expect(manifest.kind).toBe("eve-extension"); expect(manifest.builtWithEve).toMatch(/^\d+\.\d+\.\d+/); + // The set of stamped capabilities is the assertion; the versions track + // the current contract table so epoch bumps do not break this test. expect(manifest.requires).toEqual({ - extension: 1, - tool: 2, - dynamicTool: 3, - skill: 1, - config: 1, - state: 2, + extension: EXTENSION_CAPABILITY_VERSIONS.extension, + tool: EXTENSION_CAPABILITY_VERSIONS.tool, + dynamicTool: EXTENSION_CAPABILITY_VERSIONS.dynamicTool, + skill: EXTENSION_CAPABILITY_VERSIONS.skill, + config: EXTENSION_CAPABILITY_VERSIONS.config, + state: EXTENSION_CAPABILITY_VERSIONS.state, }); const dynamicToolDeclaration = await readFile( join(outDir, "extension", "tools", "crm_search.d.ts"), diff --git a/packages/eve/src/public/definitions/tool.ts b/packages/eve/src/public/definitions/tool.ts index d6f21ecab..4dd0f4cb1 100644 --- a/packages/eve/src/public/definitions/tool.ts +++ b/packages/eve/src/public/definitions/tool.ts @@ -28,7 +28,7 @@ type DynamicEventMapResult = Awaited< ReturnType> >; -export type { ToolModelOutput } from "#shared/tool-definition.js"; +export type { ToolModelOutput, ToolModelOutputPart } from "#shared/tool-definition.js"; /** * Authorization provider passed to {@link ToolContext.getToken} or diff --git a/packages/eve/src/public/tools/index.ts b/packages/eve/src/public/tools/index.ts index efc0ee235..35bd0f198 100644 --- a/packages/eve/src/public/tools/index.ts +++ b/packages/eve/src/public/tools/index.ts @@ -17,7 +17,9 @@ export { type ToolDefinition, type ToolContext, type ToolModelOutput, + type ToolModelOutputPart, } from "#public/definitions/tool.js"; +export { toolOutput, toolOutputPart } from "#public/tools/output-builders.js"; export type { Approval, ApprovalContext, ApprovalStatus } from "#public/definitions/approval.js"; export type { DynamicToolEntry, diff --git a/packages/eve/src/public/tools/output-builders.ts b/packages/eve/src/public/tools/output-builders.ts new file mode 100644 index 000000000..c15124085 --- /dev/null +++ b/packages/eve/src/public/tools/output-builders.ts @@ -0,0 +1,78 @@ +import type { ToolModelOutput, ToolModelOutputPart } from "#shared/tool-definition.js"; + +/** + * Builders for the model-facing {@link ToolModelOutput} returned by + * `toModelOutput`. Pure sugar over the union — each returns the + * corresponding literal, and hand-written literals remain valid. + * + * ```ts + * toModelOutput(output) { + * return toolOutput.content([ + * toolOutputPart.text(`Screenshot of ${output.path}:`), + * toolOutputPart.file(output.screenshotBase64, { mediaType: "image/png" }), + * ]); + * } + * ``` + */ +export const toolOutput = { + /** Builds a text output: the model sees `value` as the tool result. */ + text(value: string): ToolModelOutput { + return { type: "text", value }; + }, + /** Builds a JSON output; `value` must be JSON-serializable. */ + json(value: unknown): ToolModelOutput { + return { type: "json", value }; + }, + /** Builds a content output from ordered {@link ToolModelOutputPart} entries. */ + content(value: readonly ToolModelOutputPart[]): ToolModelOutput { + return { type: "content", value }; + }, +}; + +/** + * Builders for `content` {@link ToolModelOutput} parts. Pure sugar over + * {@link ToolModelOutputPart} — each returns the corresponding part + * literal, and hand-written literals remain valid. + * + * ```ts + * toModelOutput(output) { + * return { + * type: "content", + * value: [ + * toolOutputPart.text(`Screenshot of ${output.path}:`), + * toolOutputPart.file(output.screenshotBase64, { mediaType: "image/png" }), + * ], + * }; + * } + * ``` + */ +export const toolOutputPart = { + /** Builds a text part. */ + text(text: string): ToolModelOutputPart { + return { type: "text", text }; + }, + /** + * Builds a file part from a base64 payload. Binary data must be + * base64-encoded by the caller; raw bytes are rejected by the harness + * because they do not survive the durable JSON boundary. + */ + file( + base64: string, + options: { readonly mediaType: string; readonly filename?: string }, + ): ToolModelOutputPart { + const part: { + type: "file"; + data: { type: "data"; data: string }; + mediaType: string; + filename?: string; + } = { + type: "file", + data: { type: "data", data: base64 }, + mediaType: options.mediaType, + }; + if (options.filename !== undefined) { + part.filename = options.filename; + } + return part; + }, +}; diff --git a/packages/eve/src/shared/tool-definition.ts b/packages/eve/src/shared/tool-definition.ts index 468fb4ba9..239a905af 100644 --- a/packages/eve/src/shared/tool-definition.ts +++ b/packages/eve/src/shared/tool-definition.ts @@ -72,7 +72,31 @@ export interface PublicToolDefinitionWithExecuteFn< * eve-owned shape for the model-facing tool result produced by * `toModelOutput`. Structurally compatible with the AI SDK's * `ToolResultOutput` so the harness can forward it without conversion. + * + * The `content` variant carries an ordered list of + * {@link ToolModelOutputPart} entries, letting a tool hand the model + * text alongside inline files (e.g. a screenshot as vision input). */ export type ToolModelOutput = | { readonly type: "text"; readonly value: string } - | { readonly type: "json"; readonly value: unknown }; + | { readonly type: "json"; readonly value: unknown } + | { readonly type: "content"; readonly value: readonly ToolModelOutputPart[] }; + +/** + * One part of a `content` {@link ToolModelOutput}. Mirrors the AI SDK's + * `ToolResultOutput` content parts narrowed to the JSON-safe subset: + * file data is the SDK's tagged `FileData` union restricted to + * `{ type: "data" }` with a base64 string, so persisted tool results + * survive the durable JSON boundary. Use the `toolOutputPart` builders + * from `eve/tools` to construct parts without hand-writing the nesting. + */ +export type ToolModelOutputPart = + | { readonly type: "text"; readonly text: string } + | { + readonly type: "file"; + /** Tagged file data; only JSON-safe base64 payloads are accepted. */ + readonly data: { readonly type: "data"; readonly data: string }; + /** IANA media type, e.g. `image/png`. */ + readonly mediaType: string; + readonly filename?: string; + };