diff --git a/README.md b/README.md index ad38a07..bc4a39f 100644 --- a/README.md +++ b/README.md @@ -65,7 +65,7 @@ describe("Greeting agent", () => { - **Classification Matcher**: Built-in `toBeClassifiedAs()` matcher to grade a response against a target label on a 1-5 scale - **Concurrent Execution**: Runs eval files concurrently for faster test execution - **Model Selection**: Support for specifying custom AI models -- **Runtime Selection**: Run prompts through GitHub Copilot (default) or Codex +- **Runtime Selection**: Run prompts through GitHub Copilot (default), Codex, or Claude Code - **Configurable Timeouts**: Override prompt wait time per test or via `katt.json` ## Usage @@ -157,11 +157,27 @@ Codex: } ``` +Claude Code: + +```json +{ + "agent": "claude-code", + "agentOptions": { + "model": "claude-sonnet-4", + "permissionMode": "acceptEdits" + }, + "prompt": { + "timeoutMs": 240000 + } +} +``` + When this file exists: - Supported agents are: - `gh-copilot` (default when `agent` is missing or unsupported) - `codex` + - `claude-code` - `prompt("...")` and `promptFile("...")` merge `agentOptions` with call-time options - `prompt("...", { model: "..." })` overrides the model from config - `prompt.timeoutMs` sets the default wait timeout for long-running prompts @@ -183,15 +199,18 @@ npm install - `npm run format` - Format code using Biome - `npm run lint` - Lint code using Biome - `npm run test:build` - Test the built CLI +- `npm run test:build:codex` - Test the built CLI with `katt-codex.json` +- `npm run test:build:claude` - Test the built CLI with `katt-claude.json` ### Verification Process To verify your changes before opening a pull request, run: -1. `npm test` +1. `npm run format` 2. `npm run typecheck` -3. `npm run lint` -4. `npm run format` +3. `npm run test` +4. `npm run build` +5. `npm run test:build` For more details, see the [verification process section in CONTRIBUTING.md](./CONTRIBUTING.md#verification-process). ## How It Works @@ -236,7 +255,7 @@ flowchart LR Config --> Runtime Runtime --> Assertions["Assertions + snapshots"] Runtime --> Prompts["prompt() / promptFile()"] - Prompts --> AI["AI runtime (GitHub Copilot or Codex CLI)"] + Prompts --> AI["AI runtime (GitHub Copilot, Codex CLI, or Claude Code CLI)"] Assertions --> Report["Terminal report + exit code"] AI --> Report ``` @@ -246,6 +265,7 @@ flowchart LR - Node.js - For `gh-copilot` runtime: access to GitHub Copilot with a logged-in user - For `codex` runtime: Codex CLI installed and authenticated (`codex login`) +- For `claude-code` runtime: Claude Code CLI installed and authenticated ## License diff --git a/docs/api-documentation.md b/docs/api-documentation.md index 9ad707e..6a581ae 100644 --- a/docs/api-documentation.md +++ b/docs/api-documentation.md @@ -33,7 +33,7 @@ This document lists the currently available Katt features and how to use them. - AI-based classification matcher: `toBeClassifiedAs` - Prompt execution with provider-specific runtime options - Prompt loading from files with relative-path resolution -- Default runtime configuration via `katt.json` (or `--config-file`) for `gh-copilot`/`codex` +- Default runtime configuration via `katt.json` (or `--config-file`) for `gh-copilot`/`codex`/`claude-code` - Configurable prompt timeout with a safer long-task default - Automatic discovery and execution of `*.eval.js` and `*.eval.ts` - Concurrent eval-file execution @@ -157,6 +157,16 @@ Sends `input` to the AI model and returns the response string. - `dangerouslyBypassApprovalsAndSandbox?: boolean` - `config?: string | string[]` (forwarded as `codex exec --config`) - `workingDirectory?: string` +- For `claude-code`: Claude Code CLI options: + - `model?: string` + - `permissionMode?: string` (forwarded as `--permission-mode`) + - `dangerouslySkipPermissions?: boolean` + - `maxTurns?: number` + - `allowedTools?: string | string[]` + - `disallowedTools?: string | string[]` + - `appendSystemPrompt?: string` + - `mcpConfig?: string` + - `workingDirectory?: string` - `timeoutMs?: number` to control how long to wait for prompt completion - Explicit options override matching keys from config `agentOptions` @@ -221,10 +231,26 @@ Or use Codex: } ``` +Or use Claude Code: + +```json +{ + "agent": "claude-code", + "agentOptions": { + "model": "claude-sonnet-4", + "permissionMode": "acceptEdits" + }, + "prompt": { + "timeoutMs": 240000 + } +} +``` + Behavior: - Supported agents: - `gh-copilot` (default when `agent` is missing or unsupported) - `codex` + - `claude-code` - Default config location is `/katt.json` - CLI override: `--config-file ` or `--config-file=` - Relative paths are resolved from `process.cwd()` diff --git a/docs/articles/introduction-to-katt.md b/docs/articles/introduction-to-katt.md index 801b98f..ae1b4b1 100644 --- a/docs/articles/introduction-to-katt.md +++ b/docs/articles/introduction-to-katt.md @@ -237,6 +237,7 @@ Katt runs as a Node.js CLI and executes prompts through a selectable runtime: - `gh-copilot` (default): GitHub Copilot via `@github/copilot-sdk` - `codex`: Codex via `codex exec` +- `claude-code`: Claude Code via `claude -p` In practice that means your machine/CI runner should be configured for the runtime you choose in `katt.json`. @@ -254,7 +255,7 @@ Katt’s core idea—**agentic workflows deserve tests**—is broader than any s Future directions that fit naturally with Katt’s model include: -- **Additional runtimes/adapters**: first-class integrations for other agentic tools (for example Claude Code and others), so the same eval files can validate workflows across environments. +- **Additional runtimes/adapters**: first-class integrations for more agentic tools, so the same eval files can validate workflows across environments. - **Better eval ergonomics**: more helpers to standardize prompts, assertions, and repeatable fixtures. These are intentionally framed as *roadmap ideas*: check the repo docs/specs for what’s supported today. diff --git a/katt-claude.json b/katt-claude.json new file mode 100644 index 0000000..a7e1839 --- /dev/null +++ b/katt-claude.json @@ -0,0 +1,4 @@ +{ + "agent": "claude-code", + "ignorePatterns": ["./examples/**"] +} diff --git a/package.json b/package.json index 2542f46..6283c65 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,8 @@ "test": "vitest", "typecheck": "tsc -p tsconfig.json --noEmit", "test:build": "node ./dist/katt.js", - "test:build:codex": "node ./dist/katt.js --config-file=./katt-codex.json" + "test:build:codex": "node ./dist/katt.js --config-file=./katt-codex.json", + "test:build:claude": "node ./dist/katt.js --config-file=./katt-claude.json" }, "types": "dist/index.d.ts", "devDependencies": { diff --git a/specs/configuration.spec.md b/specs/configuration.spec.md index 6d95f80..3d7800e 100644 --- a/specs/configuration.spec.md +++ b/specs/configuration.spec.md @@ -43,7 +43,7 @@ Top-level JSON object with optional `agent`, `agentOptions`, `prompt`, and Supported keys: - `agent?: string` - - Supported values: `"gh-copilot"`, `"codex"` + - Supported values: `"gh-copilot"`, `"codex"`, `"claude-code"` - Missing/unsupported values default runtime selection to `"gh-copilot"` - `agentOptions?: object` - `agentOptions.model?: string` @@ -53,6 +53,11 @@ Supported keys: - `profile`, `sandbox`, `fullAuto`, `skipGitRepoCheck`, `dangerouslyBypassApprovalsAndSandbox`, `config`, `workingDirectory` - unsupported keys are ignored by the Codex runner + - `claude-code`: supported keys are translated to `claude -p` flags: + - `permissionMode`, `dangerouslySkipPermissions`, `maxTurns`, + `allowedTools`, `disallowedTools`, `appendSystemPrompt`, `mcpConfig`, + `workingDirectory` + - unsupported keys are ignored by the Claude Code runner - `prompt?: object` - `prompt.timeoutMs?: number` (positive values only) - `ignorePatterns?: string[]` @@ -78,7 +83,7 @@ value. - `promptTimeoutMs: undefined` 4. Parses JSON content. 5. Resolves `agent`: - - Uses `agent` when it is `"gh-copilot"` or `"codex"` + - Uses `agent` when it is `"gh-copilot"`, `"codex"`, or `"claude-code"` - Otherwise falls back to `"gh-copilot"` 6. Returns `agentOptions` only when: - `agent` in file is a supported value, and @@ -92,7 +97,7 @@ value. 1. Reads defaults via `getDefaultKattConfig()`. 2. Returns `agentOptions` only when selected `agent` is `"gh-copilot"`. -3. Returns `undefined` when selected `agent` is `"codex"`. +3. Returns `undefined` when selected `agent` is `"codex"` or `"claude-code"`. `getDefaultPromptTimeoutMs()`: @@ -148,7 +153,7 @@ value. For `prompt(input, options?)` and `promptFile(filePath, options?)`: 1. Resolve active runtime from config `agent`: - - `"gh-copilot"` or `"codex"` + - `"gh-copilot"`, `"codex"`, or `"claude-code"` - fallback `"gh-copilot"` when missing/unsupported 2. Start from config `agentOptions` when available for the selected runtime. @@ -158,6 +163,7 @@ For `prompt(input, options?)` and `promptFile(filePath, options?)`: 5. Execute with selected runtime: - `gh-copilot`: create Copilot session with merged options - `codex`: run `codex exec` with mapped supported options + - `claude-code`: run `claude -p` with mapped supported options 6. Resolve prompt timeout with precedence: - `options.timeoutMs` (valid positive number) - config `prompt.timeoutMs` (valid positive number) diff --git a/specs/feature-prompt.spec.md b/specs/feature-prompt.spec.md index 2f245c0..b68bb3d 100644 --- a/specs/feature-prompt.spec.md +++ b/specs/feature-prompt.spec.md @@ -38,7 +38,7 @@ Execution/discovery flow for eval files and CLI exit handling is specified in - default: `/katt.json` - CLI override: `--config-file ` 2. Resolves active runtime: - - `gh-copilot` or `codex` + - `gh-copilot`, `codex`, or `claude-code` - defaults to `gh-copilot` when `agent` is missing/unsupported. 3. Resolves runtime options: - merges config `agentOptions` (base) with explicit `options` @@ -66,6 +66,14 @@ Execution/discovery flow for eval files and CLI exit handling is specified in - `Error("Codex did not return a response.")` when no output is available - timeout/process errors when Codex fails to execute successfully. + - `claude-code`: + 1. runs `claude -p` non-interactively with `--output-format json` + 2. passes supported options as command flags + 3. sends prompt through stdin + 4. parses JSON response payload and extracts final result text + 5. returns response text + 6. throws for invalid/missing JSON output, missing result text, or + timeout/process failures. ### Cleanup guarantees for `prompt()` @@ -77,6 +85,7 @@ Execution/discovery flow for eval files and CLI exit handling is specified in - `Copilot cleanup encountered error(s).` - `codex` runtime always removes temporary output artifacts used for response extraction. +- `claude-code` runtime has no temporary output artifact cleanup step. Cleanup logging does not replace the primary operation result/error. @@ -95,6 +104,8 @@ Cleanup logging does not replace the primary operation result/error. - Copilot/session errors in `prompt()` are propagated to the caller. - Codex process startup/exit/timeout errors in `prompt()` are propagated to the caller. +- Claude Code process startup/exit/timeout errors in `prompt()` are propagated + to the caller. - Missing response content in `prompt()` is converted to the explicit runtime errors defined above. diff --git a/src/lib/config/config.test.ts b/src/lib/config/config.test.ts index 0562e01..0caee04 100644 --- a/src/lib/config/config.test.ts +++ b/src/lib/config/config.test.ts @@ -162,6 +162,19 @@ describe("getDefaultCopilotConfig", () => { expect(result).toBeUndefined(); }); + it("returns undefined when agent is claude-code", async () => { + readFileMock.mockResolvedValue( + JSON.stringify({ + agent: "claude-code", + agentOptions: { model: "claude-sonnet-4" }, + }), + ); + + const result = await getDefaultCopilotConfig(); + + expect(result).toBeUndefined(); + }); + it("returns undefined when agentOptions is not an object", async () => { readFileMock.mockResolvedValue( JSON.stringify({ @@ -303,6 +316,30 @@ describe("getDefaultKattConfig", () => { }); }); + it("returns claude-code defaults when configured", async () => { + readFileMock.mockResolvedValue( + JSON.stringify({ + agent: "claude-code", + agentOptions: { + model: "claude-sonnet-4", + permissionMode: "acceptEdits", + }, + prompt: { timeoutMs: 450000 }, + }), + ); + + const result = await getDefaultKattConfig(); + + expect(result).toEqual({ + agent: "claude-code", + agentOptions: { + model: "claude-sonnet-4", + permissionMode: "acceptEdits", + }, + promptTimeoutMs: 450000, + }); + }); + it("reads custom config path when configured", async () => { setKattConfigFilePath("./config/custom-katt.json"); readFileMock.mockResolvedValue( diff --git a/src/lib/config/config.ts b/src/lib/config/config.ts index 18194cd..e61764a 100644 --- a/src/lib/config/config.ts +++ b/src/lib/config/config.ts @@ -2,7 +2,7 @@ import { readFile } from "node:fs/promises"; import { dirname, resolve } from "node:path"; import type { SessionConfig } from "@github/copilot-sdk"; -export type KattAgent = "gh-copilot" | "codex"; +export type KattAgent = "gh-copilot" | "codex" | "claude-code"; type KattConfig = { agent?: unknown; @@ -112,7 +112,7 @@ async function readKattConfig(): Promise { } function readSupportedAgent(agent: unknown): KattAgent | undefined { - if (agent === "gh-copilot" || agent === "codex") { + if (agent === "gh-copilot" || agent === "codex" || agent === "claude-code") { return agent; } diff --git a/src/lib/prompt/claudeCode.test.ts b/src/lib/prompt/claudeCode.test.ts new file mode 100644 index 0000000..f13bca8 --- /dev/null +++ b/src/lib/prompt/claudeCode.test.ts @@ -0,0 +1,376 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { runClaudeCodePrompt } from "./claudeCode.js"; +import type { EventEmitter } from "node:events"; + +const spawnMock = vi.fn(); + +vi.mock("node:child_process", () => ({ + spawn: (...args: unknown[]) => spawnMock(...args), +})); + +type MockChildProcess = { + stdout: EventEmitter & { setEncoding: (encoding: string) => void }; + stderr: EventEmitter & { setEncoding: (encoding: string) => void }; + stdin: EventEmitter & { + end: (data: string) => void; + on: (event: string, handler: () => void) => void; + }; + once: (event: string, handler: (...args: unknown[]) => void) => void; + kill: (signal: string) => void; +}; + +function createMockChildProcess(): MockChildProcess { + const stdout = Object.assign(new EventTarget() as unknown as EventEmitter, { + setEncoding: vi.fn(), + on: vi.fn(), + }); + const stderr = Object.assign(new EventTarget() as unknown as EventEmitter, { + setEncoding: vi.fn(), + on: vi.fn(), + }); + const stdin = Object.assign(new EventTarget() as unknown as EventEmitter, { + end: vi.fn(), + on: vi.fn(), + }); + + const eventHandlers: Record void)[]> = {}; + + return { + stdout, + stderr, + stdin, + once: (event: string, handler: (...args: unknown[]) => void) => { + if (!eventHandlers[event]) { + eventHandlers[event] = []; + } + eventHandlers[event].push(handler); + }, + kill: vi.fn(), + triggerEvent: (event: string, ...args: unknown[]) => { + if (eventHandlers[event]) { + for (const handler of eventHandlers[event]) { + handler(...args); + } + } + }, + } as unknown as MockChildProcess; +} + +function setupSuccessfulClaude(stdout: string, stderr = "", exitCode = 0) { + const mockChild = createMockChildProcess(); + spawnMock.mockReturnValue(mockChild); + + setTimeout(() => { + if (stdout) { + (mockChild.stdout.on as ReturnType).mock.calls[0]?.[1]?.( + stdout, + ); + } + if (stderr) { + (mockChild.stderr.on as ReturnType).mock.calls[0]?.[1]?.( + stderr, + ); + } + ( + mockChild as unknown as { + triggerEvent: (event: string, ...args: unknown[]) => void; + } + ).triggerEvent("close", exitCode, null); + }, 0); + + return mockChild; +} + +describe("runClaudeCodePrompt", () => { + afterEach(() => { + spawnMock.mockReset(); + vi.restoreAllMocks(); + }); + + it("spawns claude with default arguments", async () => { + setupSuccessfulClaude(JSON.stringify({ type: "result", result: "ok" })); + + await runClaudeCodePrompt("test prompt", 30000, undefined); + + expect(spawnMock).toHaveBeenCalledWith( + "claude", + ["-p", "--output-format", "json"], + { + cwd: process.cwd(), + stdio: ["pipe", "pipe", "pipe"], + }, + ); + }); + + it("sends prompt to stdin", async () => { + const mockChild = setupSuccessfulClaude( + JSON.stringify({ type: "result", result: "ok" }), + ); + + await runClaudeCodePrompt("test prompt", 30000, undefined); + + expect(mockChild.stdin.end).toHaveBeenCalledWith("test prompt"); + }); + + it("returns response from successful JSON output", async () => { + setupSuccessfulClaude(JSON.stringify({ type: "result", result: "answer" })); + + const result = await runClaudeCodePrompt("test prompt", 30000, undefined); + + expect(result).toBe("answer"); + }); + + it("returns response from successful JSON output array", async () => { + setupSuccessfulClaude( + JSON.stringify([ + { type: "result", result: "intermediate", is_error: true }, + { type: "result", result: "final answer" }, + ]), + ); + + const result = await runClaudeCodePrompt("test prompt", 30000, undefined); + + expect(result).toBe("final answer"); + }); + + it("passes model option to claude", async () => { + setupSuccessfulClaude(JSON.stringify({ type: "result", result: "ok" })); + + await runClaudeCodePrompt("test prompt", 30000, { model: "sonnet-4" }); + + expect(spawnMock).toHaveBeenCalledWith( + "claude", + expect.arrayContaining(["--model", "sonnet-4"]), + expect.anything(), + ); + }); + + it("passes permissionMode option to claude", async () => { + setupSuccessfulClaude(JSON.stringify({ type: "result", result: "ok" })); + + await runClaudeCodePrompt("test prompt", 30000, { + permissionMode: "acceptEdits", + }); + + expect(spawnMock).toHaveBeenCalledWith( + "claude", + expect.arrayContaining(["--permission-mode", "acceptEdits"]), + expect.anything(), + ); + }); + + it("passes dangerouslySkipPermissions flag when true", async () => { + setupSuccessfulClaude(JSON.stringify({ type: "result", result: "ok" })); + + await runClaudeCodePrompt("test prompt", 30000, { + dangerouslySkipPermissions: true, + }); + + expect(spawnMock).toHaveBeenCalledWith( + "claude", + expect.arrayContaining(["--dangerously-skip-permissions"]), + expect.anything(), + ); + }); + + it("passes normalized maxTurns option", async () => { + setupSuccessfulClaude(JSON.stringify({ type: "result", result: "ok" })); + + await runClaudeCodePrompt("test prompt", 30000, { maxTurns: 3.8 }); + + expect(spawnMock).toHaveBeenCalledWith( + "claude", + expect.arrayContaining(["--max-turns", "3"]), + expect.anything(), + ); + }); + + it("ignores invalid maxTurns values", async () => { + setupSuccessfulClaude(JSON.stringify({ type: "result", result: "ok" })); + + await runClaudeCodePrompt("test prompt", 30000, { + maxTurns: -1, + }); + + const args = spawnMock.mock.calls[0][1] as string[]; + expect(args).not.toContain("--max-turns"); + }); + + it("ignores maxTurns values that floor to zero", async () => { + setupSuccessfulClaude(JSON.stringify({ type: "result", result: "ok" })); + + await runClaudeCodePrompt("test prompt", 30000, { + maxTurns: 0.5, + }); + + const args = spawnMock.mock.calls[0][1] as string[]; + expect(args).not.toContain("--max-turns"); + }); + + it("passes allowedTools string option", async () => { + setupSuccessfulClaude(JSON.stringify({ type: "result", result: "ok" })); + + await runClaudeCodePrompt("test prompt", 30000, { + allowedTools: "Edit,Bash", + }); + + expect(spawnMock).toHaveBeenCalledWith( + "claude", + expect.arrayContaining(["--allowedTools", "Edit,Bash"]), + expect.anything(), + ); + }); + + it("passes normalized allowedTools array option", async () => { + setupSuccessfulClaude(JSON.stringify({ type: "result", result: "ok" })); + + await runClaudeCodePrompt("test prompt", 30000, { + allowedTools: ["Edit", "", "Bash"], + }); + + const args = spawnMock.mock.calls[0][1] as string[]; + expect(args).toEqual( + expect.arrayContaining(["--allowedTools", "Edit", "Bash"]), + ); + }); + + it("passes normalized disallowedTools array option", async () => { + setupSuccessfulClaude(JSON.stringify({ type: "result", result: "ok" })); + + await runClaudeCodePrompt("test prompt", 30000, { + disallowedTools: ["WebFetch", "WebSearch"], + }); + + const args = spawnMock.mock.calls[0][1] as string[]; + expect(args).toEqual( + expect.arrayContaining(["--disallowedTools", "WebFetch", "WebSearch"]), + ); + }); + + it("passes appendSystemPrompt option to claude", async () => { + setupSuccessfulClaude(JSON.stringify({ type: "result", result: "ok" })); + + await runClaudeCodePrompt("test prompt", 30000, { + appendSystemPrompt: "Always be concise.", + }); + + expect(spawnMock).toHaveBeenCalledWith( + "claude", + expect.arrayContaining(["--append-system-prompt", "Always be concise."]), + expect.anything(), + ); + }); + + it("passes mcpConfig option to claude", async () => { + setupSuccessfulClaude(JSON.stringify({ type: "result", result: "ok" })); + + await runClaudeCodePrompt("test prompt", 30000, { + mcpConfig: "./mcp.json", + }); + + expect(spawnMock).toHaveBeenCalledWith( + "claude", + expect.arrayContaining(["--mcp-config", "./mcp.json"]), + expect.anything(), + ); + }); + + it("uses custom working directory when provided", async () => { + setupSuccessfulClaude(JSON.stringify({ type: "result", result: "ok" })); + + await runClaudeCodePrompt("test prompt", 30000, { + workingDirectory: "/custom/path", + }); + + expect(spawnMock).toHaveBeenCalledWith( + "claude", + expect.anything(), + expect.objectContaining({ cwd: "/custom/path" }), + ); + }); + + it("throws error when claude fails to start", async () => { + const mockChild = createMockChildProcess(); + spawnMock.mockReturnValue(mockChild); + + setTimeout(() => { + ( + mockChild as unknown as { + triggerEvent: (event: string, ...args: unknown[]) => void; + } + ).triggerEvent("error", new Error("ENOENT: claude not found")); + }, 0); + + await expect( + runClaudeCodePrompt("test prompt", 30000, undefined), + ).rejects.toThrow(/Failed to start Claude Code CLI/); + }); + + it("throws timeout error when process exceeds timeout", async () => { + const mockChild = createMockChildProcess(); + spawnMock.mockReturnValue(mockChild); + + setTimeout(() => { + ( + mockChild as unknown as { + triggerEvent: (event: string, ...args: unknown[]) => void; + } + ).triggerEvent("close", null, "SIGTERM"); + }, 50); + + const promise = runClaudeCodePrompt("test prompt", 10, undefined); + await new Promise((resolve) => setTimeout(resolve, 20)); + + await expect(promise).rejects.toThrow("Claude Code timed out after 10ms."); + }); + + it("throws error with exit code when claude exits with non-zero code", async () => { + setupSuccessfulClaude("", "error message", 1); + + await expect( + runClaudeCodePrompt("test prompt", 30000, undefined), + ).rejects.toThrow("Claude Code exited with code 1. error message"); + }); + + it("throws error when output is empty", async () => { + setupSuccessfulClaude(""); + + await expect( + runClaudeCodePrompt("test prompt", 30000, undefined), + ).rejects.toThrow("Claude Code did not return a response."); + }); + + it("throws error when output is not valid JSON", async () => { + setupSuccessfulClaude("not-json"); + + await expect( + runClaudeCodePrompt("test prompt", 30000, undefined), + ).rejects.toThrow("Claude Code returned invalid JSON output."); + }); + + it("throws error when JSON marks result as error", async () => { + setupSuccessfulClaude( + JSON.stringify({ + type: "result", + is_error: true, + result: "Permission denied", + }), + ); + + await expect( + runClaudeCodePrompt("test prompt", 30000, undefined), + ).rejects.toThrow( + "Claude Code returned an error response. Permission denied", + ); + }); + + it("throws error when JSON has no final response", async () => { + setupSuccessfulClaude(JSON.stringify({ type: "result" })); + + await expect( + runClaudeCodePrompt("test prompt", 30000, undefined), + ).rejects.toThrow( + "Claude Code JSON output did not include a final response.", + ); + }); +}); diff --git a/src/lib/prompt/claudeCode.ts b/src/lib/prompt/claudeCode.ts new file mode 100644 index 0000000..7ded21e --- /dev/null +++ b/src/lib/prompt/claudeCode.ts @@ -0,0 +1,269 @@ +import { spawn } from "node:child_process"; + +type ClaudeCodeRunOptions = { + model?: string; + permissionMode?: string; + dangerouslySkipPermissions?: boolean; + maxTurns?: number; + allowedTools?: string | string[]; + disallowedTools?: string | string[]; + appendSystemPrompt?: string; + mcpConfig?: string; + workingDirectory?: string; +}; + +type CommandResult = { + exitCode: number | null; + signal: NodeJS.Signals | null; + stdout: string; + stderr: string; + timedOut: boolean; +}; + +function isNonEmptyString(value: unknown): value is string { + return typeof value === "string" && value.length > 0; +} + +function readToolList(value: unknown): string[] { + if (isNonEmptyString(value)) { + return [value]; + } + + if (!Array.isArray(value)) { + return []; + } + + return value.filter(isNonEmptyString); +} + +function readMaxTurns(value: unknown): number | undefined { + if (typeof value !== "number" || !Number.isFinite(value) || value <= 0) { + return undefined; + } + + const maxTurns = Math.floor(value); + return maxTurns > 0 ? maxTurns : undefined; +} + +function buildClaudeCodeArgs( + options: Record | undefined, +): string[] { + const claudeOptions = (options ?? {}) as ClaudeCodeRunOptions; + const args = ["-p", "--output-format", "json"]; + + if (isNonEmptyString(claudeOptions.model)) { + args.push("--model", claudeOptions.model); + } + + if (isNonEmptyString(claudeOptions.permissionMode)) { + args.push("--permission-mode", claudeOptions.permissionMode); + } + + if (claudeOptions.dangerouslySkipPermissions === true) { + args.push("--dangerously-skip-permissions"); + } + + const maxTurns = readMaxTurns(claudeOptions.maxTurns); + if (maxTurns !== undefined) { + args.push("--max-turns", String(maxTurns)); + } + + const allowedTools = readToolList(claudeOptions.allowedTools); + if (allowedTools.length > 0) { + args.push("--allowedTools", ...allowedTools); + } + + const disallowedTools = readToolList(claudeOptions.disallowedTools); + if (disallowedTools.length > 0) { + args.push("--disallowedTools", ...disallowedTools); + } + + if (isNonEmptyString(claudeOptions.appendSystemPrompt)) { + args.push("--append-system-prompt", claudeOptions.appendSystemPrompt); + } + + if (isNonEmptyString(claudeOptions.mcpConfig)) { + args.push("--mcp-config", claudeOptions.mcpConfig); + } + + return args; +} + +function runClaudeCodeCommand( + input: string, + args: string[], + timeoutMs: number, + workingDirectory: string, +): Promise { + return new Promise((resolve, reject) => { + const child = spawn("claude", args, { + cwd: workingDirectory, + stdio: ["pipe", "pipe", "pipe"], + }); + + let stdout = ""; + let stderr = ""; + let timedOut = false; + + child.stdout.setEncoding("utf8"); + child.stderr.setEncoding("utf8"); + child.stdout.on("data", (chunk) => { + stdout += chunk; + }); + child.stderr.on("data", (chunk) => { + stderr += chunk; + }); + child.stdin.on("error", () => {}); + + const timeoutHandle = setTimeout(() => { + timedOut = true; + child.kill("SIGTERM"); + }, timeoutMs); + + child.once("error", (error) => { + clearTimeout(timeoutHandle); + reject( + new Error( + `Failed to start Claude Code CLI. Ensure claude is installed and available on PATH. ${String( + error, + )}`, + ), + ); + }); + + child.once("close", (exitCode, signal) => { + clearTimeout(timeoutHandle); + resolve({ + exitCode, + signal, + stdout: stdout.trim(), + stderr: stderr.trim(), + timedOut, + }); + }); + + child.stdin.end(input); + }); +} + +function buildProcessFailureMessage(result: CommandResult): string { + if (result.timedOut) { + return "Claude Code timed out before returning a response."; + } + + if (result.exitCode === null) { + const signal = result.signal ?? "unknown"; + return `Claude Code exited due to signal ${signal}.`; + } + + const stderrDetails = result.stderr.length > 0 ? ` ${result.stderr}` : ""; + return `Claude Code exited with code ${result.exitCode}.${stderrDetails}`; +} + +function readErrorMessage(value: unknown): string | undefined { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + return undefined; + } + + const data = value as Record; + if (typeof data.error === "string" && data.error.length > 0) { + return data.error; + } + if (typeof data.message === "string" && data.message.length > 0) { + return data.message; + } + if (typeof data.result === "string" && data.result.length > 0) { + return data.result; + } + return undefined; +} + +function readResult(value: unknown): string | undefined { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + return undefined; + } + + const data = value as Record; + if (data.is_error === true || data.subtype === "error") { + return undefined; + } + + return typeof data.result === "string" && data.result.length > 0 + ? data.result + : undefined; +} + +function parseClaudeCodeOutput(output: string): string { + if (output.length === 0) { + throw new Error("Claude Code did not return a response."); + } + + let parsed: unknown; + try { + parsed = JSON.parse(output); + } catch (error) { + throw new Error( + `Claude Code returned invalid JSON output. ${String(error)}`, + ); + } + + if (Array.isArray(parsed)) { + for (let index = parsed.length - 1; index >= 0; index -= 1) { + const response = readResult(parsed[index]); + if (response) { + return response; + } + } + + const errorMessage = readErrorMessage(parsed[parsed.length - 1]); + if (errorMessage) { + throw new Error( + `Claude Code returned an error response. ${errorMessage}`, + ); + } + + throw new Error( + "Claude Code JSON output did not include a final response.", + ); + } + + const response = readResult(parsed); + if (response) { + return response; + } + + const errorMessage = readErrorMessage(parsed); + if (errorMessage) { + throw new Error(`Claude Code returned an error response. ${errorMessage}`); + } + + throw new Error("Claude Code JSON output did not include a final response."); +} + +export async function runClaudeCodePrompt( + input: string, + timeoutMs: number, + options: Record | undefined, +): Promise { + const claudeOptions = (options ?? {}) as ClaudeCodeRunOptions; + const workingDirectory = isNonEmptyString(claudeOptions.workingDirectory) + ? claudeOptions.workingDirectory + : process.cwd(); + const args = buildClaudeCodeArgs(options); + const result = await runClaudeCodeCommand( + input, + args, + timeoutMs, + workingDirectory, + ); + + if (result.timedOut) { + throw new Error(`Claude Code timed out after ${timeoutMs}ms.`); + } + + if (result.exitCode !== 0) { + throw new Error(buildProcessFailureMessage(result)); + } + + return parseClaudeCodeOutput(result.stdout); +} diff --git a/src/lib/prompt/prompt.test.ts b/src/lib/prompt/prompt.test.ts index adc9c62..c9d6a2c 100644 --- a/src/lib/prompt/prompt.test.ts +++ b/src/lib/prompt/prompt.test.ts @@ -8,6 +8,7 @@ const addUsedTokensToCurrentTestMock = vi.fn(); const setCurrentTestModelMock = vi.fn(); const getDefaultKattConfigMock = vi.fn(); const runCodexPromptMock = vi.fn(); +const runClaudeCodePromptMock = vi.fn(); vi.mock("node:fs/promises", () => ({ readFile: (...args: unknown[]) => readFileMock(...args), @@ -28,6 +29,10 @@ vi.mock("./codex.js", () => ({ runCodexPrompt: (...args: unknown[]) => runCodexPromptMock(...args), })); +vi.mock("./claudeCode.js", () => ({ + runClaudeCodePrompt: (...args: unknown[]) => runClaudeCodePromptMock(...args), +})); + let sendAndWaitMock: ReturnType< typeof vi.fn<() => Promise<{ data: { content: string } }>> >; @@ -71,6 +76,7 @@ function setupSessionMocks( promptTimeoutMs: undefined, }); runCodexPromptMock.mockResolvedValue("codex response"); + runClaudeCodePromptMock.mockResolvedValue("claude response"); sendAndWaitMock = vi.fn().mockResolvedValue({ data: { content: "ok" } }); destroyMock = destroyError ? vi.fn().mockRejectedValue(destroyError) @@ -118,6 +124,7 @@ describe("prompt", () => { setCurrentTestModelMock.mockReset(); getDefaultKattConfigMock.mockReset(); runCodexPromptMock.mockReset(); + runClaudeCodePromptMock.mockReset(); vi.restoreAllMocks(); }); @@ -351,6 +358,54 @@ describe("prompt", () => { ); expect(setCurrentTestModelMock).toHaveBeenCalledWith("gpt-5.2-codex"); }); + + it("uses Claude Code runtime when configured", async () => { + setupSessionMocks(); + getDefaultKattConfigMock.mockResolvedValue({ + agent: "claude-code", + agentOptions: { model: "claude-sonnet-4", permissionMode: "acceptEdits" }, + promptTimeoutMs: 450000, + }); + runClaudeCodePromptMock.mockResolvedValue("claude answer"); + + const result = await prompt("Hello"); + + expect(result).toBe("claude answer"); + expect(runClaudeCodePromptMock).toHaveBeenCalledWith("Hello", 450000, { + model: "claude-sonnet-4", + permissionMode: "acceptEdits", + }); + expect(createSessionMock).not.toHaveBeenCalled(); + }); + + it("lets explicit options override Claude Code defaults", async () => { + setupSessionMocks(); + getDefaultKattConfigMock.mockResolvedValue({ + agent: "claude-code", + agentOptions: { + model: "claude-sonnet-4", + permissionMode: "acceptEdits", + }, + promptTimeoutMs: undefined, + }); + + await prompt("Hello", { + model: "claude-opus-4", + permissionMode: "plan", + maxTurns: 5, + }); + + expect(runClaudeCodePromptMock).toHaveBeenCalledWith( + "Hello", + DEFAULT_PROMPT_TIMEOUT_MS, + { + model: "claude-opus-4", + permissionMode: "plan", + maxTurns: 5, + }, + ); + expect(setCurrentTestModelMock).toHaveBeenCalledWith("claude-opus-4"); + }); }); describe("promptFile", () => { @@ -360,6 +415,7 @@ describe("promptFile", () => { setCurrentTestModelMock.mockReset(); getDefaultKattConfigMock.mockReset(); runCodexPromptMock.mockReset(); + runClaudeCodePromptMock.mockReset(); vi.restoreAllMocks(); }); diff --git a/src/lib/prompt/prompt.ts b/src/lib/prompt/prompt.ts index 07a3be9..c8906ec 100644 --- a/src/lib/prompt/prompt.ts +++ b/src/lib/prompt/prompt.ts @@ -12,13 +12,27 @@ import { } from "../context/context.js"; import { evalFileStorage } from "../context/evalFileContext.js"; import { getDefaultKattConfig } from "../config/config.js"; +import { runClaudeCodePrompt } from "./claudeCode.js"; import { runCodexPrompt } from "./codex.js"; export const DEFAULT_PROMPT_TIMEOUT_MS = 600_000; -export type PromptOptions = Omit & { - onPermissionRequest?: PermissionHandler; -} & Record & { +export type ClaudeCodePromptOptions = { + model?: string; + permissionMode?: string; + dangerouslySkipPermissions?: boolean; + maxTurns?: number; + allowedTools?: string | string[]; + disallowedTools?: string | string[]; + appendSystemPrompt?: string; + mcpConfig?: string; + workingDirectory?: string; +}; + +export type PromptOptions = Omit & + ClaudeCodePromptOptions & { + onPermissionRequest?: PermissionHandler; + } & Record & { timeoutMs?: number; }; @@ -117,6 +131,20 @@ export async function prompt( return response; } + if (defaults.agent === "claude-code") { + const response = await runClaudeCodePrompt( + input, + timeoutMs, + sessionOptions, + ); + + if (model) { + setCurrentTestModel(model); + } + + return response; + } + const client = new CopilotClient({ useLoggedInUser: true }); let session: CopilotSession | undefined; let unsubscribeUsage: (() => void) | undefined;