From 573ea63a454e5406a5ccae3a66c5e14213142a4a Mon Sep 17 00:00:00 2001 From: tawnymanticore Date: Tue, 8 Sep 2026 11:40:21 -0400 Subject: [PATCH 1/6] Add openai_responses_api provider flag and route via litellm responses bridge OpenAI rejects function tools alongside reasoning_effort on /v1/chat/completions for its newer reasoning models, and only /v1/responses accepts both. The new flag makes the task adapter emit `openai/responses/` so litellm bridges the call, allow-lists reasoning_effort so drop_params can't silently strip it, and drops top_p when an effort is requested. Both of the latter cover models missing from litellm's gpt-5.x family check (eg gpt-6-astra), which would otherwise lose the thinking level or 400. Shared utils/litellm.py is untouched: the extractor/embedding/reranker adapters have no responses path. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_014jrCq27Qa4VQeVEvYjSfTs --- libs/core/kiln_ai/adapters/ml_model_list.py | 13 ++ .../model_adapters/litellm_adapter.py | 34 ++++- .../model_adapters/test_litellm_adapter.py | 137 +++++++++++++++++- .../kiln_ai/adapters/test_ml_model_list.py | 39 +++++ libs/core/kiln_ai/utils/test_litellm.py | 16 ++ 5 files changed, 229 insertions(+), 10 deletions(-) diff --git a/libs/core/kiln_ai/adapters/ml_model_list.py b/libs/core/kiln_ai/adapters/ml_model_list.py index f00d6f6fc8..b281085dee 100644 --- a/libs/core/kiln_ai/adapters/ml_model_list.py +++ b/libs/core/kiln_ai/adapters/ml_model_list.py @@ -391,6 +391,11 @@ class KilnModelProvider(BaseModel): # When true, send `reasoning: {effort: }` instead of `reasoning_effort`. # Use only for OpenRouter models that require the reasoning-object format. openrouter_reasoning_object: bool = False + + # OpenAI-specific endpoint toggle. When true, route this provider's calls to + # OpenAI's /v1/responses endpoint via litellm's `openai/responses/` bridge. + # Required for reasoning models that reject tools on /v1/chat/completions. + openai_responses_api: bool = False available_thinking_levels: dict[str, str] | None = None default_thinking_level: str | None = None ollama_model_aliases: List[str] | None = None @@ -433,6 +438,14 @@ def validate_openrouter_reasoning_object(self) -> "KilnModelProvider": ) return self + @model_validator(mode="after") + def validate_openai_responses_api(self) -> "KilnModelProvider": + if self.openai_responses_api and self.name != ModelProviderName.openai: + raise ValueError( + "openai_responses_api can only be true when provider is openai" + ) + return self + @model_validator(mode="after") def validate_default_thinking_level(self) -> "KilnModelProvider": if self.available_thinking_levels: diff --git a/libs/core/kiln_ai/adapters/model_adapters/litellm_adapter.py b/libs/core/kiln_ai/adapters/model_adapters/litellm_adapter.py index 9ae67e4302..0df4679e04 100644 --- a/libs/core/kiln_ai/adapters/model_adapters/litellm_adapter.py +++ b/libs/core/kiln_ai/adapters/model_adapters/litellm_adapter.py @@ -652,17 +652,27 @@ def litellm_model_id(self) -> str: if self._litellm_model_id: return self._litellm_model_id - litellm_provider_info = get_litellm_provider_info(self.model_provider()) + provider = self.model_provider() + litellm_provider_info = get_litellm_provider_info(provider) if litellm_provider_info.is_custom and self._api_base is None: raise ValueError( "Explicit Base URL is required for OpenAI compatible APIs (custom models, ollama, fine tunes, and custom registry models)" ) - self._litellm_model_id = litellm_provider_info.litellm_model_id + litellm_model_id = litellm_provider_info.litellm_model_id + if provider.openai_responses_api: + # litellm bridges `openai/responses/` to OpenAI's /v1/responses + # endpoint: the only one that accepts tools alongside reasoning effort + # for these models. + litellm_model_id = ( + f"{litellm_provider_info.provider_name}/responses/{provider.model_id}" + ) + + self._litellm_model_id = litellm_model_id return self._litellm_model_id def _allowed_openai_params_for_completion_kwargs( - self, completion_kwargs: dict[str, Any] + self, completion_kwargs: dict[str, Any], provider: KilnModelProvider ) -> list[str]: """ LiteLLM drops params it thinks are not supported by the model when drop_params is True. Sometimes it is wrong @@ -693,6 +703,11 @@ def _allowed_openai_params_for_completion_kwargs( automatic_allowed_params.append("tools") if "tool_choice" in completion_kwargs: automatic_allowed_params.append("tool_choice") + if provider.openai_responses_api and "reasoning_effort" in completion_kwargs: + # litellm's param map doesn't list reasoning_effort for every model we + # route to /v1/responses (eg gpt-6-astra), and drop_params would silently + # strip it, making the thinking level a no-op. + automatic_allowed_params.append("reasoning_effort") return list(set(explicit_allowed_params_validated + automatic_allowed_params)) @@ -737,6 +752,17 @@ async def build_completion_kwargs( completion_kwargs["tools"] = tool_calls completion_kwargs["tool_choice"] = "auto" + # OpenAI's reasoning models reject top_p, and any temperature other than the + # default 1.0, once a reasoning effort is in play. litellm drops both for the + # gpt-5.x family, but only matches names containing "gpt-5", so models like + # gpt-6-astra reach /v1/responses verbatim and 400. Dropping temperature is + # equivalent to sending the only value these models accept. + if provider.openai_responses_api and completion_kwargs.get( + "reasoning_effort" + ) not in (None, "none"): + completion_kwargs.pop("top_p", None) + completion_kwargs.pop("temperature", None) + # Special condition for Claude Opus 4.1 and Sonnet 4.5, where we can only specify top_p or temp, not both. # Remove default values (1.0) prioritizing anything the user customized, then error with helpful message if they are both custom. if provider.temp_top_p_exclusive: @@ -771,7 +797,7 @@ async def build_completion_kwargs( # any params listed in this list will be passed to the model regardless of LiteLLM's own validation allowed_openai_params = self._allowed_openai_params_for_completion_kwargs( - completion_kwargs + completion_kwargs, provider ) if len(allowed_openai_params) > 0: completion_kwargs["allowed_openai_params"] = allowed_openai_params diff --git a/libs/core/kiln_ai/adapters/model_adapters/test_litellm_adapter.py b/libs/core/kiln_ai/adapters/model_adapters/test_litellm_adapter.py index c9e7945d47..d23d51c1f5 100644 --- a/libs/core/kiln_ai/adapters/model_adapters/test_litellm_adapter.py +++ b/libs/core/kiln_ai/adapters/model_adapters/test_litellm_adapter.py @@ -375,6 +375,7 @@ def test_litellm_model_id_standard_providers( mock_provider = Mock() mock_provider.name = provider_name mock_provider.model_id = "test-model" + mock_provider.openai_responses_api = False with patch.object(adapter, "model_provider", return_value=mock_provider): model_id = adapter.litellm_model_id() @@ -384,6 +385,34 @@ def test_litellm_model_id_standard_providers( assert adapter._litellm_model_id == model_id +@pytest.mark.parametrize( + "openai_responses_api,expected_model_id", + [ + (True, "openai/responses/test-model"), + (False, "openai/test-model"), + ], +) +def test_litellm_model_id_openai_responses_api( + config, mock_task, openai_responses_api, expected_model_id +): + """Providers with openai_responses_api use litellm's `openai/responses/` + bridge, which calls OpenAI's /v1/responses endpoint instead of chat completions.""" + adapter = LiteLlmAdapter(config=config, kiln_task=mock_task) + + provider = KilnModelProvider( + name=ModelProviderName.openai, + model_id="test-model", + openai_responses_api=openai_responses_api, + ) + + with patch.object(adapter, "model_provider", return_value=provider): + model_id = adapter.litellm_model_id() + + assert model_id == expected_model_id + # Verify caching works + assert adapter._litellm_model_id == model_id + + @pytest.mark.parametrize( "provider_name", [ @@ -401,6 +430,7 @@ def test_litellm_model_id_custom_providers(config, mock_task, provider_name): mock_provider = Mock() mock_provider.name = provider_name mock_provider.model_id = "custom-model" + mock_provider.openai_responses_api = False with patch.object(adapter, "model_provider", return_value=mock_provider): model_id = adapter.litellm_model_id() @@ -829,6 +859,7 @@ async def test_build_completion_kwargs( adapter = LiteLlmAdapter(config=config, kiln_task=mock_task) mock_provider = Mock() mock_provider.temp_top_p_exclusive = False + mock_provider.openai_responses_api = False messages = [{"role": "user", "content": "Hello"}] with ( @@ -1092,6 +1123,23 @@ async def test_litellm_tools_returns_empty_list_without_tools(config, mock_task) assert tools == [] +@pytest.fixture +def plain_provider(): + return KilnModelProvider( + name=ModelProviderName.openai, + model_id="test-model", + ) + + +@pytest.fixture +def responses_api_provider(): + return KilnModelProvider( + name=ModelProviderName.openai, + model_id="test-model", + openai_responses_api=True, + ) + + @pytest.mark.parametrize( "kwargs_in,expected", [ @@ -1104,29 +1152,58 @@ async def test_litellm_tools_returns_empty_list_without_tools(config, mock_task) {"tools": [], "allowed_openai_params": ["custom_param"]}, ["custom_param", "tools"], ), + # reasoning_effort is only allow-listed for providers routed to /v1/responses + ({"reasoning_effort": "high"}, []), ], ) def test_allowed_openai_params_for_completion_kwargs_independent_keys( - config, mock_task, kwargs_in, expected + config, mock_task, plain_provider, kwargs_in, expected +): + adapter = LiteLlmAdapter(config=config, kiln_task=mock_task) + result = adapter._allowed_openai_params_for_completion_kwargs( + kwargs_in, plain_provider + ) + assert sorted(result) == sorted(expected) + + +@pytest.mark.parametrize( + "kwargs_in,expected", + [ + ({}, []), + ({"reasoning_effort": "high"}, ["reasoning_effort"]), + ( + {"tools": [], "tool_choice": "auto", "reasoning_effort": "high"}, + ["tools", "tool_choice", "reasoning_effort"], + ), + ], +) +def test_allowed_openai_params_for_completion_kwargs_openai_responses_api( + config, mock_task, responses_api_provider, kwargs_in, expected ): + """litellm's param map is missing reasoning_effort for some responses-routed models + (eg gpt-6-astra), so drop_params would silently strip it without the allow-list.""" adapter = LiteLlmAdapter(config=config, kiln_task=mock_task) - result = adapter._allowed_openai_params_for_completion_kwargs(kwargs_in) + result = adapter._allowed_openai_params_for_completion_kwargs( + kwargs_in, responses_api_provider + ) assert sorted(result) == sorted(expected) -def test_allowed_openai_params_raises_for_non_list(config, mock_task): +def test_allowed_openai_params_raises_for_non_list(config, mock_task, plain_provider): adapter = LiteLlmAdapter(config=config, kiln_task=mock_task) with pytest.raises(ValueError, match="expected list"): adapter._allowed_openai_params_for_completion_kwargs( - {"allowed_openai_params": "not_a_list"} + {"allowed_openai_params": "not_a_list"}, plain_provider ) -def test_allowed_openai_params_raises_for_non_string_items(config, mock_task): +def test_allowed_openai_params_raises_for_non_string_items( + config, mock_task, plain_provider +): adapter = LiteLlmAdapter(config=config, kiln_task=mock_task) with pytest.raises(ValueError, match="items are not strings"): adapter._allowed_openai_params_for_completion_kwargs( - {"allowed_openai_params": ["valid", 123]} + {"allowed_openai_params": ["valid", 123]}, plain_provider ) @@ -1164,6 +1241,54 @@ async def test_build_completion_kwargs_includes_tools( assert "function" in tool +@pytest.mark.asyncio +@pytest.mark.parametrize( + "openai_responses_api,thinking_level,temperature,expect_sampling_params", + [ + (True, "high", 1.0, False), + (True, "high", 0.4, False), + # both are legal for these models when no thinking is requested + (True, "none", 0.4, True), + (False, "high", 0.4, True), + ], +) +async def test_build_completion_kwargs_drops_sampling_params_for_openai_responses_api( + config, + mock_task, + openai_responses_api, + thinking_level, + temperature, + expect_sampling_params, +): + """OpenAI's reasoning models reject top_p, and any temperature but the default, + once a reasoning effort is in play. litellm only strips them for models it + recognises as gpt-5.x, so /v1/responses returns a 400 for the others (eg + gpt-6-astra) unless we drop them ourselves.""" + config.run_config_properties.thinking_level = thinking_level + config.run_config_properties.temperature = temperature + adapter = LiteLlmAdapter(config=config, kiln_task=mock_task) + + provider = KilnModelProvider( + name=ModelProviderName.openai, + model_id="test-model", + openai_responses_api=openai_responses_api, + available_thinking_levels={"Off/None": "none", "High": "high"}, + default_thinking_level="none", + ) + messages = [{"role": "user", "content": "Hello"}] + + with ( + patch.object(adapter, "model_provider", return_value=provider), + patch.object(adapter, "litellm_model_id", return_value="openai/test-model"), + patch.object(adapter, "response_format_options", return_value={}), + patch.object(adapter, "available_tools", return_value=[]), + ): + kwargs = await adapter.build_completion_kwargs(provider, messages, None) + + assert ("top_p" in kwargs) is expect_sampling_params + assert ("temperature" in kwargs) is expect_sampling_params + + @pytest.mark.asyncio async def test_build_completion_kwargs_omits_allowed_openai_params_without_tools( config, mock_task diff --git a/libs/core/kiln_ai/adapters/test_ml_model_list.py b/libs/core/kiln_ai/adapters/test_ml_model_list.py index f06ba8b76c..8a1a340891 100644 --- a/libs/core/kiln_ai/adapters/test_ml_model_list.py +++ b/libs/core/kiln_ai/adapters/test_ml_model_list.py @@ -213,6 +213,45 @@ def test_openrouter_reasoning_object_requires_openrouter(self): ) +class TestOpenAIResponsesApi: + def test_openai_responses_api_allowed_on_openai(self): + provider = KilnModelProvider( + name=ModelProviderName.openai, + model_id="gpt-5.4", + openai_responses_api=True, + ) + assert provider.openai_responses_api is True + + @pytest.mark.parametrize( + "provider_name", + [ModelProviderName.openrouter, ModelProviderName.azure_openai], + ) + def test_openai_responses_api_requires_openai(self, provider_name): + with pytest.raises( + ValueError, + match="openai_responses_api can only be true when provider is openai", + ): + KilnModelProvider( + name=provider_name, + model_id="gpt-5.4", + openai_responses_api=True, + ) + + def test_built_in_models_with_flag_are_openai_and_support_tools(self): + """Routing to /v1/responses exists so these models can use tools. If one of them + ever ships with function calling disabled again, the workaround has returned.""" + for model in built_in_models: + for provider in model.providers: + if not provider.openai_responses_api: + continue + assert provider.name == ModelProviderName.openai, ( + f"{model.name} has openai_responses_api on {provider.name}" + ) + assert provider.supports_function_calling is True, ( + f"{model.name} routes to /v1/responses but disables function calling" + ) + + class TestBuiltInModelsFromProvider: """Test cases for built_in_models_from_provider function""" diff --git a/libs/core/kiln_ai/utils/test_litellm.py b/libs/core/kiln_ai/utils/test_litellm.py index f56655db04..18892f7847 100644 --- a/libs/core/kiln_ai/utils/test_litellm.py +++ b/libs/core/kiln_ai/utils/test_litellm.py @@ -301,3 +301,19 @@ def test_non_ollama_providers_dont_resolve_variants(self, mock_resolve_variant): # Verify the original model ID is used assert result.litellm_model_id == "openai/gpt-4" + + def test_openai_responses_api_flag_does_not_change_provider_info(self): + """This helper is shared with the extractor/embedding/reranker adapters, which + have no responses-API path. Only the task adapter rewrites the slug to + `openai/responses/`.""" + provider = KilnModelProvider( + name=ModelProviderName.openai, + model_id="gpt-5.4", + openai_responses_api=True, + ) + + result = get_litellm_provider_info(provider) + + assert result.provider_name == "openai" + assert result.is_custom is False + assert result.litellm_model_id == "openai/gpt-5.4" From 2b326d675b306350822c97898dd4cb6c5fd8b028 Mon Sep 17 00:00:00 2001 From: tawnymanticore Date: Tue, 8 Sep 2026 11:48:29 -0400 Subject: [PATCH 2/6] Enable function calling on GPT-5.4+ and GPT-6 Astra (OpenAI direct) These six OpenAI-direct providers shipped with supports_function_calling disabled because OpenAI rejects tools alongside reasoning_effort on /v1/chat/completions. Routing them through /v1/responses removes the need for that workaround, so drop it and set openai_responses_api instead. A paid regression test drives the real tool loop for every flagged provider and checks endpoint, effort on the wire, the tool result, and usage/cost, with gpt-5.2 as a chat-completions control. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_014jrCq27Qa4VQeVEvYjSfTs --- libs/core/kiln_ai/adapters/ml_model_list.py | 36 +- .../test_openai_responses_routing_paid.py | 463 ++++++++++++++++++ .../kiln_ai/adapters/test_ml_model_list.py | 26 +- 3 files changed, 497 insertions(+), 28 deletions(-) create mode 100644 libs/core/kiln_ai/adapters/model_adapters/test_openai_responses_routing_paid.py diff --git a/libs/core/kiln_ai/adapters/ml_model_list.py b/libs/core/kiln_ai/adapters/ml_model_list.py index b281085dee..49a13f7448 100644 --- a/libs/core/kiln_ai/adapters/ml_model_list.py +++ b/libs/core/kiln_ai/adapters/ml_model_list.py @@ -709,10 +709,8 @@ class KilnModel(BaseModel): structured_output_mode=StructuredOutputMode.json_schema, available_thinking_levels=GPT_6_ASTRA_OPENAI_THINKING_LEVELS, default_thinking_level="medium", - # OpenAI rejects reasoning_effort + tools on /v1/chat/completions - # for gpt-5.4+. Disable function calling until Kiln routes these - # models to /v1/responses. - supports_function_calling=False, + # Reasoning models reject tools on /v1/chat/completions; route to /v1/responses. + openai_responses_api=True, supports_doc_extraction=True, supports_vision=True, multimodal_capable=True, @@ -769,10 +767,8 @@ class KilnModel(BaseModel): structured_output_mode=StructuredOutputMode.json_schema, available_thinking_levels=GPT_5_4_OPENAI_THINKING_LEVELS, default_thinking_level="none", - # OpenAI rejects reasoning_effort + tools on /v1/chat/completions - # for gpt-5.4+. Disable function calling until Kiln routes these - # models to /v1/responses. - supports_function_calling=False, + # Reasoning models reject tools on /v1/chat/completions; route to /v1/responses. + openai_responses_api=True, supports_doc_extraction=True, supports_vision=True, multimodal_capable=True, @@ -827,10 +823,8 @@ class KilnModel(BaseModel): structured_output_mode=StructuredOutputMode.json_schema, available_thinking_levels=GPT_5_4_OPENAI_THINKING_LEVELS, default_thinking_level="none", - # OpenAI rejects reasoning_effort + tools on /v1/chat/completions - # for gpt-5.4+. Disable function calling until Kiln routes these - # models to /v1/responses. - supports_function_calling=False, + # Reasoning models reject tools on /v1/chat/completions; route to /v1/responses. + openai_responses_api=True, supports_doc_extraction=True, supports_vision=True, multimodal_capable=True, @@ -883,10 +877,8 @@ class KilnModel(BaseModel): structured_output_mode=StructuredOutputMode.json_schema, available_thinking_levels=GPT_5_4_OPENAI_THINKING_LEVELS, default_thinking_level="none", - # OpenAI rejects reasoning_effort + tools on /v1/chat/completions - # for gpt-5.4+. Disable function calling until Kiln routes these - # models to /v1/responses. - supports_function_calling=False, + # Reasoning models reject tools on /v1/chat/completions; route to /v1/responses. + openai_responses_api=True, supports_doc_extraction=True, supports_vision=True, multimodal_capable=True, @@ -938,10 +930,8 @@ class KilnModel(BaseModel): structured_output_mode=StructuredOutputMode.json_schema, available_thinking_levels=GPT_5_4_OPENAI_THINKING_LEVELS, default_thinking_level="none", - # OpenAI rejects reasoning_effort + tools on /v1/chat/completions - # for gpt-5.4+. Disable function calling until Kiln routes these - # models to /v1/responses. - supports_function_calling=False, + # Reasoning models reject tools on /v1/chat/completions; route to /v1/responses. + openai_responses_api=True, supports_doc_extraction=True, supports_vision=True, multimodal_capable=True, @@ -993,10 +983,8 @@ class KilnModel(BaseModel): structured_output_mode=StructuredOutputMode.json_schema, available_thinking_levels=GPT_5_4_OPENAI_THINKING_LEVELS, default_thinking_level="none", - # OpenAI rejects reasoning_effort + tools on /v1/chat/completions - # for gpt-5.4 direct. Disable function calling until Kiln routes - # these models to /v1/responses. The OpenRouter route is unaffected. - supports_function_calling=False, + # Reasoning models reject tools on /v1/chat/completions; route to /v1/responses. + openai_responses_api=True, supports_doc_extraction=True, supports_vision=True, multimodal_capable=True, diff --git a/libs/core/kiln_ai/adapters/model_adapters/test_openai_responses_routing_paid.py b/libs/core/kiln_ai/adapters/model_adapters/test_openai_responses_routing_paid.py new file mode 100644 index 0000000000..c194343239 --- /dev/null +++ b/libs/core/kiln_ai/adapters/model_adapters/test_openai_responses_routing_paid.py @@ -0,0 +1,463 @@ +"""Red-first reproduction of the "tools + thinking level" problem for OpenAI-direct +GPT-5.4+ / GPT-6 models. + +OpenAI rejects ``reasoning_effort`` together with ``tools`` on +``/v1/chat/completions`` for gpt-5.4 and newer. The fix is to route those requests to +``/v1/responses``. litellm 1.87.1 already contains an auto-bridge +(``litellm.main.responses_api_bridge_check``) that does exactly that, but its model +matcher is ``"gpt-5" in model`` + ``gpt-5.`` version parsing, so it does not +match ``gpt-6-astra``. + +There are therefore TWO distinct failure modes, and this test detects both: + +1. Hard failure: the request goes to ``/v1/chat/completions`` carrying both + ``reasoning_effort`` and ``tools`` and OpenAI returns a 400. +2. Silent failure: litellm's ``drop_params`` strips ``reasoning_effort`` before the + call, the request succeeds on ``/v1/chat/completions``, and the requested thinking + level is simply ignored. + +To distinguish them we record every outgoing HTTP request made during the run by +patching ``httpx.AsyncClient.send``. That is the single choke point shared by the +OpenAI python SDK (used by litellm's chat-completions path) and litellm's own +``AsyncHTTPHandler`` (used by the responses path), so it reliably records the +bridge's *inner* request, which a litellm ``CustomLogger`` callback does not always +surface. + +Regression test for that routing. Every built-in provider carrying +``openai_responses_api=True`` must reach ``/v1/responses`` with the requested effort +on the wire, run a real tool loop to a correct answer, and report usage/cost back to +Kiln. ``gpt_5_2`` is the chat-completions control: it predates the restriction, so it +must keep using ``/v1/chat/completions``. +""" + +import contextlib +import json +from dataclasses import dataclass, field +from typing import Any, Iterator +from unittest.mock import Mock, patch + +import httpx +import pytest + +from kiln_ai.adapters.adapter_registry import adapter_for_task +from kiln_ai.adapters.ml_model_list import ModelName, built_in_models +from kiln_ai.adapters.model_adapters.test_litellm_adapter_tools import build_test_task +from kiln_ai.adapters.model_adapters.test_paid_utils import ( + skip_if_missing_provider_keys, +) +from kiln_ai.adapters.model_adapters.test_thinking_level_paid import ( + reasoning_content_from_run, +) +from kiln_ai.datamodel.datamodel_enums import ModelProviderName, StructuredOutputMode +from kiln_ai.datamodel.run_config import KilnAgentRunConfigProperties +from kiln_ai.tools.built_in_tools.math_tools import AddTool + +# The prompt is trivial on purpose: one tool call, then a final answer. That is two +# HTTP requests to the provider, which is the minimum needed to exercise the +# tool-result turn (the turn where a responses-API call must echo back reasoning +# items). +TOOL_PROMPT = "what is 2+2" + +# Thinking level used for every case. "high" is the highest-but-one level for the +# GPT-5.4 family (none/low/medium/high/xhigh) and is a valid non-default level for +# GPT-6 Astra (low/medium/high/xhigh/max), so one value keeps the cases comparable. +REQUESTED_EFFORT = "high" + + +@dataclass +class RecordedRequest: + """One outgoing HTTP request to a model provider.""" + + method: str + url: str + path: str + body: dict[str, Any] | None + + @property + def effort(self) -> Any: + """The reasoning effort actually present in the wire payload, or None. + + Chat Completions carries ``reasoning_effort: "high"``. + The Responses API carries ``reasoning: {"effort": "high"}``. + litellm may also emit ``reasoning_effort: {"effort": ..., "summary": ...}``. + """ + body = self.body or {} + raw = body.get("reasoning_effort") + if isinstance(raw, str): + return raw + if isinstance(raw, dict): + return raw.get("effort") + reasoning = body.get("reasoning") + if isinstance(reasoning, dict): + return reasoning.get("effort") + return None + + @property + def has_tools(self) -> bool: + return bool((self.body or {}).get("tools")) + + @property + def payload_shape(self) -> str: + """`messages` => chat-completions shape, `input` => responses shape.""" + body = self.body or {} + if "input" in body: + return "input" + if "messages" in body: + return "messages" + return "?" + + @property + def reasoning_fields(self) -> dict[str, Any]: + """Every reasoning-ish key actually on the wire, verbatim. + + Anthropic never sees `reasoning_effort`: litellm maps it to + `thinking: {type: enabled, budget_tokens: N}`, so `effort` is None there + even though a thinking budget was sent. Recording the raw keys keeps the + report honest across providers. + """ + body = self.body or {} + return { + key: body[key] + for key in ("reasoning_effort", "reasoning", "thinking") + if key in body + } + + def __str__(self) -> str: + return ( + f"{self.path} (payload={self.payload_shape}, effort={self.effort!r}, " + f"tools={self.has_tools}, reasoning_fields={self.reasoning_fields})" + ) + + +@dataclass +class RequestLog: + host_fragment: str + requests: list[RecordedRequest] = field(default_factory=list) + + @property + def paths(self) -> list[str]: + return [r.path for r in self.requests] + + @property + def efforts(self) -> list[Any]: + return [r.effort for r in self.requests] + + def summary(self) -> str: + if not self.requests: + return " (no provider requests recorded)" + return "\n".join(f" #{i + 1} {r}" for i, r in enumerate(self.requests)) + + +@contextlib.contextmanager +def record_provider_requests(host_fragment: str) -> Iterator[RequestLog]: + """Record every outgoing request whose host contains ``host_fragment``. + + Patching ``httpx.AsyncClient.send`` catches both the OpenAI SDK path and + litellm's own http handler, so the bridge's inner /v1/responses call is + recorded even though it is issued by a different client than the outer + ``litellm.acompletion`` call. + """ + log = RequestLog(host_fragment=host_fragment) + original_send = httpx.AsyncClient.send + + async def spy_send(self, *args, **kwargs): + request = args[0] if args else kwargs.get("request") + try: + host = request.url.host or "" + if host_fragment in host: + try: + body = json.loads(request.content.decode("utf-8")) + except Exception: + body = None + log.requests.append( + RecordedRequest( + method=request.method, + url=str(request.url), + path=request.url.path, + body=body if isinstance(body, dict) else None, + ) + ) + except Exception: + # Recording must never change the behaviour under test. + pass + return await original_send(self, *args, **kwargs) + + with patch.object(httpx.AsyncClient, "send", spy_send): + yield log + + +def provider_thinking_levels(model_name: str, provider_name: str) -> list[str]: + for model in built_in_models: + if model.name != model_name: + continue + for provider in model.providers: + if provider.name != provider_name: + continue + return list((provider.available_thinking_levels or {}).values()) + raise RuntimeError(f"No model {model_name} on provider {provider_name}") + + +def effort_for(model_name: str, provider_name: str) -> str: + levels = provider_thinking_levels(model_name, provider_name) + if not levels: + raise RuntimeError(f"{model_name}/{provider_name} has no thinking levels") + if REQUESTED_EFFORT in levels: + return REQUESTED_EFFORT + # Highest-but-one, as a fallback for any model without a "high" level. + return levels[-2] if len(levels) > 1 else levels[-1] + + +async def run_tool_loop( + tmp_path, + model_name: str, + provider_name: str, + thinking_level: str, + host_fragment: str, + temperature: float = 1.0, + top_p: float = 1.0, +): + """Run the real Kiln adapter tool loop, recording every provider request. + + Returns (run, error, add_spy, request_log). Exactly one of run/error is set. + """ + task = build_test_task(tmp_path) + adapter = adapter_for_task( + task, + KilnAgentRunConfigProperties( + structured_output_mode=StructuredOutputMode.json_schema, + model_name=model_name, + model_provider_name=ModelProviderName(provider_name), + prompt_id="simple_prompt_builder", + thinking_level=thinking_level, + temperature=temperature, + top_p=top_p, + ), + ) + + add_spy = Mock(wraps=AddTool()) + + run = None + error: Exception | None = None + with record_provider_requests(host_fragment) as request_log: + with patch.object(adapter, "available_tools", return_value=[add_spy]): + try: + run = await adapter.invoke(TOOL_PROMPT) + except Exception as e: + error = e + + return run, error, add_spy, request_log + + +def first_error_line(error: Exception | None) -> str: + """First line of the deepest cause. + + Kiln wraps adapter failures in KilnRunError("An unexpected error occurred."), + which hides the provider's 400 body. Unwrap `.original` / `__cause__` so the + verbatim provider error is what gets reported. + """ + if error is None: + return "(none)" + chain: list[str] = [] + current: BaseException | None = error + seen: set[int] = set() + while current is not None and id(current) not in seen: + seen.add(id(current)) + chain.append(f"{type(current).__name__}: {current}".splitlines()[0]) + current = getattr(current, "original", None) or current.__cause__ + # The last link is the root cause (the provider error); show it first. + return " <- ".join(reversed(chain))[:900] + + +def build_context( + model_name: str, + provider_name: str, + thinking_level: str, + request_log: RequestLog, + error: Exception | None, + run, + tool_called: bool, +) -> str: + reasoning = reasoning_content_from_run(run) if run is not None else None + final_answer = repr(run.output.output[:200]) if run is not None else "None" + return ( + f"\ncase: model={model_name} provider={provider_name} " + f"requested_effort={thinking_level!r}" + f"\nrequests observed ({len(request_log.requests)}):\n{request_log.summary()}" + f"\nendpoints: {request_log.paths}" + f"\nefforts on the wire: {request_log.efforts}" + f"\ntool called: {tool_called}" + f"\nfinal answer: {final_answer}" + f"\nreasoning surfaced: " + f"{'yes (%d chars)' % len(reasoning) if reasoning else 'no'}" + f"\nfirst error line: {first_error_line(error)}\n" + ) + + +def responses_api_cases() -> list[Any]: + """Every built-in (model, provider) pair Kiln routes through /v1/responses.""" + cases: list[Any] = [] + for model in built_in_models: + for provider in model.providers: + if not provider.openai_responses_api: + continue + cases.append( + pytest.param( + model.name, + provider.name, + True, + id=f"{model.name}_{provider.name.value}", + ) + ) + return cases + + +# gpt_5_2 is the control: it predates the 5.4 restriction, so reasoning_effort + +# tools is legal on /v1/chat/completions and no bridging should happen. +OPENAI_CASES = [ + *responses_api_cases(), + pytest.param( + ModelName.gpt_5_2.value, + ModelProviderName.openai, + False, + id="gpt_5_2_chat_completions_control", + ), +] + + +@pytest.mark.paid +@pytest.mark.parametrize( + ("model_name", "provider_name", "expect_responses_endpoint"), OPENAI_CASES +) +async def test_openai_tools_with_thinking_level_routing( + tmp_path, model_name: str, provider_name: str, expect_responses_endpoint: bool +): + """Tools + an explicit thinking level must work on the OpenAI-direct provider.""" + skip_if_missing_provider_keys(provider_name) + + thinking_level = effort_for(model_name, provider_name) + + run, error, add_spy, request_log = await run_tool_loop( + tmp_path, + model_name=model_name, + provider_name=provider_name, + thinking_level=thinking_level, + host_fragment="openai.com", + ) + + tool_called = add_spy.run.called + ctx = build_context( + model_name, provider_name, thinking_level, request_log, error, run, tool_called + ) + print(ctx) + + expected_path_fragment = ( + "/responses" if expect_responses_endpoint else "/chat/completions" + ) + + assert request_log.requests, f"No provider request was recorded at all.{ctx}" + + # 1. Every request in the loop must hit the right endpoint. For gpt-5.4+ and + # GPT-6 that is /v1/responses, which is the only endpoint that accepts + # reasoning effort alongside tools. + wrong_endpoint = [ + r for r in request_log.requests if expected_path_fragment not in r.path + ] + assert not wrong_endpoint, ( + f"Expected every request to go to {expected_path_fragment}, but " + f"{len(wrong_endpoint)} of {len(request_log.requests)} did not. " + f"Observed endpoints: {request_log.paths}.{ctx}" + ) + + # 2. Every request must actually carry the requested reasoning effort. litellm's + # drop_params silently removes it for models it does not recognise, which + # makes the thinking level a no-op rather than an error. + missing_effort = [r for r in request_log.requests if r.effort != thinking_level] + assert not missing_effort, ( + f"Expected every request to carry effort={thinking_level!r}, but observed " + f"efforts {request_log.efforts}.{ctx}" + ) + + # 3. The run must have completed without error. + assert error is None, f"Run raised: {first_error_line(error)}{ctx}" + assert run is not None, f"No run produced.{ctx}" + + # 4. The tool loop must have made at least two provider calls: the turn that + # produced the tool call, and the turn that consumed the tool result. + assert len(request_log.requests) >= 2, ( + f"Expected >=2 provider requests (tool call turn + tool result turn), " + f"got {len(request_log.requests)}.{ctx}" + ) + + # 5. The tool was actually called, with the right arguments. + assert tool_called, f"The 'add' tool was never called.{ctx}" + add_kwargs = add_spy.run.call_args.kwargs + assert add_kwargs.get("a") == 2 and add_kwargs.get("b") == 2, ( + f"Expected add(a=2, b=2), got {add_kwargs}.{ctx}" + ) + + # 6. The final answer is correct. + assert "4" in run.output.output, f"Final answer missing '4'.{ctx}" + + # 7. Usage and cost survive the responses bridge. litellm reports cost in a + # different place for the responses path, and Kiln's usage_from_response has + # to find it there too or runs show as free. + assert run.usage is not None, f"No usage recorded on the run.{ctx}" + assert run.usage.cost is not None, f"No cost recorded on the run.{ctx}" + assert run.usage.output_tokens, f"No output tokens recorded on the run.{ctx}" + + +# Proving the sampling-param drop costs real calls, so cover only gpt-6 (which litellm +# does not recognise as a reasoning model) and one gpt-5.x (which it does). +CUSTOM_SAMPLING_CASES = [ + pytest.param(ModelName.gpt_6_astra.value, id="gpt_6_astra"), + pytest.param(ModelName.gpt_5_4.value, id="gpt_5_4"), +] + + +@pytest.mark.paid +@pytest.mark.parametrize("model_name", CUSTOM_SAMPLING_CASES) +async def test_openai_responses_drops_custom_sampling_params(tmp_path, model_name: str): + """A run config with custom temperature/top_p must still work once routed. + + OpenAI's reasoning models reject top_p outright and accept only the default + temperature on /v1/responses. litellm strips both for the gpt-5.x family it + recognises, so Kiln does the same for the models it doesn't (eg gpt-6-astra), + which would otherwise 400 on every call. + """ + provider_name = ModelProviderName.openai + skip_if_missing_provider_keys(provider_name) + + thinking_level = effort_for(model_name, provider_name) + + run, error, add_spy, request_log = await run_tool_loop( + tmp_path, + model_name=model_name, + provider_name=provider_name, + thinking_level=thinking_level, + host_fragment="openai.com", + temperature=0.4, + top_p=0.9, + ) + + tool_called = add_spy.run.called + ctx = build_context( + model_name, provider_name, thinking_level, request_log, error, run, tool_called + ) + print(ctx) + + assert request_log.requests, f"No provider request was recorded at all.{ctx}" + + kept_sampling = [ + r + for r in request_log.requests + if "top_p" in (r.body or {}) or "temperature" in (r.body or {}) + ] + assert not kept_sampling, ( + f"Expected temperature and top_p to be dropped, but {len(kept_sampling)} of " + f"{len(request_log.requests)} requests still carried one.{ctx}" + ) + + assert error is None, f"Run raised: {first_error_line(error)}{ctx}" + assert run is not None, f"No run produced.{ctx}" + assert tool_called, f"The 'add' tool was never called.{ctx}" + assert "4" in run.output.output, f"Final answer missing '4'.{ctx}" diff --git a/libs/core/kiln_ai/adapters/test_ml_model_list.py b/libs/core/kiln_ai/adapters/test_ml_model_list.py index 8a1a340891..c544ee9847 100644 --- a/libs/core/kiln_ai/adapters/test_ml_model_list.py +++ b/libs/core/kiln_ai/adapters/test_ml_model_list.py @@ -213,6 +213,18 @@ def test_openrouter_reasoning_object_requires_openrouter(self): ) +# Models we route to /v1/responses because OpenAI rejects tools alongside +# reasoning_effort on /v1/chat/completions for them. +RESPONSES_API_MODELS = [ + ModelName.gpt_6_astra, + ModelName.gpt_5_6_sol, + ModelName.gpt_5_6_terra, + ModelName.gpt_5_6_luna, + ModelName.gpt_5_5, + ModelName.gpt_5_4, +] + + class TestOpenAIResponsesApi: def test_openai_responses_api_allowed_on_openai(self): provider = KilnModelProvider( @@ -237,20 +249,26 @@ def test_openai_responses_api_requires_openai(self, provider_name): openai_responses_api=True, ) - def test_built_in_models_with_flag_are_openai_and_support_tools(self): + def test_built_in_models_with_flag_support_tools(self): """Routing to /v1/responses exists so these models can use tools. If one of them ever ships with function calling disabled again, the workaround has returned.""" for model in built_in_models: for provider in model.providers: if not provider.openai_responses_api: continue - assert provider.name == ModelProviderName.openai, ( - f"{model.name} has openai_responses_api on {provider.name}" - ) assert provider.supports_function_calling is True, ( f"{model.name} routes to /v1/responses but disables function calling" ) + @pytest.mark.parametrize("model_name", RESPONSES_API_MODELS) + def test_reasoning_models_route_to_responses_api(self, model_name): + provider = built_in_models_from_provider( + provider_name=ModelProviderName.openai, + model_name=model_name, + ) + assert provider is not None + assert provider.openai_responses_api is True + class TestBuiltInModelsFromProvider: """Test cases for built_in_models_from_provider function""" From ca276a7221391837e2196526b9a0490412ef0d1f Mon Sep 17 00:00:00 2001 From: tawnymanticore Date: Tue, 8 Sep 2026 11:56:01 -0400 Subject: [PATCH 3/6] Request reasoning summaries for responses-routed OpenAI models OpenAI's /v1/responses returns no reasoning at all unless the request asks for a summary, so these models lost their thinking output the moment we routed them off chat completions. litellm folds a reasoning_summary kwarg into reasoning={"effort": ..., "summary": ...} and fills message.reasoning_content from what comes back. "none" is left alone: it disables reasoning, so there is no summary to ask for. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_014jrCq27Qa4VQeVEvYjSfTs --- .../model_adapters/litellm_adapter.py | 5 +++ .../model_adapters/test_litellm_adapter.py | 32 +++++++++++++++++++ .../test_openai_responses_routing_paid.py | 15 +++++++++ 3 files changed, 52 insertions(+) diff --git a/libs/core/kiln_ai/adapters/model_adapters/litellm_adapter.py b/libs/core/kiln_ai/adapters/model_adapters/litellm_adapter.py index 0df4679e04..853e66da0d 100644 --- a/libs/core/kiln_ai/adapters/model_adapters/litellm_adapter.py +++ b/libs/core/kiln_ai/adapters/model_adapters/litellm_adapter.py @@ -568,6 +568,11 @@ def build_extra_body(self, provider: KilnModelProvider) -> dict[str, Any]: pass else: extra_body["reasoning_effort"] = thinking_level + if provider.openai_responses_api and thinking_level != "none": + # litellm folds this into reasoning={"effort": ..., "summary": ...} + # for the responses bridge, and fills message.reasoning_content from + # the summary. Without it these models surface no reasoning at all. + extra_body["reasoning_summary"] = "auto" # Opus 4.7/4.8 default thinking display to "omitted", returning empty # thinking text. Request the summary so reasoning is surfaced. litellm # still maps reasoning_effort to output_config.effort; this only adds the diff --git a/libs/core/kiln_ai/adapters/model_adapters/test_litellm_adapter.py b/libs/core/kiln_ai/adapters/model_adapters/test_litellm_adapter.py index d23d51c1f5..f743bd1cfb 100644 --- a/libs/core/kiln_ai/adapters/model_adapters/test_litellm_adapter.py +++ b/libs/core/kiln_ai/adapters/model_adapters/test_litellm_adapter.py @@ -616,6 +616,38 @@ def test_build_extra_body_thinking_level_anthropic_summarized_thinking( assert extra_body.get("thinking") == {"type": "adaptive", "display": "summarized"} +@pytest.mark.parametrize( + "openai_responses_api,thinking_level,expected_summary", + [ + (True, "high", "auto"), + # "none" disables reasoning, so there is no summary to ask for + (True, "none", None), + (False, "high", None), + ], +) +def test_build_extra_body_reasoning_summary_for_openai_responses_api( + config, mock_task, openai_responses_api, thinking_level, expected_summary +): + """The responses API returns no reasoning unless a summary is requested. litellm + folds reasoning_summary into reasoning={"effort": ..., "summary": ...} and fills + message.reasoning_content from what comes back.""" + config.run_config_properties.thinking_level = thinking_level + adapter = LiteLlmAdapter(config=config, kiln_task=mock_task) + + provider = KilnModelProvider( + name=ModelProviderName.openai, + model_id="test-model", + openai_responses_api=openai_responses_api, + available_thinking_levels={"Off/None": "none", "High": "high"}, + default_thinking_level="none", + ) + + extra_body = adapter.build_extra_body(provider) + + assert extra_body.get("reasoning_effort") == thinking_level + assert extra_body.get("reasoning_summary") == expected_summary + + def test_build_extra_body_thinking_level_anthropic_none(config, mock_task): """Anthropic's native API has no reasoning_effort="none" (litellm crashes on it), so a "none" thinking level must omit reasoning_effort entirely to disable thinking.""" diff --git a/libs/core/kiln_ai/adapters/model_adapters/test_openai_responses_routing_paid.py b/libs/core/kiln_ai/adapters/model_adapters/test_openai_responses_routing_paid.py index c194343239..f6b919a2c8 100644 --- a/libs/core/kiln_ai/adapters/model_adapters/test_openai_responses_routing_paid.py +++ b/libs/core/kiln_ai/adapters/model_adapters/test_openai_responses_routing_paid.py @@ -405,6 +405,21 @@ async def test_openai_tools_with_thinking_level_routing( assert run.usage.cost is not None, f"No cost recorded on the run.{ctx}" assert run.usage.output_tokens, f"No output tokens recorded on the run.{ctx}" + # 8. The responses API returns no reasoning unless a summary is requested, so Kiln + # asks for one on every call. Whether the model then emits a summary is its own + # decision (it usually declines on a prompt this trivial), so the assertion is + # on what Kiln controls. `ctx` reports whether reasoning came back either way. + if expect_responses_endpoint: + missing_summary = [ + r + for r in request_log.requests + if (r.body or {}).get("reasoning", {}).get("summary") != "auto" + ] + assert not missing_summary, ( + f"Expected every request to ask for a reasoning summary, but " + f"{len(missing_summary)} of {len(request_log.requests)} did not.{ctx}" + ) + # Proving the sampling-param drop costs real calls, so cover only gpt-6 (which litellm # does not recognise as a reasoning model) and one gpt-5.x (which it does). From eb9fdc2ee4c3eb7aab8c5e5470d7c39847fdcee5 Mon Sep 17 00:00:00 2001 From: tawnymanticore Date: Tue, 8 Sep 2026 13:11:05 -0400 Subject: [PATCH 4/6] Remove debug prints from the responses routing paid test The debug detector CI check rejects print() in Python files. The captured request context is already part of every assertion message, so the prints added nothing. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_014jrCq27Qa4VQeVEvYjSfTs --- .../model_adapters/test_openai_responses_routing_paid.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/libs/core/kiln_ai/adapters/model_adapters/test_openai_responses_routing_paid.py b/libs/core/kiln_ai/adapters/model_adapters/test_openai_responses_routing_paid.py index f6b919a2c8..7a242fa92e 100644 --- a/libs/core/kiln_ai/adapters/model_adapters/test_openai_responses_routing_paid.py +++ b/libs/core/kiln_ai/adapters/model_adapters/test_openai_responses_routing_paid.py @@ -348,7 +348,6 @@ async def test_openai_tools_with_thinking_level_routing( ctx = build_context( model_name, provider_name, thinking_level, request_log, error, run, tool_called ) - print(ctx) expected_path_fragment = ( "/responses" if expect_responses_endpoint else "/chat/completions" @@ -458,7 +457,6 @@ async def test_openai_responses_drops_custom_sampling_params(tmp_path, model_nam ctx = build_context( model_name, provider_name, thinking_level, request_log, error, run, tool_called ) - print(ctx) assert request_log.requests, f"No provider request was recorded at all.{ctx}" From 0b7bc78068b950729123c220868ca3447389bd68 Mon Sep 17 00:00:00 2001 From: tawnymanticore Date: Wed, 9 Sep 2026 10:40:47 -0400 Subject: [PATCH 5/6] Merge split choices from the litellm responses bridge so tool calls are not dropped litellm's responses bridge returns one Choices per assistant content part plus a trailing Choices holding every tool call, so a turn that narrates before calling a tool puts the call in choices[1]. Kiln read only choices[0], silently dropping the call: the loop ended after one turn and the narration was saved as the answer. Merge them back into one real Choices/Message, gated on openai_responses_api so providers that legitimately return several choices are untouched. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_014jrCq27Qa4VQeVEvYjSfTs --- .../model_adapters/litellm_adapter.py | 48 +++++ .../model_adapters/test_litellm_adapter.py | 180 ++++++++++++++++++ .../test_openai_responses_routing_paid.py | 85 +++++++-- 3 files changed, 297 insertions(+), 16 deletions(-) diff --git a/libs/core/kiln_ai/adapters/model_adapters/litellm_adapter.py b/libs/core/kiln_ai/adapters/model_adapters/litellm_adapter.py index 853e66da0d..3239908973 100644 --- a/libs/core/kiln_ai/adapters/model_adapters/litellm_adapter.py +++ b/libs/core/kiln_ai/adapters/model_adapters/litellm_adapter.py @@ -428,8 +428,56 @@ async def acompletion_checking_response( raise RuntimeError( f"Expected ModelResponse with Choices, got {type(response)}." ) + + if self.model_provider().openai_responses_api and len(response.choices) > 1: + # litellm's responses bridge splits one assistant turn into a Choices per + # content part plus a trailing Choices holding the tool calls. Reading only + # choices[0] would drop the tool call whenever the model narrates first. + merged = self._merge_split_choices(response.choices) + response.choices = [merged] + return response, merged + return response, response.choices[0] + def _merge_split_choices(self, choices: List[Any]) -> Choices: + """Collapse the choices litellm's responses bridge split into one turn.""" + messages = [ + choice.message + for choice in choices + if isinstance(choice, Choices) and choice.message is not None + ] + + contents = [m.content for m in messages if m.content] + tool_calls: List[Any] = [] + for message in messages: + tool_calls.extend(message.tool_calls or []) + + def first(field: str) -> Any: + return next( + ( + value + for value in (getattr(m, field, None) for m in messages) + if value is not None + ), + None, + ) + + merged_message = LiteLLMMessage( + content="\n".join(contents) if contents else None, + role=first("role") or "assistant", + tool_calls=tool_calls or None, + reasoning_content=first("reasoning_content"), + reasoning_items=first("reasoning_items"), + annotations=first("annotations"), + provider_specific_fields=first("provider_specific_fields"), + ) + + return Choices( + finish_reason="tool_calls" if tool_calls else choices[0].finish_reason, + index=0, + message=merged_message, + ) + def adapter_name(self) -> str: return "kiln_openai_compatible_adapter" diff --git a/libs/core/kiln_ai/adapters/model_adapters/test_litellm_adapter.py b/libs/core/kiln_ai/adapters/model_adapters/test_litellm_adapter.py index f743bd1cfb..8925632eeb 100644 --- a/libs/core/kiln_ai/adapters/model_adapters/test_litellm_adapter.py +++ b/libs/core/kiln_ai/adapters/model_adapters/test_litellm_adapter.py @@ -7,9 +7,11 @@ from litellm.types.utils import ( ChatCompletionMessageToolCall, ChoiceLogprobs, + Choices, Function, ModelResponse, ) +from litellm.types.utils import Message as LiteLLMMessage from kiln_ai.adapters.ml_model_list import ( KilnModelProvider, @@ -3521,3 +3523,181 @@ async def test_missing_finish_reason_raises_generic_error(self, adapter, provide await adapter._run_model_turn( provider, [{"role": "user", "content": "Hi"}], None, False ) + + +def responses_bridge_provider(openai_responses_api: bool = True) -> KilnModelProvider: + return KilnModelProvider( + name=ModelProviderName.openai, + model_id="gpt-6-astra", + openai_responses_api=openai_responses_api, + ) + + +def add_tool_call(call_id: str, a: int, b: int) -> ChatCompletionMessageToolCall: + return ChatCompletionMessageToolCall( + id=call_id, + type="function", + function=Function(name="add", arguments=json.dumps({"a": a, "b": b})), + ) + + +def split_choices_response( + contents: list[str], + tool_calls: list[ChatCompletionMessageToolCall], + reasoning_content: str | None = "I should use the add tool.", +) -> ModelResponse: + """A response shaped the way litellm's responses bridge returns one turn. + + One Choices per content part (finish_reason "stop"), then a trailing Choices + holding every tool call (finish_reason "tool_calls", content None). + """ + choices = [ + Choices( + finish_reason="stop", + index=index, + message=LiteLLMMessage( + content=content, + role="assistant", + reasoning_content=reasoning_content if index == 0 else None, + ), + ) + for index, content in enumerate(contents) + ] + if tool_calls: + choices.append( + Choices( + finish_reason="tool_calls", + index=len(choices), + message=LiteLLMMessage( + content=None, role="assistant", tool_calls=tool_calls + ), + ) + ) + return ModelResponse(model="gpt-6-astra", choices=choices) + + +@pytest.mark.asyncio +async def test_acompletion_merges_split_choices_from_responses_bridge( + config, mock_task +): + """litellm's responses bridge splits text and tool calls across several Choices. + Reading only choices[0] would drop the tool call whenever the model narrates.""" + adapter = LiteLlmAdapter(config=config, kiln_task=mock_task) + response = split_choices_response( + ["I will use the add tool."], [add_tool_call("call_1", 2, 2)] + ) + + with ( + patch.object( + adapter, "model_provider", return_value=responses_bridge_provider() + ), + patch("litellm.acompletion", new=AsyncMock(return_value=response)), + ): + returned_response, choice = await adapter.acompletion_checking_response() + + # The tool loop appends choice.message to the history and dispatches on the real + # litellm types, so the merge must produce real Choices/Message objects. + assert isinstance(choice, Choices) + assert isinstance(choice.message, LiteLLMMessage) + assert choice.message.content == "I will use the add tool." + assert choice.message.tool_calls is not None + assert [t.function.name for t in choice.message.tool_calls] == ["add"] + assert choice.finish_reason == "tool_calls" + assert choice.index == 0 + assert choice.message.role == "assistant" + assert choice.message.reasoning_content == "I should use the add tool." + # Usage/cost readers walk response.choices, so it must agree with what we returned. + assert returned_response.choices == [choice] + + +@pytest.mark.asyncio +async def test_acompletion_merges_multiple_text_parts_and_tool_calls(config, mock_task): + adapter = LiteLlmAdapter(config=config, kiln_task=mock_task) + response = split_choices_response( + ["First I plan.", "Then I act."], + [add_tool_call("call_1", 2, 2), add_tool_call("call_2", 3, 4)], + ) + + with ( + patch.object( + adapter, "model_provider", return_value=responses_bridge_provider() + ), + patch("litellm.acompletion", new=AsyncMock(return_value=response)), + ): + _, choice = await adapter.acompletion_checking_response() + + assert choice.message.content == "First I plan.\nThen I act." + assert [t.id for t in choice.message.tool_calls or []] == ["call_1", "call_2"] + + +@pytest.mark.asyncio +async def test_acompletion_does_not_merge_without_responses_api_flag(config, mock_task): + """Providers that legitimately return several choices are untouched.""" + adapter = LiteLlmAdapter(config=config, kiln_task=mock_task) + response = split_choices_response( + ["I will use the add tool."], [add_tool_call("call_1", 2, 2)] + ) + + with ( + patch.object( + adapter, + "model_provider", + return_value=responses_bridge_provider(openai_responses_api=False), + ), + patch("litellm.acompletion", new=AsyncMock(return_value=response)), + ): + returned_response, choice = await adapter.acompletion_checking_response() + + assert len(returned_response.choices) == 2 + assert choice is returned_response.choices[0] + + +@pytest.mark.asyncio +async def test_acompletion_single_choice_is_returned_unchanged(config, mock_task): + adapter = LiteLlmAdapter(config=config, kiln_task=mock_task) + response = split_choices_response(["Just an answer."], []) + + with ( + patch.object( + adapter, "model_provider", return_value=responses_bridge_provider() + ), + patch("litellm.acompletion", new=AsyncMock(return_value=response)), + ): + returned_response, choice = await adapter.acompletion_checking_response() + + assert returned_response is response + assert choice is response.choices[0] + + +@pytest.mark.asyncio +async def test_run_model_turn_executes_tool_from_split_choices(config, mock_task): + """End to end through the tool loop: the merged tool call must actually run and + the loop must make a second LLM call with the tool result.""" + adapter = LiteLlmAdapter(config=config, kiln_task=mock_task) + provider = responses_bridge_provider() + + split_response = split_choices_response( + ["I will use the add tool."], [add_tool_call("call_1", 2, 2)] + ) + final_response = ModelResponse( + model="gpt-6-astra", + choices=[{"message": {"content": "The answer is 4", "tool_calls": None}}], + ) + acompletion = AsyncMock(side_effect=[split_response, final_response]) + + add_spy = Mock(wraps=AddTool()) + messages: list = [{"role": "user", "content": "what is 2+2"}] + + with ( + patch.object(adapter, "model_provider", return_value=provider), + patch.object(adapter, "cached_available_tools", return_value=[add_spy]), + patch.object(adapter, "build_completion_kwargs", return_value={}), + patch("litellm.acompletion", new=acompletion), + ): + result = await adapter._run_model_turn(provider, messages, None, False) + + add_spy.run.assert_called_once() + assert add_spy.run.call_args.kwargs == {"a": 2, "b": 2} + assert acompletion.await_count == 2 + assert isinstance(result, ModelTurnResult) + assert result.assistant_message == "The answer is 4" diff --git a/libs/core/kiln_ai/adapters/model_adapters/test_openai_responses_routing_paid.py b/libs/core/kiln_ai/adapters/model_adapters/test_openai_responses_routing_paid.py index 7a242fa92e..f1c9334030 100644 --- a/libs/core/kiln_ai/adapters/model_adapters/test_openai_responses_routing_paid.py +++ b/libs/core/kiln_ai/adapters/model_adapters/test_openai_responses_routing_paid.py @@ -215,6 +215,7 @@ async def run_tool_loop( host_fragment: str, temperature: float = 1.0, top_p: float = 1.0, + prompt: str = TOOL_PROMPT, ): """Run the real Kiln adapter tool loop, recording every provider request. @@ -241,7 +242,7 @@ async def run_tool_loop( with record_provider_requests(host_fragment) as request_log: with patch.object(adapter, "available_tools", return_value=[add_spy]): try: - run = await adapter.invoke(TOOL_PROMPT) + run = await adapter.invoke(prompt) except Exception as e: error = e @@ -293,22 +294,26 @@ def build_context( ) -def responses_api_cases() -> list[Any]: +def responses_api_model_providers() -> list[tuple[str, ModelProviderName]]: """Every built-in (model, provider) pair Kiln routes through /v1/responses.""" - cases: list[Any] = [] - for model in built_in_models: - for provider in model.providers: - if not provider.openai_responses_api: - continue - cases.append( - pytest.param( - model.name, - provider.name, - True, - id=f"{model.name}_{provider.name.value}", - ) - ) - return cases + return [ + (model.name, provider.name) + for model in built_in_models + for provider in model.providers + if provider.openai_responses_api + ] + + +def responses_api_cases() -> list[Any]: + return [ + pytest.param( + model_name, + provider_name, + True, + id=f"{model_name}_{provider_name.value}", + ) + for model_name, provider_name in responses_api_model_providers() + ] # gpt_5_2 is the control: it predates the 5.4 restriction, so reasoning_effort + @@ -474,3 +479,51 @@ async def test_openai_responses_drops_custom_sampling_params(tmp_path, model_nam assert run is not None, f"No run produced.{ctx}" assert tool_called, f"The 'add' tool was never called.{ctx}" assert "4" in run.output.output, f"Final answer missing '4'.{ctx}" + + +# Forces text and a tool call in the same assistant turn, which is the shape the +# litellm bridge splits across several `Choices`. +NARRATION_PROMPT = ( + "First write one short sentence saying you will use the add tool, then call it " + "to add 2 and 2, then report the result." +) + +NARRATION_CASES = [ + pytest.param(model_name, provider_name, id=f"{model_name}_{provider_name.value}") + for model_name, provider_name in responses_api_model_providers() +] + + +@pytest.mark.paid +@pytest.mark.parametrize(("model_name", "provider_name"), NARRATION_CASES) +async def test_openai_responses_narration_then_tool_call( + tmp_path, model_name: str, provider_name: str +): + """Text plus a tool call in one turn must not lose the tool call. + + litellm's responses bridge returns one `Choices` per content part and a trailing + `Choices` carrying the tool calls. Reading only `choices[0]` drops the tool call: + the loop ends after one turn and the narration is saved as the final answer. + """ + skip_if_missing_provider_keys(provider_name) + + run, error, add_spy, request_log = await run_tool_loop( + tmp_path, + model_name=model_name, + provider_name=provider_name, + # A low effort keeps the narration turn cheap and still reasons enough to + # both narrate and call the tool. + thinking_level="low", + host_fragment="openai.com", + prompt=NARRATION_PROMPT, + ) + + tool_called = add_spy.run.called + ctx = build_context( + model_name, provider_name, "low", request_log, error, run, tool_called + ) + + assert error is None, f"Run raised: {first_error_line(error)}{ctx}" + assert run is not None, f"No run produced.{ctx}" + assert tool_called, f"The 'add' tool was never called.{ctx}" + assert "[4]" in run.output.output, f"Final answer missing '[4]'.{ctx}" From ffb9562fdacf01d50cd1335a1e07b7ec97a9d15f Mon Sep 17 00:00:00 2001 From: tawnymanticore Date: Wed, 9 Sep 2026 11:01:50 -0400 Subject: [PATCH 6/6] Send strict json_schema on the responses bridge so structured output stays constrained litellm's responses bridge reads `json_schema.get("strict", False)`, so an absent strict becomes an explicit false on the wire even though /v1/responses defaults it to true. Every flagged model was running structured output unconstrained; large models only passed the sweeps by following the schema out of habit. The schema is already built strict-compatible, so send strict, gated on openai_responses_api because json_schema mode is shared by hundreds of providers. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_014jrCq27Qa4VQeVEvYjSfTs --- .../model_adapters/litellm_adapter.py | 15 ++- .../model_adapters/test_litellm_adapter.py | 34 +++++++ .../test_openai_responses_routing_paid.py | 93 +++++++++++++++++++ 3 files changed, 138 insertions(+), 4 deletions(-) diff --git a/libs/core/kiln_ai/adapters/model_adapters/litellm_adapter.py b/libs/core/kiln_ai/adapters/model_adapters/litellm_adapter.py index 3239908973..26eaec8ac0 100644 --- a/libs/core/kiln_ai/adapters/model_adapters/litellm_adapter.py +++ b/libs/core/kiln_ai/adapters/model_adapters/litellm_adapter.py @@ -540,13 +540,20 @@ def json_schema_response_format(self) -> dict[str, Any]: # The valid ranges are still enforced by the prompt + post-hoc # validation, so this only affects the schema sent over the wire. output_schema = strip_numeric_bounds(output_schema) + json_schema: dict[str, Any] = { + "name": "task_response", + "schema": output_schema, + } + if self.model_provider().openai_responses_api: + # litellm's responses bridge turns an absent strict into an explicit + # false, and /v1/responses defaults it to true, so leaving it out runs + # structured output unconstrained. Only set it for the bridge: json_schema + # mode is shared by hundreds of providers, some of which reject the key. + json_schema["strict"] = True return { "response_format": { "type": "json_schema", - "json_schema": { - "name": "task_response", - "schema": output_schema, - }, + "json_schema": json_schema, } } diff --git a/libs/core/kiln_ai/adapters/model_adapters/test_litellm_adapter.py b/libs/core/kiln_ai/adapters/model_adapters/test_litellm_adapter.py index 8925632eeb..6a9cd36fb0 100644 --- a/libs/core/kiln_ai/adapters/model_adapters/test_litellm_adapter.py +++ b/libs/core/kiln_ai/adapters/model_adapters/test_litellm_adapter.py @@ -347,6 +347,40 @@ async def test_json_schema_response_format_adds_required_to_nested(config, tmp_p assert result_schema["properties"]["result"]["required"] == ["value", "unit"] +@pytest.mark.asyncio +@pytest.mark.parametrize("openai_responses_api", [True, False]) +async def test_json_schema_response_format_strict_for_responses_api( + config, mock_task, openai_responses_api +): + """litellm's responses bridge turns an absent strict into an explicit false, and + /v1/responses defaults it to true, so structured output would run unconstrained. + Other providers keep the existing shape: some reject an unknown strict key.""" + config.run_config_properties.structured_output_mode = ( + StructuredOutputMode.json_schema + ) + adapter = LiteLlmAdapter(config=config, kiln_task=mock_task) + + provider = KilnModelProvider( + name=ModelProviderName.openai, + model_id="gpt-6-astra", + openai_responses_api=openai_responses_api, + ) + + with ( + patch.object(adapter, "model_provider", return_value=provider), + patch.object(adapter, "has_structured_output", return_value=True), + ): + options = await adapter.response_format_options() + + json_schema = options["response_format"]["json_schema"] + assert json_schema["name"] == "task_response" + assert json_schema["schema"]["properties"]["test"] == {"type": "string"} + if openai_responses_api: + assert json_schema["strict"] is True + else: + assert "strict" not in json_schema + + @pytest.mark.parametrize( "provider_name,expected_prefix", [ diff --git a/libs/core/kiln_ai/adapters/model_adapters/test_openai_responses_routing_paid.py b/libs/core/kiln_ai/adapters/model_adapters/test_openai_responses_routing_paid.py index f1c9334030..4c25b55bbf 100644 --- a/libs/core/kiln_ai/adapters/model_adapters/test_openai_responses_routing_paid.py +++ b/libs/core/kiln_ai/adapters/model_adapters/test_openai_responses_routing_paid.py @@ -39,6 +39,7 @@ import httpx import pytest +from kiln_ai import datamodel from kiln_ai.adapters.adapter_registry import adapter_for_task from kiln_ai.adapters.ml_model_list import ModelName, built_in_models from kiln_ai.adapters.model_adapters.test_litellm_adapter_tools import build_test_task @@ -527,3 +528,95 @@ async def test_openai_responses_narration_then_tool_call( assert run is not None, f"No run produced.{ctx}" assert tool_called, f"The 'add' tool was never called.{ctx}" assert "[4]" in run.output.output, f"Final answer missing '[4]'.{ctx}" + + +# A minimal structured task: one required string field, which is enough for OpenAI to +# either honour the schema or not. +STRUCTURED_SCHEMA = { + "type": "object", + "properties": { + "answer": {"type": "string", "description": "The answer, spelled out in words"}, + }, + "required": ["answer"], +} + +STRUCTURED_PROMPT = "What is 2+2?" + + +def build_structured_task(tmp_path) -> datamodel.Task: + project = datamodel.Project(name="test", path=tmp_path / "test.kiln") + project.save_to_file() + task = datamodel.Task( + parent=project, + name="test task", + instruction="Answer the question. Spell the answer out in words.", + output_json_schema=json.dumps(STRUCTURED_SCHEMA), + ) + task.save_to_file() + return task + + +@pytest.mark.paid +@pytest.mark.parametrize(("model_name", "provider_name"), NARRATION_CASES) +async def test_openai_responses_structured_output_is_strict( + tmp_path, model_name: str, provider_name: str +): + """Structured output must stay schema-constrained on the responses bridge. + + litellm turns an absent `strict` into an explicit `strict: false`, and + /v1/responses defaults it to true, so omitting it silently runs structured + output unconstrained. + """ + skip_if_missing_provider_keys(provider_name) + + task = build_structured_task(tmp_path) + adapter = adapter_for_task( + task, + KilnAgentRunConfigProperties( + structured_output_mode=StructuredOutputMode.json_schema, + model_name=model_name, + model_provider_name=ModelProviderName(provider_name), + prompt_id="simple_prompt_builder", + thinking_level="low", + temperature=1.0, + top_p=1.0, + ), + ) + + run = None + error: Exception | None = None + with record_provider_requests("openai.com") as request_log: + try: + run = await adapter.invoke(STRUCTURED_PROMPT) + except Exception as e: + error = e + + formats = [ + ((r.body or {}).get("text") or {}).get("format") or {} + for r in request_log.requests + ] + ctx = ( + f"\ncase: model={model_name} provider={provider_name}" + f"\nendpoints: {request_log.paths}" + f"\ntext.format on the wire: {formats}" + f"\nfirst error line: {first_error_line(error)}\n" + ) + + assert error is None, f"Run raised: {first_error_line(error)}{ctx}" + assert run is not None, f"No run produced.{ctx}" + assert request_log.requests, f"No provider request was recorded at all.{ctx}" + + not_json_schema = [f for f in formats if f.get("type") != "json_schema"] + assert not not_json_schema, ( + f"Expected every request to send a json_schema format.{ctx}" + ) + + not_strict = [f for f in formats if f.get("strict") is not True] + assert not not_strict, ( + f"Expected every request to send strict=True, but " + f"{len(not_strict)} of {len(formats)} did not.{ctx}" + ) + + assert isinstance(json.loads(run.output.output), dict), ( + f"Output did not parse as a JSON object.{ctx}" + )