From 576a9e81ba22a108da38902a50999d63d312b7db Mon Sep 17 00:00:00 2001 From: Abhi Ojha Date: Tue, 4 Aug 2026 12:50:37 +0530 Subject: [PATCH 1/2] fix(backends): strict-ify OpenAI response schemas for OpenAI-compatible providers OpenAI structured outputs (and OpenAI-compatible proxies that terminate on the OpenAI platform, e.g. OpenRouter routing to an OpenAI model) require `additionalProperties: false` on every object and reject `$ref` entries, so schemas must be self-contained. Pydantic emits `$defs`/`$ref` for nested models, and the `@generative` wrapper references the result type via `$ref`, so any such call failed with a 400 regardless of server-type detection. Remove the `_ServerType.OPENAI` gate from the `response_format` block and apply the same inlined, fully-patched schema to the raw completions path (`structured_outputs`/`guided_json` for vLLM-style backends). Closes #1491 Assisted-by: Reasonix Signed-off-by: Abhi Ojha --- mellea/backends/openai.py | 106 ++++++++++----- test/backends/test_openai_unit.py | 212 +++++++++++++++++++++++++++++- 2 files changed, 283 insertions(+), 35 deletions(-) diff --git a/mellea/backends/openai.py b/mellea/backends/openai.py index af67cc608..a48878bab 100644 --- a/mellea/backends/openai.py +++ b/mellea/backends/openai.py @@ -56,6 +56,7 @@ from .backend import FormatterBackend from .model_options import ModelOption from .tools import ( + _recursively_inline_refs, add_tools_from_context_actions, add_tools_from_model_options, convert_tools_to_json, @@ -67,6 +68,48 @@ format: None = None # typing this variable in order to shadow the global format function and ensure mypy checks for errors +def _make_response_schema_openai_strict(schema: dict[str, Any]) -> dict[str, Any]: + """Make a JSON schema acceptable to OpenAI's structured outputs. + + OpenAI's strict mode (and OpenAI-compatible proxies that terminate on the + OpenAI platform, e.g. OpenRouter routing to an OpenAI model) requires + `additionalProperties: false` on every object and rejects `$ref` entries, + so schemas must be self-contained. Pydantic emits `$ref`/`$defs` for + nested models, so inline them and patch every object before sending. + Providers with less strict requirements ignore the extra property. + + Args: + schema: Pydantic JSON schema, e.g. from `model_json_schema()`. + + Returns: + A new schema with `$ref`s inlined and `additionalProperties: false` + set on every object, suitable for OpenAI `response_format`. + """ + defs = schema.get("$defs") + if defs is not None: + _recursively_inline_refs(schema, defs) + schema.pop("$defs", None) + + def _patch_object(obj: dict[str, Any]) -> None: + if obj.get("type") == "object": + obj["additionalProperties"] = False + props = obj.get("properties") + if isinstance(props, dict): + for prop_schema in props.values(): + if isinstance(prop_schema, dict): + _patch_object(prop_schema) + items = obj.get("items") + if isinstance(items, dict): + _patch_object(items) + for key in ("anyOf", "oneOf", "allOf"): + for branch in obj.get(key, []): + if isinstance(branch, dict): + _patch_object(branch) + + _patch_object(schema) + return schema + + class OpenAIBackend(FormatterBackend, AdapterMixin): """A generic OpenAI compatible backend. @@ -912,38 +955,24 @@ async def _generate_from_chat_context_standard( extra_params: dict[str, Any] = {} if _format is not None: - if self._server_type == _ServerType.OPENAI: - # The OpenAI platform requires that additionalProperties=False on all response_format schemas. - # However, not all schemas generates by Mellea include additionalProperties. - # GenerativeStub, in particular, does not add this property. - # The easiest way to address this disparity between OpenAI and other inference providers is to - # monkey-patch the response format exactly when we are actually using the OpenAI server. - # - # This only addresses the additionalProperties=False constraint. - # Other constraints we should be checking/patching are described here: - # https://platform.openai.com/docs/guides/structured-outputs?api-mode=chat - monkey_patched_response_schema = _format.model_json_schema() # type: ignore - monkey_patched_response_schema["additionalProperties"] = False - extra_params["response_format"] = { - "type": "json_schema", - "json_schema": { - "name": _format.__name__, - "schema": monkey_patched_response_schema, - "strict": True, - }, - } - else: - MelleaLogger.get_logger().info( - "Mellea assumes you are NOT using the OpenAI platform, and that other model providers have less strict requirements on supporting JSON schemas passed into `format=`. If you encounter a server-side error following this message, then you found an exception to this assumption. Please open an issue at github.com/generative_computing/mellea with this stack trace and your inference engine / model provider." - ) - extra_params["response_format"] = { - "type": "json_schema", - "json_schema": { - "name": _format.__name__, - "schema": _format.model_json_schema(), # type: ignore - "strict": True, - }, - } + # OpenAI's structured outputs (and OpenAI-compatible proxies that + # terminate on the OpenAI platform, e.g. OpenRouter routing to an + # OpenAI model) require strict schemas: additionalProperties=False + # on every object and no $ref entries. Pydantic emits $ref/$defs + # for nested models, so inline them and patch the schema before + # sending; providers with less strict requirements ignore the + # extra property. See #1491. + schema = _make_response_schema_openai_strict( + _format.model_json_schema() # type: ignore + ) + extra_params["response_format"] = { + "type": "json_schema", + "json_schema": { + "name": _format.__name__, + "schema": schema, + "strict": True, + }, + } # Append tool call information if applicable. tools: dict[str, AbstractMelleaTool] = dict() @@ -1243,10 +1272,19 @@ async def _generate_from_raw( # Some versions (like vllm's version) of the OpenAI API support structured decoding for completions requests. # It's dependent on the vllm version though. We check at backend init. + # The same strict-schema treatment as response_format applies here: OpenAI-style + # structured decoding requires additionalProperties=false and self-contained + # schemas. See #1491. if self._use_structured_output_for_raw: - extra_body["structured_outputs"] = {"json": format.model_json_schema()} # type: ignore + extra_body["structured_outputs"] = { + "json": _make_response_schema_openai_strict( + format.model_json_schema() + ) # type: ignore + } else: - extra_body["guided_json"] = format.model_json_schema() # type: ignore + extra_body["guided_json"] = _make_response_schema_openai_strict( # type: ignore + format.model_json_schema() # type: ignore + ) if tool_calls: MelleaLogger.get_logger().warning( "The completion endpoint does not support tool calling at the moment." diff --git a/test/backends/test_openai_unit.py b/test/backends/test_openai_unit.py index a40db14c5..ca90af298 100644 --- a/test/backends/test_openai_unit.py +++ b/test/backends/test_openai_unit.py @@ -7,6 +7,7 @@ _simplify_and_merge, and _make_backend_specific_and_remove. """ +import asyncio from unittest.mock import AsyncMock, MagicMock, PropertyMock, patch import pytest @@ -14,9 +15,10 @@ from openai.types.chat import ChatCompletion, ChatCompletionChunk, ChatCompletionMessage from openai.types.chat.chat_completion import Choice from openai.types.completion_choice import CompletionChoice +from pydantic import BaseModel from mellea.backends import ModelOption -from mellea.backends.openai import OpenAIBackend +from mellea.backends.openai import OpenAIBackend, _make_response_schema_openai_strict from mellea.core.base import ModelOutputThunk @@ -429,5 +431,213 @@ class Answer(pydantic.BaseModel): assert "guided_json" in extra_body or "structured_outputs" in extra_body +# --- _make_response_schema_openai_strict --- + + +class _NestedProfile(BaseModel): + """A nested Pydantic model used in response-schema tests.""" + + name: str + age: int + + +class _ExtractUserResponse(BaseModel): + """Wrapper that forces Pydantic to emit a $ref for the nested model.""" + + result: _NestedProfile + + +def test_make_response_schema_openai_strict_inlines_refs_and_patches_all_objects(): + """$refs are inlined and every object gets additionalProperties: false.""" + schema = _make_response_schema_openai_strict( + _ExtractUserResponse.model_json_schema() + ) + + # No $ref / $defs may remain: OpenAI strict mode rejects both. + assert "$defs" not in schema + result = schema["properties"]["result"] + assert "$ref" not in result + + assert schema["additionalProperties"] is False + assert result["type"] == "object" + assert result["additionalProperties"] is False + for prop in result["properties"].values(): + # Leaf scalar properties are not objects, so they stay untouched. + assert "additionalProperties" not in prop + + +class _ExtractUserListResponse(BaseModel): + """Wrapper with a list-of-model field to exercise items recursion.""" + + users: list[_NestedProfile] + + +def test_make_response_schema_openai_strict_patches_list_items(): + """Objects inside array items are patched too.""" + schema = _make_response_schema_openai_strict( + _ExtractUserListResponse.model_json_schema() + ) + + item_schema = schema["properties"]["users"]["items"] + assert item_schema["type"] == "object" + assert item_schema["additionalProperties"] is False + + +def test_make_response_schema_openai_strict_patches_anyof_branches(): + """$refs inside anyOf branches are inlined and object branches patched.""" + schema = { + "type": "object", + "properties": { + "result": {"anyOf": [{"$ref": "#/$defs/UserProfile"}, {"type": "null"}]} + }, + "$defs": { + "UserProfile": { + "type": "object", + "properties": {"name": {"type": "string"}}, + "required": ["name"], + } + }, + } + + out = _make_response_schema_openai_strict(schema) + result = out["properties"]["result"] + assert "$ref" not in result + obj_branch = next(b for b in result["anyOf"] if b.get("type") == "object") + assert obj_branch["additionalProperties"] is False + # The null branch is not an object, so it stays untouched. + null_branch = next(b for b in result["anyOf"] if b.get("type") == "null") + assert "additionalProperties" not in null_branch + + +# --- Payload tests: what actually reaches the provider (#1491) --- + + +def _schema_from_chat_payload(payload: dict) -> dict: + """Pull the response_format schema out of the chat completions call kwargs.""" + return payload["response_format"]["json_schema"]["schema"] + + +async def test_generate_from_chat_sends_inlined_strict_response_format(backend): + """The chat path sends an inlined schema with additionalProperties everywhere. + + Regression test for #1491: the payload handed to an OpenAI-compatible + provider must have no $ref / $defs and set additionalProperties: false on + every object, regardless of server type. + """ + from mellea.core.base import CBlock + from mellea.stdlib.context import ChatContext + + mock_create = AsyncMock() + mock_client = MagicMock() + mock_client.chat.completions.create = mock_create + + with ( + patch.object( + OpenAIBackend, + "_async_client", + new_callable=PropertyMock, + return_value=mock_client, + ), + patch("mellea.backends.openai.send_to_queue", new=AsyncMock()), + ): + await backend._generate_from_chat_context_standard( + CBlock(value="User log 42: Alice is 31 years old."), + ChatContext(), + _format=_ExtractUserResponse, + ) + + schema = _schema_from_chat_payload(mock_create.call_args.kwargs) + assert "$defs" not in schema + assert "$ref" not in schema["properties"]["result"] + assert schema["additionalProperties"] is False + assert schema["properties"]["result"]["additionalProperties"] is False + + # Let the background send_to_queue task settle so nothing is left pending. + await asyncio.sleep(0) + + +async def test_generate_from_raw_sends_strict_guided_schema(backend): + """The raw completions path patches guided_json/structured_outputs too. + + vLLM-style structured decoding on the completions endpoint gets the same + strict-schema treatment as the chat response_format (see #1491). + """ + from openai.types import Completion + from openai.types.completion_choice import CompletionChoice + + from mellea.core.base import CBlock + from mellea.stdlib.context import ChatContext + + mock_create = AsyncMock( + return_value=Completion( + id="raw-test", + created=0, + model="fake", + object="text_completion", + choices=[CompletionChoice(index=0, finish_reason="stop", text="ok")], + ) + ) + mock_client = MagicMock() + mock_client.completions.create = mock_create + + with patch.object( + OpenAIBackend, + "_async_client", + new_callable=PropertyMock, + return_value=mock_client, + ): + await backend._generate_from_raw( + [CBlock(value="what is 1+1?")], ChatContext(), format=_ExtractUserResponse + ) + + extra_body = mock_create.call_args.kwargs["extra_body"] + if "structured_outputs" in extra_body: + schema = extra_body["structured_outputs"]["json"] + else: + schema = extra_body["guided_json"] + assert "$defs" not in schema + assert "$ref" not in schema["properties"]["result"] + assert schema["additionalProperties"] is False + assert schema["properties"]["result"]["additionalProperties"] is False + + +async def test_generate_from_raw_sends_strict_structured_outputs(backend): + """vLLM-style structured_outputs on the raw path get the strict schema too.""" + from openai.types import Completion + from openai.types.completion_choice import CompletionChoice + + from mellea.core.base import CBlock + from mellea.stdlib.context import ChatContext + + mock_create = AsyncMock( + return_value=Completion( + id="raw-test", + created=0, + model="fake", + object="text_completion", + choices=[CompletionChoice(index=0, finish_reason="stop", text="ok")], + ) + ) + mock_client = MagicMock() + mock_client.completions.create = mock_create + + backend._use_structured_output_for_raw = True # vLLM-style server probe result + with patch.object( + OpenAIBackend, + "_async_client", + new_callable=PropertyMock, + return_value=mock_client, + ): + await backend._generate_from_raw( + [CBlock(value="what is 1+1?")], ChatContext(), format=_ExtractUserResponse + ) + + schema = mock_create.call_args.kwargs["extra_body"]["structured_outputs"]["json"] + assert "$defs" not in schema + assert "$ref" not in schema["properties"]["result"] + assert schema["additionalProperties"] is False + assert schema["properties"]["result"]["additionalProperties"] is False + + if __name__ == "__main__": pytest.main([__file__, "-v"]) From 3ceb30c2bf74d7936ce30d1f5bf017d786f98e62 Mon Sep 17 00:00:00 2001 From: Abhi Ojha Date: Thu, 6 Aug 2026 21:12:20 +0530 Subject: [PATCH 2/2] fix(backends): preserve additionalProperties value schemas for map fields Address review feedback on #1493: when pydantic emits a schema dict as `additionalProperties` (e.g. `dict[str, Model]`), recurse into it instead of overwriting it with `False`, which would silently drop the value type. Add the reviewer-suggested regression tests for `dict[str, Model]` and `dict[str, int]`. Assisted-by: Reasonix Signed-off-by: Abhi Ojha --- mellea/backends/openai.py | 8 ++++- test/backends/test_openai_unit.py | 53 +++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 1 deletion(-) diff --git a/mellea/backends/openai.py b/mellea/backends/openai.py index a48878bab..02eb1bef4 100644 --- a/mellea/backends/openai.py +++ b/mellea/backends/openai.py @@ -91,8 +91,14 @@ def _make_response_schema_openai_strict(schema: dict[str, Any]) -> dict[str, Any schema.pop("$defs", None) def _patch_object(obj: dict[str, Any]) -> None: - if obj.get("type") == "object": + add_props = obj.get("additionalProperties") + if obj.get("type") == "object" and not isinstance(add_props, dict): + # Only assert a closed object when the value schema isn't itself a + # constraint (e.g. dict[str, Model] emits additionalProperties as a + # schema — overwriting it with False would drop the value type). obj["additionalProperties"] = False + elif isinstance(add_props, dict): + _patch_object(add_props) props = obj.get("properties") if isinstance(props, dict): for prop_schema in props.values(): diff --git a/test/backends/test_openai_unit.py b/test/backends/test_openai_unit.py index ca90af298..a7467f522 100644 --- a/test/backends/test_openai_unit.py +++ b/test/backends/test_openai_unit.py @@ -472,6 +472,16 @@ class _ExtractUserListResponse(BaseModel): users: list[_NestedProfile] +class _ExtractUserMapResponse(BaseModel): + """Wrapper with a dict-of-model field. + + Pydantic emits the value type as `additionalProperties: {schema}` (not + `false`) on the map object, which is the case that must not be clobbered. + """ + + users: dict[str, _NestedProfile] + + def test_make_response_schema_openai_strict_patches_list_items(): """Objects inside array items are patched too.""" schema = _make_response_schema_openai_strict( @@ -509,6 +519,49 @@ def test_make_response_schema_openai_strict_patches_anyof_branches(): assert "additionalProperties" not in null_branch +def test_make_response_schema_openai_strict_patches_freeform_dict_scalar_values(): + """A `dict[str, scalar]` value-type schema (e.g. int) is preserved too.""" + + class _Counts(BaseModel): + counts: dict[str, int] + + schema = _make_response_schema_openai_strict(_Counts.model_json_schema()) + + counts = schema["properties"]["counts"] + assert counts["type"] == "object" + # Scalar value type stays intact instead of being turned into False. + assert counts["additionalProperties"] == {"type": "integer"} + + +def test_make_response_schema_openai_strict_preserves_dict_value_schema(): + """A `dict[str, Model]` value-type schema is preserved, not overwritten. + + Regression guard for the `additionalProperties`-as-schema case: pydantic + emits `additionalProperties: {}` for `dict[str, Model]` + fields. Overwriting that with `False` would silently drop the value type + (turning "string keys -> Model" into "no extra properties allowed"), so + the patcher must recurse into the value schema instead of clobbering it. + """ + schema = _make_response_schema_openai_strict( + _ExtractUserMapResponse.model_json_schema() + ) + + users = schema["properties"]["users"] + assert users["type"] == "object" + + # The value-type schema must survive as a dict, NOT be replaced by False. + value_schema = users["additionalProperties"] + assert isinstance(value_schema, dict), ( + "dict[str, Model] value type was clobbered by additionalProperties=False" + ) + + # The nested model reachable through the map is inlined and closed. + assert "$ref" not in value_schema + assert value_schema["type"] == "object" + assert value_schema["additionalProperties"] is False + assert set(value_schema["properties"]) == {"name", "age"} + + # --- Payload tests: what actually reaches the provider (#1491) ---