diff --git a/cli/serve/app.py b/cli/serve/app.py index aa40b4a27..434297b6b 100644 --- a/cli/serve/app.py +++ b/cli/serve/app.py @@ -162,11 +162,25 @@ def _build_model_options(request: ChatCompletionRequest) -> dict: return ModelOption.replace_keys(filtered_options, openai_to_model_option) +def _build_client_options(request: ChatCompletionRequest) -> dict: + """Return the full raw client request as a plain dict. + + Passed to serve() as client_options when the function declares that + parameter, giving it access to every field the client sent (including + routing and metadata fields like model, user, and n) while ensuring those + specific named routing fields are excluded from model_options. Note that + arbitrary extra fields are forwarded to model_options as potential + backend-specific generation parameters. + """ + return request.model_dump(exclude_none=True) + + def make_chat_endpoint(module): """Makes a chat endpoint using a custom module.""" # Inspect serve function once at endpoint creation time serve_sig = inspect.signature(module.serve) accepts_format = "format" in serve_sig.parameters + accepts_client_options = "client_options" in serve_sig.parameters is_async = inspect.iscoroutinefunction(module.serve) async def endpoint(request: ChatCompletionRequest): @@ -221,6 +235,8 @@ async def endpoint(request: ChatCompletionRequest): } if accepts_format: serve_kwargs["format"] = format_model + if accepts_client_options: + serve_kwargs["client_options"] = _build_client_options(request) # Detect if serve is async or sync and handle accordingly if is_async: diff --git a/docs/examples/m_serve/README.md b/docs/examples/m_serve/README.md index 48c5f9e81..4d739fccd 100644 --- a/docs/examples/m_serve/README.md +++ b/docs/examples/m_serve/README.md @@ -15,6 +15,7 @@ Each subdirectory contains a server implementation and its matching client(s): | `multimodal-image/` | Vision model serving with image inputs | | `multimodal-audio/` | Audio-text-to-text serving (llama-server and Ollama/Granite variants) | | `pii/` | PII detection service | +| `model-routing/` | Using or ignoring the client-supplied model ID | ## Files @@ -74,6 +75,20 @@ OpenAI-compatible `input_audio` content part request. ### pii/pii_serve.py Example of serving a PII (Personally Identifiable Information) detection service. +### model-routing/m_serve_example_model_routing.py +Example showing how to use `client_options` to route on the client-supplied `model` field. + +**Key Concepts:** +- The `model` field in an OpenAI-compatible request is routing/metadata: `m serve` echoes + it back in the response but does **not** include it in `model_options`. +- Declare `client_options` in `serve()` and `m serve` passes the full raw client request + as a dict, giving access to `model` and every other field without them leaking into + `model_options` to be used by the backend. +- To ignore the client model ID entirely, omit `client_options` (see the `simple/` examples). + +### model-routing/client_model_routing.py +Client code demonstrating routing via the standard `model` field and fallback to the default backend. + ### simple/client.py Client code for testing the served API endpoints with non-streaming requests. @@ -104,6 +119,7 @@ Client code demonstrating streaming responses combined with tool calling. - **Structured Output**: Using `response_format` for JSON schema validation - **Multimodal Inputs**: Sending text plus image content to vision-capable models - **Audio-Text-to-Text**: Sending base64-encoded audio alongside text in a chat request; the model responds in text (not transcription) +- **Model Routing**: Reading the client `model` field via `client_options` to route to an allowlisted backend ## Basic Pattern @@ -215,6 +231,16 @@ uv run python docs/examples/m_serve/tool-calling/client_tool_calling.py uv run python docs/examples/m_serve/tool-calling/client_streaming_tool_calling.py ``` +### Model Routing + +```bash +# Start the model-routing example server +uv run m serve docs/examples/m_serve/model-routing/m_serve_example_model_routing.py + +# In another terminal, run the client +uv run python docs/examples/m_serve/model-routing/client_model_routing.py +``` + ## Response Format Support The server supports structured output via the `response_format` parameter, which allows you to control the format of the model's response. This is compatible with OpenAI's response format API. diff --git a/docs/examples/m_serve/model-routing/client_model_routing.py b/docs/examples/m_serve/model-routing/client_model_routing.py new file mode 100644 index 000000000..78f3af868 --- /dev/null +++ b/docs/examples/m_serve/model-routing/client_model_routing.py @@ -0,0 +1,35 @@ +# pytest: skip_always + +"""Client demonstrating how to interact with the model-routing m serve example. + +Pattern A: use the standard `model` field to select the backend via client_options. +Pattern B: unknown model falls back to the default backend. + +The allowlist and default behavior is an implementation decision in the example +Mellea program being served. It could easily be changed to ignore the requested +model or to throw an error instead of having a default model. + +Run the server first: + uv run m serve docs/examples/m_serve/model-routing/m_serve_example_model_routing.py +""" + +import openai + +PORT = 8080 +client = openai.OpenAI(api_key="na", base_url=f"http://0.0.0.0:{PORT}/v1") + +print("=== Pattern A: standard model field routes via client_options ===") +# The standard `model` field is read by the server via +# client_options and used to select the backend. +response_a = client.chat.completions.create( + model="granite4.1:8b", messages=[{"role": "user", "content": "What is 2 + 2?"}] +) +print(f"model echoed back : {response_a.model}") +print(f"response : {response_a.choices[0].message.content}\n") + +print("=== Pattern B: unknown model falls back to default ===") +response_b = client.chat.completions.create( + model="some-unknown-model", messages=[{"role": "user", "content": "What is 2 + 2?"}] +) +print(f"model echoed back : {response_b.model}") +print(f"response : {response_b.choices[0].message.content}") diff --git a/docs/examples/m_serve/model-routing/m_serve_example_model_routing.py b/docs/examples/m_serve/model-routing/m_serve_example_model_routing.py new file mode 100644 index 000000000..ef866eac6 --- /dev/null +++ b/docs/examples/m_serve/model-routing/m_serve_example_model_routing.py @@ -0,0 +1,73 @@ +# pytest: ollama, e2e + +"""Example showing how to use client_options to route on the client model ID. + +In an OpenAI-compatible request the client sends a `model` string, e.g.: + + client.chat.completions.create(model="granite4.1:8b", messages=[...]) + +That string is **routing / metadata** from the server's perspective: `m serve` +echoes it back in the response but does NOT include it in `model_options` (which +is filtered for backend consumption). + +Declare `client_options` in `serve()` and `m serve` passes the full raw client +request as a dict, giving access to `model` and every other field the client +sent — without any of those values leaking into `model_options`. + +To ignore the client model ID entirely and always use a fixed backend, simply +omit the `client_options` parameter (see the simple/ examples). + +Run the server: + m serve docs/examples/m_serve/model-routing/m_serve_example_model_routing.py + +Test with the client: + python docs/examples/m_serve/model-routing/client_model_routing.py +""" + +from typing import Any + +import mellea +from mellea.backends.model_ids import IBM_GRANITE_4_1_3B, IBM_GRANITE_4_1_8B +from mellea.core import ModelOutputThunk +from mellea.serve import ChatMessage + +_DEFAULT_MODEL = IBM_GRANITE_4_1_3B + +_ALLOWED_MODELS: dict[str, Any] = { + IBM_GRANITE_4_1_3B.ollama_name: IBM_GRANITE_4_1_3B, # type: ignore[dict-item] + IBM_GRANITE_4_1_8B.ollama_name: IBM_GRANITE_4_1_8B, # type: ignore[dict-item] +} + + +def serve( + input: list[ChatMessage], + requirements: list[str] | None = None, + model_options: dict[str, Any] | None = None, + client_options: dict[str, Any] | None = None, +) -> ModelOutputThunk: + """Serve with backend selected from the standard client `model` field. + + Reads `client_options["model"]` (the standard OpenAI `model` field) and + routes to an allowlisted Ollama backend. Falls back to `granite4.1:3b` + when the value is unrecognised. `model_options` is clean — it contains + only backend generation parameters, never routing metadata. + + Args: + input: Chat messages from the client. + requirements: Optional requirement strings forwarded from the client. + model_options: Generation parameters filtered for backend consumption. + client_options: Full raw client request fields, including `model`. + + Returns: + ModelOutputThunk with the generated response. + """ + model_name = (client_options or {}).get("model") + chosen_model = _ALLOWED_MODELS.get(model_name, _DEFAULT_MODEL) # type: ignore[arg-type] + + message = input[-1].get_text_content() or "No message provided" + session = mellea.start_session(model_id=chosen_model) + return session.instruct( + description=message, + requirements=requirements, # type: ignore[arg-type] + model_options=model_options, + ) diff --git a/test/cli/test_build_model_options.py b/test/cli/test_build_model_options.py index 18358246a..a7cddcabc 100644 --- a/test/cli/test_build_model_options.py +++ b/test/cli/test_build_model_options.py @@ -1,9 +1,9 @@ # Copyright IBM Corp. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -"""Unit tests for _build_model_options function.""" +"""Unit tests for _build_model_options and _build_client_options functions.""" -from cli.serve.app import _build_model_options +from cli.serve.app import _build_client_options, _build_model_options from cli.serve.models import ChatCompletionRequest, ChatMessage from mellea.backends.model_options import ModelOption @@ -109,3 +109,79 @@ def test_requirements_excluded(self): options = _build_model_options(request) assert "requirements" not in options assert ModelOption.TEMPERATURE in options + + +class TestBuildClientOptions: + """Unit tests for _build_client_options.""" + + def test_includes_model_field(self): + """model is present in client_options (it is excluded from model_options).""" + request = ChatCompletionRequest( + model="granite4.1:8b", messages=[ChatMessage(role="user", content="hi")] + ) + options = _build_client_options(request) + assert options["model"] == "granite4.1:8b" + + def test_includes_user_field(self): + """user tracking field is present in client_options.""" + request = ChatCompletionRequest( + model="test-model", + messages=[ChatMessage(role="user", content="hi")], + user="user-123", + ) + options = _build_client_options(request) + assert options["user"] == "user-123" + + def test_includes_generation_params(self): + """Generation params (temperature, max_tokens) are also present.""" + request = ChatCompletionRequest( + model="test-model", + messages=[ChatMessage(role="user", content="hi")], + temperature=0.5, + max_tokens=200, + ) + options = _build_client_options(request) + assert options["temperature"] == 0.5 + assert options["max_tokens"] == 200 + + def test_excludes_none_values(self): + """None fields are excluded (exclude_none=True).""" + request = ChatCompletionRequest( + model="test-model", + messages=[ChatMessage(role="user", content="hi")], + max_tokens=None, + user=None, + ) + options = _build_client_options(request) + assert "max_tokens" not in options + assert "user" not in options + + def test_model_absent_from_model_options_but_present_in_client_options(self): + """model is stripped from model_options but preserved in client_options.""" + request = ChatCompletionRequest( + model="granite4.1:8b", + messages=[ChatMessage(role="user", content="hi")], + temperature=0.7, + ) + model_opts = _build_model_options(request) + client_opts = _build_client_options(request) + assert "model" not in model_opts + assert client_opts["model"] == "granite4.1:8b" + + def test_extra_fields_allowed_and_passed_through(self): + """Extra fields not defined on the schema are allowed and passed through.""" + request = ChatCompletionRequest( + model="test-model", + messages=[ChatMessage(role="user", content="hi")], + custom_backend_param="some-value", + ) + # Extra fields are allowed in ChatCompletionRequest + assert getattr(request, "custom_backend_param", None) == "some-value" + + # Extra fields are included in client_options + client_opts = _build_client_options(request) + assert client_opts.get("custom_backend_param") == "some-value" + + # Extra fields are also forwarded as potential model_options + model_opts = _build_model_options(request) + assert model_opts.get("custom_backend_param") == "some-value" diff --git a/test/cli/test_serve.py b/test/cli/test_serve.py index 42b3cdf54..0d68085dc 100644 --- a/test/cli/test_serve.py +++ b/test/cli/test_serve.py @@ -333,6 +333,126 @@ async def test_n_less_than_1_rejected_by_pydantic(self, mock_module): assert errors[0]["loc"] == ("n",) assert errors[0]["type"] == "greater_than_equal" + @pytest.mark.asyncio + async def test_client_options_not_passed_when_not_declared( + self, mock_module, sample_request + ): + """serve() without client_options param is not called with it.""" + mock_output = ModelOutputThunk("Test response") + mock_module.serve.return_value = mock_output + + endpoint = make_chat_endpoint(mock_module) + await endpoint(sample_request) + + call_kwargs = mock_module.serve.call_args.kwargs + assert "client_options" not in call_kwargs + + @pytest.mark.asyncio + async def test_client_options_passed_when_declared( + self, mock_module, sample_request + ): + """serve() declaring client_options receives the full raw request dict.""" + mock_output = ModelOutputThunk("Test response") + received: dict = {} + + def serve_with_client_options( + input, requirements=None, model_options=None, client_options=None + ): + if client_options is not None: + received.update(client_options) + return mock_output + + mock_module.serve = serve_with_client_options + + endpoint = make_chat_endpoint(mock_module) + response = await endpoint(sample_request) + + assert isinstance(response, ChatCompletion) + assert "model" in received + + @pytest.mark.asyncio + async def test_client_options_passed_when_declared_async( + self, mock_module, sample_request + ): + """serve() is async and declares client_options; receives full raw request dict.""" + mock_output = ModelOutputThunk("Test response") + received: dict = {} + + async def serve_with_client_options_async( + input, requirements=None, model_options=None, client_options=None + ): + if client_options is not None: + received.update(client_options) + return mock_output + + mock_module.serve = serve_with_client_options_async + + endpoint = make_chat_endpoint(mock_module) + response = await endpoint(sample_request) + + assert isinstance(response, ChatCompletion) + assert "model" in received + + @pytest.mark.asyncio + async def test_client_options_contains_model(self, mock_module): + """client_options includes the model field that is absent from model_options.""" + captured_client_options: dict = {} + + mock_output = ModelOutputThunk("Test response") + + def serve_capturing( + input, requirements=None, model_options=None, client_options=None + ): + if client_options is not None: + captured_client_options.update(client_options) + return mock_output + + mock_module.serve = serve_capturing + + request = ChatCompletionRequest( + model="granite4.1:8b", + messages=[ChatMessage(role="user", content="Hello")], + temperature=0.5, + user="alice", + ) + + endpoint = make_chat_endpoint(mock_module) + await endpoint(request) + + assert captured_client_options["model"] == "granite4.1:8b" + assert captured_client_options["user"] == "alice" + assert captured_client_options["temperature"] == 0.5 + + @pytest.mark.asyncio + async def test_client_options_does_not_affect_model_options(self, mock_module): + """model_options stays clean — model/user are absent even when client_options is used.""" + captured_model_options: dict = {} + + mock_output = ModelOutputThunk("Test response") + + def serve_capturing( + input, requirements=None, model_options=None, client_options=None + ): + if model_options is not None: + captured_model_options.update(model_options) + return mock_output + + mock_module.serve = serve_capturing + + request = ChatCompletionRequest( + model="granite4.1:8b", + messages=[ChatMessage(role="user", content="Hello")], + temperature=0.5, + user="alice", + ) + + endpoint = make_chat_endpoint(mock_module) + await endpoint(request) + + assert "model" not in captured_model_options + assert "user" not in captured_model_options + assert ModelOption.TEMPERATURE in captured_model_options + class TestHTTPValidation: """Tests for HTTP-level validation via FastAPI TestClient."""