feature/anthropic messages api

backends/exllamav3/model.py

@@ -846,7 +846,9 @@ def validate_context_length(
         prompt: str,
         params: BaseSamplerRequest,
         mm_embeddings: Optional[MultimodalEmbeddingWrapper] = None,
-    ):
+    ) -> int:
+        """Validate the prompt against the context limit and return its length."""
+
         context_len = len(
             self.encode_tokens(
                 prompt,
@@ -869,6 +871,8 @@ def validate_context_length(
             allocation_boundary,
         )
 
+        return context_len
+
     async def generate(
         self,
         request_id: str,

common/config_models.py

@@ -66,10 +66,11 @@ class NetworkConfig(BaseConfigModel):
             "NOTE: Only enable this for debug purposes."
         ),
     )
-    api_servers: Optional[List[Literal["oai", "kobold"]]] = Field(
+    api_servers: Optional[List[Literal["oai", "kobold", "anthropic"]]] = Field(
         ["OAI"],
         description=(
-            'Select API servers to enable (default: ["OAI"]).\nPossible values: OAI, Kobold.'
+            'Select API servers to enable (default: ["OAI"]).\n'
+            "Possible values: OAI, Kobold, Anthropic."
         ),
     )
     sse_ping_interval: Optional[int] = Field(

common/model.py

@@ -309,15 +309,24 @@ def check_context_length(
     prompts: str | list[str],
     params: BaseSamplerRequest,
     mm_embeddings: Optional[MultimodalEmbeddingWrapper] = None,
-):
-    """Reject oversized prompts before a streaming response commits HTTP 200."""
+) -> int:
+    """
+    Reject oversized prompts before a streaming response commits HTTP 200.
+
+    Returns the longest prompt's length in tokens, so a caller that needs the
+    prompt token count up front doesn't have to tokenize a second time.
+    """
 
     if isinstance(prompts, str):
         prompts = [prompts]
 
+    lengths = []
+
     try:
         for prompt in prompts:
-            container.validate_context_length(prompt, params, mm_embeddings)
+            lengths.append(container.validate_context_length(prompt, params, mm_embeddings))
     except ContextLengthExceededError as exc:
         error_message = handle_request_error(str(exc), exc_info=False).error.message
         raise ContextLengthHTTPException(error_message) from exc
+
+    return max(lengths, default=0)

config_sample.yml

@@ -28,7 +28,7 @@ network:
   send_tracebacks: false
 
   # Select API servers to enable (default: ["OAI"]).
-  # Possible values: OAI, Kobold.
+  # Possible values: OAI, Kobold, Anthropic.
   api_servers: ["OAI"]
 
   # Seconds between SSE keep-alive pings on streaming responses (default: 15).

docs/02.-Server-options.md

@@ -21,7 +21,7 @@ All of these options have descriptive comments above them. You should not need t
 | disable_auth           | Bool (False)           | Disables API authentication                                                                                             |
 | disable_fetch_requests | Bool (False)           | Disables fetching external content when responding to requests (ex. fetching images from URLs)                          |
 | send_tracebacks        | Bool (False)           | Send server tracebacks to client.<br><br>Note: It's not recommended to enable this if sharing the instance with others. |
-| api_servers            | List[String] (["OAI"]) | API servers to enable. Possible values `"OAI", "Kobold"`                                                                |
+| api_servers            | List[String] (["OAI"]) | API servers to enable. Possible values `"OAI", "Kobold", "Anthropic"`                                                   |
 | sse_ping_interval      | Int (15)               | Interval in seconds between SSE keep-alive pings on streaming responses. Set to 0 to disable pings.                     |
 
 ### Logging Options

docs/03.-Usage.md

@@ -4,7 +4,7 @@ TabbyAPI's main use-case is to be an API server for running ExllamaV3 models.
 
 ### API Server
 
-Currently TabbyAPI supports clients that use the [OpenAI](https://platform.openai.com/docs/api-reference) standard and [KoboldAI](https://lite.koboldai.net/koboldcpp_api)'s API.
+Currently TabbyAPI supports clients that use the [OpenAI](https://platform.openai.com/docs/api-reference) standard, [KoboldAI](https://lite.koboldai.net/koboldcpp_api)'s API, and [Anthropic](https://docs.claude.com/en/api/messages)'s Messages API.
 
 In addition, there are expanded parameters to generation endpoints along with administrative endpoints for loading, unloading, loras, sampling overrides, etc.
 
@@ -26,6 +26,120 @@ curl http://localhost:5000/v1/completions \
 }'
 ```
 
+### Anthropic API
+
+The Anthropic Messages API is off by default. Enable it in `config.yml`:
+
+```yml
+network:
+  api_servers: ["OAI", "Anthropic"]
+```
+
+This exposes two endpoints:
+- `/v1/messages`: Generates a message from a conversation.
+- `/v1/messages/count_tokens`: Counts the tokens an equivalent request would consume.
+
+Requests are translated into a chat completion internally, so the loaded model's
+prompt template, reasoning tags, and sampler settings apply exactly as they do on
+`/v1/chat/completions`. A prompt template is required.
+
+Anthropic clients authenticate with an `x-api-key` header, which TabbyAPI already
+accepts, so pointing an Anthropic SDK at the server only takes a base URL change.
+Errors are returned in the Anthropic envelope rather than TabbyAPI's usual one.
+
+Streaming is supported and emits the Anthropic event sequence (`message_start`,
+the `content_block_*` lifecycle, `message_delta`, `message_stop`). Reasoning
+models stream a `thinking` block ahead of the `text` block. There is no `[DONE]`
+sentinel; `message_stop` ends the stream.
+
+Tool use is supported and works with every tool call format the model's
+`tool_format` setting supports, since tools are translated into the same shape
+`/v1/chat/completions` uses. `tool_choice` maps as `auto`/`none` unchanged, `any`
+onto "required", and `{"type": "tool", "name": ...}` onto a forced call.
+Because tool calls are parsed once generation finishes, a streamed `tool_use`
+block arrives as one `input_json_delta` carrying the whole argument object
+rather than incrementally.
+
+A `system` role is accepted inside `messages`, which is how recent Anthropic
+models take operator instructions mid-conversation and how Claude Code sends
+them. Chat templates generally only allow a system turn in first position, so a
+later one is carried in a user turn wrapped in `<system-reminder>` rather than
+being folded into the leading system prompt, which would rewrite the front of
+the prefix and invalidate the prompt cache on every turn.
+
+Two details are worth knowing. A `tool_result` marked `is_error` has its text
+prefixed with `Error: `, because chat templates have no concept of a failed call
+and the model can only react to the failure if it can read it. And Anthropic
+packs every tool result for a turn into a single user message, which is fanned
+out into one `tool` message per result, since that is what chat templates
+expect.
+
+Images are supported on vision models, from either a `base64` or a `url` source.
+Sending one to a model without vision is rejected rather than dropped, since the
+model would otherwise be asked about a picture it was never shown. Images are
+also accepted inside a `tool_result`, which is how screenshot-returning tools
+report back.
+
+`GET /v1/models` serves the Anthropic field names (`type`, `display_name`,
+`created_at`) alongside the OpenAI ones, so an Anthropic SDK can list models
+without a separate endpoint. The loaded model's card reports its context window
+as `max_input_tokens`, the name the Anthropic model schema uses, with
+`max_tokens` beside it; both come from `max_seq_len`, which is also served under
+its own name inside `parameters` for OpenAI-side clients. TabbyAPI has no output
+limit separate from the context, so the two are equal. The prompt template
+content is left out of a listing because of its size, so fetch `/v1/model` for
+that.
+
+Support is currently partial. These are not implemented yet and are rejected with
+a descriptive error rather than being silently dropped:
+- Documents (PDF input)
+- Image `file` sources, which reference the Anthropic Files API
+- Anthropic's server-side tools (web search, code execution)
+
+Fields with no local equivalent are accepted and ignored: `cache_control`,
+`service_tier`, `container`, `mcp_servers`, and `thinking.budget_tokens`. The
+`thinking` type maps onto the model's `enable_thinking` template variable.
+
+As a TabbyAPI extension, a request may carry `template_vars` (alias
+`chat_template_kwargs`), exactly as on `/v1/chat/completions`. The Anthropic
+SDKs won't send an unknown field, so pass it through their escape hatch —
+`extra_body={"template_vars": {...}}` in the Python SDK, or plain HTTP:
+
+```bash
+curl http://localhost:5000/v1/messages \
+-H "Content-Type: application/json" \
+-H "x-api-key: <api key>" \
+-d '{
+  "model": "Meta-Llama-3-8B-exl3",
+  "max_tokens": 400,
+  "template_vars": {"reasoning_effort": "high"},
+  "messages": [{"role": "user", "content": "Plan a birthday party."}]
+}'
+```
+
+Precedence matches the chat completion path: the model's `template_vars_default`
+first, then anything derived from `thinking`, then the request's
+`template_vars`, and finally `template_vars_force`, which always wins. So a
+model configured with `template_vars_force: {enable_thinking: true}` keeps
+thinking on no matter what the request asks for.
+
+Below is an example CURL request using the messages endpoint:
+
+```bash
+curl http://localhost:5000/v1/messages \
+-H "Content-Type: application/json" \
+-H "x-api-key: <api key>" \
+-H "anthropic-version: 2023-06-01" \
+-d '{
+  "model": "Meta-Llama-3-8B-exl3",
+  "max_tokens": 400,
+  "system": "You are a concise assistant.",
+  "messages": [
+    {"role": "user", "content": "Name three primary colors."}
+  ]
+}'
+```
+
 ### Authentication
 
 Every call to a TabbyAPI endpoint requires some form of authentication. Keys have two types of permissions:

endpoints/Anthropic/errors.py

@@ -0,0 +1,105 @@
+"""Error handling for the Anthropic API.
+
+Anthropic clients parse a different error envelope than the one TabbyAPI
+returns elsewhere, so responses on these routes are reshaped into:
+
+    {"type": "error", "error": {"type": <error type>, "message": <message>}}
+
+The reshaping lives in a route class rather than an app-level exception
+handler so it also covers errors raised while solving dependencies (a failed
+API key check) and request validation, neither of which reach the endpoint.
+"""
+
+from fastapi import HTTPException, Request, Response
+from fastapi.exceptions import RequestValidationError
+from fastapi.responses import JSONResponse
+from fastapi.routing import APIRoute
+from typing import Callable, Optional, Tuple
+
+from common.networking import handle_request_error
+
+
+# Anthropic error types by status code. Codes TabbyAPI raises that Anthropic
+# does not define (422 from a template failure, 503 from an unloaded model)
+# map onto the closest documented type.
+ERROR_TYPES = {
+    400: "invalid_request_error",
+    401: "authentication_error",
+    403: "permission_error",
+    404: "not_found_error",
+    413: "request_too_large",
+    422: "invalid_request_error",
+    429: "rate_limit_error",
+    500: "api_error",
+    503: "api_error",
+    529: "overloaded_error",
+}
+
+
+def error_type_for_status(status_code: int) -> str:
+    """Get the Anthropic error type for a status code."""
+
+    if status_code in ERROR_TYPES:
+        return ERROR_TYPES[status_code]
+
+    return "invalid_request_error" if status_code < 500 else "api_error"
+
+
+def error_content(message: str, error_type: str) -> dict:
+    """Build the error envelope Anthropic clients expect."""
+
+    return {"type": "error", "error": {"type": error_type, "message": message}}
+
+
+class AnthropicHTTPException(HTTPException):
+    """An HTTP error carrying an explicit Anthropic error type."""
+
+    def __init__(self, status_code: int, message: str, error_type: Optional[str] = None):
+        super().__init__(status_code=status_code, detail=message)
+
+        self.error_type = error_type or error_type_for_status(status_code)
+
+
+def request_error(
+    status_code: int, message: str, error_type: Optional[str] = None, exc_info: bool = False
+) -> AnthropicHTTPException:
+    """Log a request error and return the exception to raise."""
+
+    error_message = handle_request_error(message, exc_info=exc_info).error.message
+
+    return AnthropicHTTPException(status_code, error_message, error_type)
+
+
+def exception_to_response(exc: Exception) -> Tuple[int, dict]:
+    """Map an exception raised while serving a route onto an error response."""
+
+    if isinstance(exc, AnthropicHTTPException):
+        return exc.status_code, error_content(exc.detail, exc.error_type)
+
+    if isinstance(exc, HTTPException):
+        # Raised by shared TabbyAPI code: auth, inline model loading, chat
+        # template rendering, context length
+        detail = exc.detail if isinstance(exc.detail, str) else str(exc.detail)
+        return exc.status_code, error_content(detail, error_type_for_status(exc.status_code))
+
+    if isinstance(exc, RequestValidationError):
+        return 422, error_content(str(exc.errors()), "invalid_request_error")
+
+    raise exc
+
+
+class AnthropicRoute(APIRoute):
+    """Route class that returns errors in the Anthropic envelope."""
+
+    def get_route_handler(self) -> Callable:
+        original_route_handler = super().get_route_handler()
+
+        async def anthropic_route_handler(request: Request) -> Response:
+            try:
+                return await original_route_handler(request)
+            except (HTTPException, RequestValidationError) as exc:
+                status_code, content = exception_to_response(exc)
+
+                return JSONResponse(status_code=status_code, content=content)
+
+        return anthropic_route_handler