feature/metrics

backends/exllamav3/model.py

@@ -37,6 +37,7 @@
 from common.health import HealthManager
 from common.errors import ContextLengthExceededError, validate_context_requirements
 from common.logger import xlogger
+from common.metrics import MetricsManager
 from common.multimodal import MultimodalEmbeddingWrapper
 from common.networking import DisconnectHandler
 from common.optional_dependencies import check_package_version
@@ -351,6 +352,12 @@ async def create(cls, model_directory: pathlib.Path, hf_model: HFModel, **kwargs
         self.max_seq_len = max_seq_len
         self.cache_size = cache_size
 
+        # Size the /metrics token histograms to this model's context length.
+        # Their buckets are meaningless until the range they have to cover is
+        # known, and a ladder that overshoots it reports percentiles above any
+        # request the server can even accept.
+        MetricsManager.configure_token_buckets(max_seq_len)
+
         # Max batch size
         default_mbs = 4 if self.model.caps.get("recurrent_states") else 128
         self.max_batch_size = unwrap(kwargs.get("max_batch_size"), default_mbs)
@@ -1065,7 +1072,8 @@ def handle_finish_chunk(self, result: dict, request_id: str, full_text: str):
         # Prompt
         prompt_tokens = result.get("prompt_tokens")
         cached_tokens = round(result.get("cached_tokens"), 2)
-        prompt_time = round(result.get("time_prefill"), 2)
+        raw_prompt_time = result.get("time_prefill")
+        prompt_time = round(raw_prompt_time, 2)
         prompt_ts = (
             "Indeterminate"
             if prompt_time == 0
@@ -1111,6 +1119,20 @@ def handle_finish_chunk(self, result: dict, request_id: str, full_text: str):
                 }
             )
 
+        # Accumulate server-wide metrics for the /metrics endpoint
+        MetricsManager.record_generation(
+            prompt_tokens=prompt_tokens,
+            cached_tokens=cached_tokens,
+            gen_tokens=gen_tokens,
+            # Unrounded, so the aggregate rates are not skewed by the 0.01s
+            # display rounding on short prefills.
+            prompt_time=raw_prompt_time,
+            gen_time=gen_time,
+            queue_time=queue_time,
+            accepted_draft_tokens=accepted_draft_tokens,
+            rejected_draft_tokens=rejected_draft_tokens,
+        )
+
         return finish_chunk
 
     async def generate_gen(

common/config_models.py

@@ -81,6 +81,14 @@ class NetworkConfig(BaseConfigModel):
         ),
         ge=0,
     )
+    enable_metrics: Optional[bool] = Field(
+        False,
+        description=(
+            "Enable the Prometheus-compatible /metrics endpoint (default: False).\n"
+            "Exposes aggregate inference stats in the text exposition format.\n"
+            "NOTE: This endpoint is not protected by API key authentication."
+        ),
+    )
 
     # Converts all strings in the api_servers list to lowercase
     # NOTE: Expand if more models need this validator