diff --git a/src/vllm_router/stats/request_stats.py b/src/vllm_router/stats/request_stats.py index f0409b912..db7fc3990 100644 --- a/src/vllm_router/stats/request_stats.py +++ b/src/vllm_router/stats/request_stats.py @@ -108,10 +108,9 @@ class RequestStatsMonitor(metaclass=SingletonMeta): Monitors the request statistics of all serving engines. """ - # NOTE (ApostaC): Currently, QPS is calculated based on the number of - # arrived requests in the sliding window, but the inter_token_latency and - # ttft are calculated based on the number of completed requests in the - # sliding window. + # NOTE (ApostaC): Currently, QPS is calculated based on requests arriving + # in the sliding window, while TTFT is calculated when the first response + # signal is observed for requests in the sliding window. def __init__(self, sliding_window_size: float = None): if hasattr(self, "_initialized"): return @@ -125,7 +124,7 @@ def __init__(self, sliding_window_size: float = None): # The time when the request is coming (engine_url, request_id) -> timestamp self.request_start_time: Dict[Tuple[str, str], float] = {} - # Record time when first token is received: (engine_url, request_id) -> timestamp + # First response signal time: (engine_url, request_id) -> timestamp self.first_token_time: Dict[Tuple[str, str], float] = {} # Number of requests in different stages (from the start of the router) @@ -173,12 +172,15 @@ def on_new_request(self, engine_url: str, request_id: str, timestamp: float): def on_request_response(self, engine_url: str, request_id: str, timestamp: float): """ - Tell the monitor that a response token has been received for a request. + Tell the monitor that the first response signal has been observed. + + This is the first response body chunk for response-body proxy paths + and the response headers for non-streaming multipart responses. Args: engine_url: The URL of the serving engine request_id: The global request ID - timestamp: The timestamp when the response token was received + timestamp: The timestamp when the first response signal was observed """ if (engine_url, request_id) not in self.request_start_time: return @@ -245,8 +247,8 @@ def get_request_stats(self, current_time: float) -> Dict[str, RequestStats]: Returns: A dictionary where the key is the serving engine URL and the value is the request statistics for that engine. - The TTFT and inter token latency will be -1 if there is no requests - finished in the sliding window. + TTFT will be -1 if no request produced its first response signal + in the sliding window. """ ret = {} urls = set(self.in_prefill_requests.keys()).union(