From 8137dd8847f42af429166cbdd9f3e8bbf2085146 Mon Sep 17 00:00:00 2001 From: Asthenia Date: Sat, 29 Aug 2026 10:57:26 +0000 Subject: [PATCH 1/2] fix: Run KvawareRouter /tokenize fallback in executor to avoid blocking event loop The remote /tokenize fallback in KvawareRouter.route_request() used a synchronous requests.post call directly on the event loop. Under load, this blocks the loop and can stall health checks and other requests. Mirror the tokenize_prompt() fallback pattern by running the blocking HTTP call via run_in_executor. Signed-off-by: Asthenia --- src/vllm_router/routers/routing_logic.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/vllm_router/routers/routing_logic.py b/src/vllm_router/routers/routing_logic.py index b25e9184a..eaf8a97d5 100644 --- a/src/vllm_router/routers/routing_logic.py +++ b/src/vllm_router/routers/routing_logic.py @@ -402,10 +402,16 @@ async def route_request( "model": endpoints[0].model_names[0], "prompt": request_json.get("prompt", ""), } - body = requests.post( - remote_url, headers=headers, json=data, timeout=10 - ).json() - token_ids = body["tokens"] + # Run the blocking HTTP call in an executor so it does not + # stall the event loop (mirrors tokenize_prompt fallback). + loop = asyncio.get_running_loop() + response = await loop.run_in_executor( + None, + lambda: requests.post( + remote_url, headers=headers, json=data, timeout=10 + ), + ) + token_ids = response.json()["tokens"] event_id = "Lookup" + str(uuid.uuid4()) msg = LookupMsg(tokens=token_ids, event_id=event_id) From 2a62aa441bb8a7f58d4f087040bb2c7ba137a2c5 Mon Sep 17 00:00:00 2001 From: Asthenia Date: Sat, 29 Aug 2026 11:00:17 +0000 Subject: [PATCH 2/2] =?UTF-8?q?fix:=20Address=20review=20feedback=20?= =?UTF-8?q?=E2=80=94=20use=20shared=20aiohttp=20session=20for=20/tokenize?= =?UTF-8?q?=20fallback?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per review feedback, replace the executor-wrapped requests.post with the shared aiohttp client session from request.app.state.aiohttp_client_wrapper(). This reuses TCP connections (avoiding socket exhaustion under load) and keeps the call off the event loop. Signed-off-by: Asthenia --- src/vllm_router/routers/routing_logic.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/src/vllm_router/routers/routing_logic.py b/src/vllm_router/routers/routing_logic.py index eaf8a97d5..6bf23a2bc 100644 --- a/src/vllm_router/routers/routing_logic.py +++ b/src/vllm_router/routers/routing_logic.py @@ -402,16 +402,14 @@ async def route_request( "model": endpoints[0].model_names[0], "prompt": request_json.get("prompt", ""), } - # Run the blocking HTTP call in an executor so it does not - # stall the event loop (mirrors tokenize_prompt fallback). - loop = asyncio.get_running_loop() - response = await loop.run_in_executor( - None, - lambda: requests.post( - remote_url, headers=headers, json=data, timeout=10 - ), - ) - token_ids = response.json()["tokens"] + # Use the shared aiohttp session so connections are reused and + # the event loop is not blocked (mirrors route_general_request). + client = request.app.state.aiohttp_client_wrapper() + async with client.post( + remote_url, headers=headers, json=data, timeout=10 + ) as response: + response_json = await response.json() + token_ids = response_json["tokens"] event_id = "Lookup" + str(uuid.uuid4()) msg = LookupMsg(tokens=token_ids, event_id=event_id)