From f162a0a5977580adcfe78ebf24438c8310566161 Mon Sep 17 00:00:00 2001 From: Yancy Date: Sat, 29 Aug 2026 10:21:32 +0000 Subject: [PATCH 1/2] fix: Improve JSON parsing robustness under high concurrency MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Under high concurrency, the router could encounter JSONDecodeError ("Unterminated string") when parsing request bodies. The root cause is a race window where the request body can be read in a truncated state, particularly when `request.json()` is called before `request.body()` in the semantic cache integration path. Changes: 1. Add diagnostic logging in `route_general_request` when JSON parsing fails — logs actual body length, Content-Length header, and the parser error to help debug truncation issues 2. Return detailed error info (error detail + body_length) in the 400 response so callers can diagnose the problem 3. Add same diagnostic logging in `process_request` for consistency 4. Replace `request.json()` with `request.body()` + `json.loads()` in `check_semantic_cache` to avoid double body reading and ensure consistent body access pattern Closes #369 Signed-off-by: Asthenia Signed-off-by: Yancy --- .../semantic_cache_integration.py | 3 ++- .../services/request_service/request.py | 26 ++++++++++++++++--- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/src/vllm_router/experimental/semantic_cache_integration.py b/src/vllm_router/experimental/semantic_cache_integration.py index 8ec976c78..f2a8c8c74 100644 --- a/src/vllm_router/experimental/semantic_cache_integration.py +++ b/src/vllm_router/experimental/semantic_cache_integration.py @@ -194,7 +194,8 @@ async def check_semantic_cache(request: Request) -> Optional[JSONResponse]: return None # Get the request body - body = await request.json() + body = await request.body() + body = json.loads(body) if body else {} logger.info("Checking semantic cache for potential cache hit") # Check if semantic cache is initialized diff --git a/src/vllm_router/services/request_service/request.py b/src/vllm_router/services/request_service/request.py index ae10819fc..a71f91421 100644 --- a/src/vllm_router/services/request_service/request.py +++ b/src/vllm_router/services/request_service/request.py @@ -283,9 +283,16 @@ async def process_request( request_json = json.loads(body) is_streaming = request_json.get("stream", False) model_name = request_json.get("model", "unknown") - except (JSONDecodeError, UnicodeDecodeError, ValueError): + except (JSONDecodeError, UnicodeDecodeError, ValueError) as e: # If we can't parse the body as JSON, assume it's not streaming - raise HTTPException(status=400, detail="Request body is not JSON parsable.") + logger.warning( + f"Failed to parse request body in process_request: {e}. " + f"Body length: {len(body)} bytes" + ) + raise HTTPException( + status=400, + detail=f"Request body is not JSON parsable: {e}", + ) # Add streaming info to span after parsing if span is not None: @@ -418,10 +425,21 @@ async def route_general_request( request_body = await request.body() try: request_json = json.loads(request_body) if request_body else {} - except (json.JSONDecodeError, UnicodeDecodeError, RecursionError): + except (json.JSONDecodeError, UnicodeDecodeError, RecursionError) as e: + content_length = len(request_body) + expected_length = request.headers.get("content-length") + logger.warning( + f"Failed to parse request body as JSON for request {request_id}: " + f"{e}. Body length: {content_length} bytes, " + f"Content-Length header: {expected_length}" + ) return JSONResponse( status_code=400, - content={"error": "Invalid request: request body must be valid JSON."}, + content={ + "error": "Invalid request: request body must be valid JSON.", + "detail": str(e), + "body_length": content_length, + }, headers={"X-Request-Id": request_id}, ) From 95c0768a4cdf1977e1190ac9d37df452e5af17a0 Mon Sep 17 00:00:00 2001 From: Yancy Date: Sat, 29 Aug 2026 10:24:31 +0000 Subject: [PATCH 2/2] fix: Address review feedback on JSON parsing error handling - Fix HTTPException: use status_code=400 instead of status=400 - Use json.JSONDecodeError instead of requests.JSONDecodeError for correct exception type matching - Wrap json.loads in check_semantic_cache with try-except, return None on parse failure to gracefully fall through to main handler Signed-off-by: Asthenia Signed-off-by: Yancy --- .../experimental/semantic_cache_integration.py | 8 ++++++-- src/vllm_router/services/request_service/request.py | 4 ++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/vllm_router/experimental/semantic_cache_integration.py b/src/vllm_router/experimental/semantic_cache_integration.py index f2a8c8c74..ef0fda7ba 100644 --- a/src/vllm_router/experimental/semantic_cache_integration.py +++ b/src/vllm_router/experimental/semantic_cache_integration.py @@ -194,8 +194,12 @@ async def check_semantic_cache(request: Request) -> Optional[JSONResponse]: return None # Get the request body - body = await request.body() - body = json.loads(body) if body else {} + try: + body = await request.body() + body = json.loads(body) if body else {} + except (json.JSONDecodeError, UnicodeDecodeError): + logger.warning("Failed to parse request body in semantic cache check, skipping") + return None logger.info("Checking semantic cache for potential cache hit") # Check if semantic cache is initialized diff --git a/src/vllm_router/services/request_service/request.py b/src/vllm_router/services/request_service/request.py index a71f91421..3d9f006d7 100644 --- a/src/vllm_router/services/request_service/request.py +++ b/src/vllm_router/services/request_service/request.py @@ -283,14 +283,14 @@ async def process_request( request_json = json.loads(body) is_streaming = request_json.get("stream", False) model_name = request_json.get("model", "unknown") - except (JSONDecodeError, UnicodeDecodeError, ValueError) as e: + except (json.JSONDecodeError, UnicodeDecodeError, ValueError) as e: # If we can't parse the body as JSON, assume it's not streaming logger.warning( f"Failed to parse request body in process_request: {e}. " f"Body length: {len(body)} bytes" ) raise HTTPException( - status=400, + status_code=400, detail=f"Request body is not JSON parsable: {e}", )