test: validate upstream-native vLLM Router topologies

.github/workflows/e2e-tests.yml

@@ -261,6 +261,7 @@ jobs:
                   MULTI_AGENTIC_EVAL=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if x.get('scenario-type') == 'agentic-coding' and 'prefill' in x and x.get('run-eval', False)]))" | score_matrix multi-agentic-eval)
                   SINGLE=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' not in x and x.get('scenario-type') != 'agentic-coding' and not x.get('eval-only', False)]))" | score_matrix single)
                   MULTI=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' in x and x.get('scenario-type') != 'agentic-coding' and not x.get('eval-only', False)]))" | score_matrix multi)
+                  EVALS=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' not in x and x.get('scenario-type') != 'agentic-coding' and x.get('run-eval', False)]))" | score_matrix eval)
                   MULTI_EVAL=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' in x and x.get('scenario-type') != 'agentic-coding' and x.get('run-eval', False)]))" | score_matrix multi-eval)
                   {
                     echo "agentic-config=$AGENTIC"

.github/workflows/run-sweep.yml

@@ -249,7 +249,7 @@ jobs:
                       --model 'claude-fable-5'
                       --max-turns 8
                       --allowedTools "Read,Glob,Grep,Bash(git diff:*)"
-                      --json-schema '{"type":"object","properties":{"criteria":{"type":"array","items":{"type":"string","enum":["multi-node","agentic","eval-only","fp4","mtp","eagle","eagle3","sglang","vllm","dynamo-sglang","dynamo-vllm","glm5","glm5.1","kimik2.5","kimik3","dsv4","minimaxm3","qwen3.5","dsr1","checklist-complete","patchwork"]},"uniqueItems":true},"reason":{"type":"string"}},"required":["criteria","reason"]}'
+                      --json-schema '{"type":"object","properties":{"criteria":{"type":"array","items":{"type":"string","enum":["multi-node","agentic","eval-only","fp4","mtp","eagle","eagle3","sglang","vllm","vllm-router","dynamo-sglang","dynamo-vllm","glm5","glm5.1","kimik2.5","kimik3","dsv4","minimaxm3","qwen3.5","dsr1","checklist-complete","patchwork"]},"uniqueItems":true},"reason":{"type":"string"}},"required":["criteria","reason"]}'
                   prompt: |
                       Inspect this Actions run's change range.
 
@@ -264,7 +264,7 @@ jobs:
                       - eval-only: evaluation without throughput measurement
                       - fp4: FP4 precision
                       - mtp, eagle, eagle3: speculative decoding method
-                      - sglang, vllm, dynamo-vllm: runtime framework
+                      - sglang, vllm, vllm-router, dynamo-vllm: runtime framework
                       - model criterion: matching configured model family
                       - checklist-complete: PR checklist is satisfied
                       - patchwork: modified upstream engine or runtime source

benchmarks/benchmark_lib.sh

@@ -67,8 +67,10 @@ require_agentic_kv_offload_backend() {
     esac
 }
 
-# Agentic replays must use the model's native context limit. Ignore inherited
-# workflow or shell overrides so neither the server nor AIPerf applies a cap.
+# Agentic replays normally use the model's native context limit. Ignore the
+# workflow's generated MAX_MODEL_LEN so it cannot accidentally cap a native
+# AgentX recipe. Recipes that intentionally serve below the native limit can
+# opt in with AIPERF_MAX_CONTEXT_LENGTH.
 _benchmark_caller="${BASH_SOURCE[1]:-}"
 if [[ "$_benchmark_caller" == */agentic/* ||
       "$_benchmark_caller" == */agentic_*.sh ||
@@ -1973,6 +1975,7 @@ build_replay_cmd() {
     local result_dir="$1"
     local duration="$DURATION"
     local warmup_requests_per_lane="${AIPERF_WARMUP_REQUESTS_PER_LANE:-10}"
+    local max_context_length="${AIPERF_MAX_CONTEXT_LENGTH:-${MAX_MODEL_LEN:-}}"
 
     # Fast mode minimizes setup by advancing each trajectory lane only once
     # and shortens profiling to 20 minutes.
@@ -2092,8 +2095,12 @@ build_replay_cmd() {
     # server. The WEKA corpus contains a few very long parent/subagent traces;
     # if we mmap and replay them against a smaller-context server they become
     # deterministic 4xxs and can still pressure the engine while queued.
-    if [ -n "${MAX_MODEL_LEN:-}" ] && [ "$MAX_MODEL_LEN" != "0" ]; then
-        REPLAY_CMD+=" --max-context-length $MAX_MODEL_LEN"
+    if [ -n "$max_context_length" ] && [ "$max_context_length" != "0" ]; then
+        if ! [[ "$max_context_length" =~ ^[1-9][0-9]*$ ]]; then
+            echo "Error: max context length must be a positive integer, got '$max_context_length'" >&2
+            return 1
+        fi
+        REPLAY_CMD+=" --max-context-length $max_context_length"
     fi
     # Default --num-dataset-entries is 100; the with-subagents Weka corpus
     # has 393. Cap at 393 so all unique traces are loaded (the loader treats

benchmarks/multi_node/srt-slurm-recipes/configs/native-vllm-router-deps.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+python3 -m pip install --no-cache-dir --upgrade "vllm-router==0.1.15"
+command -v vllm-router >/dev/null
+python3 - <<'PY'
+from importlib.metadata import version
+
+assert version("vllm-router") == "0.1.15"
+print(f"vllm-router {version('vllm-router')}")
+PY

benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-r1/agentic/agg-gb200-dep4-vllm-router-agentic.yaml

@@ -0,0 +1,82 @@
+name: "dsr1-gb200-vllm-router-agg-dep4-agentic"
+
+model:
+  path: "dsr1-nvfp4"
+  container: "vllm/vllm-openai:v0.25.1"
+  precision: "fp4"
+
+setup_script: native-vllm-router-deps.sh
+
+slurm:
+  time_limit: "4:00:00"
+
+health_check:
+  max_attempts: 720
+  interval_seconds: 10
+
+resources:
+  gpu_type: "gb200"
+  gpus_per_node: 4
+  agg_nodes: 1
+  agg_workers: 1
+  gpus_per_agg: 4
+
+infra:
+  etcd_nats_dedicated_node: false
+
+frontend:
+  type: vllm-router
+  enable_multiple_frontends: false
+  args:
+    policy: consistent_hash
+
+backend:
+  type: vllm
+  connector: null
+  dp_launch_mode: per_node
+  aggregated_environment:
+    VLLM_ENGINE_READY_TIMEOUT_S: "3600"
+    VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800"
+    VLLM_LOG_STATS_INTERVAL: "1"
+    VLLM_USE_V2_MODEL_RUNNER: "1"
+    VLLM_USE_FLASHINFER_MOE_FP4: "1"
+  vllm_config:
+    aggregated:
+      served-model-name: "nvidia/DeepSeek-R1-0528-NVFP4-v2"
+      kv-cache-dtype: "fp8"
+      tensor-parallel-size: 1
+      pipeline-parallel-size: 1
+      data-parallel-size: 4
+      data-parallel-rpc-port: 13345
+      enable-expert-parallel: true
+      enable-prefix-caching: true
+      enable-prompt-tokens-details: true
+      max-model-len: 163840
+      max-num-seqs: 128
+      max-num-batched-tokens: 8192
+      trust-remote-code: true
+      block-size: 128
+      gpu-memory-utilization: 0.85
+      stream-interval: 10
+
+sbatch_directives:
+  cpus-per-task: "144"
+  mem: "0"
+
+srun_options:
+  container-remap-root: ""
+
+benchmark:
+  type: custom
+  command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh
+  env:
+    INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace"
+    RESULT_DIR: "/logs/agentic"
+    PORT: "8000"
+    IS_MULTINODE: "true"
+    AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:"
+    AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: "1"
+    AIPERF_MAX_CONTEXT_LENGTH: "163840"
+    AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache"
+    HF_HUB_CACHE: "/hf_hub_cache"
+    WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126_256k"

benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-r1/agentic/agg-gb200-dep8-vllm-router-agentic.yaml

@@ -0,0 +1,82 @@
+name: "dsr1-gb200-vllm-router-agg-dep8-agentic"
+
+model:
+  path: "dsr1-nvfp4"
+  container: "vllm/vllm-openai:v0.25.1"
+  precision: "fp4"
+
+setup_script: native-vllm-router-deps.sh
+
+slurm:
+  time_limit: "4:00:00"
+
+health_check:
+  max_attempts: 720
+  interval_seconds: 10
+
+resources:
+  gpu_type: "gb200"
+  gpus_per_node: 4
+  agg_nodes: 2
+  agg_workers: 1
+  gpus_per_agg: 8
+
+infra:
+  etcd_nats_dedicated_node: false
+
+frontend:
+  type: vllm-router
+  enable_multiple_frontends: false
+  args:
+    policy: consistent_hash
+
+backend:
+  type: vllm
+  connector: null
+  dp_launch_mode: per_node
+  aggregated_environment:
+    VLLM_ENGINE_READY_TIMEOUT_S: "3600"
+    VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800"
+    VLLM_LOG_STATS_INTERVAL: "1"
+    VLLM_USE_V2_MODEL_RUNNER: "1"
+    VLLM_USE_FLASHINFER_MOE_FP4: "1"
+  vllm_config:
+    aggregated:
+      served-model-name: "nvidia/DeepSeek-R1-0528-NVFP4-v2"
+      kv-cache-dtype: "fp8"
+      tensor-parallel-size: 1
+      pipeline-parallel-size: 1
+      data-parallel-size: 8
+      data-parallel-rpc-port: 13345
+      enable-expert-parallel: true
+      enable-prefix-caching: true
+      enable-prompt-tokens-details: true
+      max-model-len: 163840
+      max-num-seqs: 128
+      max-num-batched-tokens: 8192
+      trust-remote-code: true
+      block-size: 128
+      gpu-memory-utilization: 0.85
+      stream-interval: 10
+
+sbatch_directives:
+  cpus-per-task: "144"
+  mem: "0"
+
+srun_options:
+  container-remap-root: ""
+
+benchmark:
+  type: custom
+  command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh
+  env:
+    INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace"
+    RESULT_DIR: "/logs/agentic"
+    PORT: "8000"
+    IS_MULTINODE: "true"
+    AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:"
+    AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: "1"
+    AIPERF_MAX_CONTEXT_LENGTH: "163840"
+    AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache"
+    HF_HUB_CACHE: "/hf_hub_cache"
+    WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126_256k"

benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-r1/agentic/disagg-gb200-1p2d-dep4-vllm-router-agentic.yaml

@@ -0,0 +1,90 @@
+name: "dsr1-gb200-vllm-router-disagg-1p2d-dep4-agentic"
+
+model:
+  path: "dsr1-nvfp4"
+  container: "vllm/vllm-openai:v0.25.1"
+  precision: "fp4"
+
+setup_script: native-vllm-router-deps.sh
+
+slurm:
+  time_limit: "4:00:00"
+
+health_check:
+  max_attempts: 720
+  interval_seconds: 10
+
+resources:
+  gpu_type: "gb200"
+  gpus_per_node: 4
+  prefill_nodes: 1
+  prefill_workers: 1
+  gpus_per_prefill: 4
+  decode_nodes: 2
+  decode_workers: 2
+  gpus_per_decode: 4
+
+infra:
+  etcd_nats_dedicated_node: false
+
+frontend:
+  type: vllm-router
+  enable_multiple_frontends: false
+  args:
+    policy: consistent_hash
+    prefill-policy: consistent_hash
+    decode-policy: consistent_hash
+
+backend:
+  type: vllm
+  connector: null
+  dp_launch_mode: per_node
+  prefill_environment: &vllm_environment
+    VLLM_ENGINE_READY_TIMEOUT_S: "3600"
+    VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800"
+    VLLM_LOG_STATS_INTERVAL: "1"
+    VLLM_USE_V2_MODEL_RUNNER: "1"
+    VLLM_USE_FLASHINFER_MOE_FP4: "1"
+  decode_environment: *vllm_environment
+  vllm_config:
+    prefill: &vllm_config
+      kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both"}'
+      served-model-name: "nvidia/DeepSeek-R1-0528-NVFP4-v2"
+      kv-cache-dtype: "fp8"
+      tensor-parallel-size: 1
+      pipeline-parallel-size: 1
+      data-parallel-size: 4
+      data-parallel-rpc-port: 13345
+      enable-expert-parallel: true
+      enable-prefix-caching: true
+      enable-prompt-tokens-details: true
+      max-model-len: 163840
+      max-num-seqs: 128
+      max-num-batched-tokens: 8192
+      trust-remote-code: true
+      block-size: 128
+      gpu-memory-utilization: 0.85
+      stream-interval: 10
+    decode: *vllm_config
+
+sbatch_directives:
+  cpus-per-task: "144"
+  mem: "0"
+
+srun_options:
+  container-remap-root: ""
+
+benchmark:
+  type: custom
+  command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh
+  env:
+    INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace"
+    RESULT_DIR: "/logs/agentic"
+    PORT: "8000"
+    IS_MULTINODE: "true"
+    AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:"
+    AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: "1"
+    AIPERF_MAX_CONTEXT_LENGTH: "163840"
+    AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache"
+    HF_HUB_CACHE: "/hf_hub_cache"
+    WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126_256k"