Tune B200 DSV4 SGLang AgentX prefill scheduling / 调整 B200 DSV4 SGLang AgentX 预填充调度

benchmarks/single_node/agentic/dsv4_fp4_b200_sglang_mtp.sh

@@ -67,11 +67,15 @@ if require_agentic_kv_offload_backend hicache; then
     # DeepSeek V4 HiCache currently rejects --hicache-size and supports
     # capacity control only through a host/device token-capacity ratio.
     # DSv4 exposes capacity as a host/device token ratio rather than bytes.
-    # B200 ratio=8 stays below the configured host-memory capacity for the
-    # currently supported TP8 shape.
-    DEFAULT_HICACHE_RATIO=8
+    # DEP8 shards the host pools and fits ratio=8 on NScale. The replicated
+    # TP8 pools need a lower ratio: 2.75 allocates about 121 GiB per rank and
+    # leaves startup headroom on the 1.7 TiB NScale hosts.
+    DEFAULT_HICACHE_RATIO=2.75
+    if [ "$DP_ATTENTION" = "true" ]; then
+        DEFAULT_HICACHE_RATIO=8
+    fi
     HICACHE_RATIO="${HICACHE_RATIO:-$DEFAULT_HICACHE_RATIO}"
-    if [ "$HICACHE_RATIO" -gt "$DEFAULT_HICACHE_RATIO" ]; then
+    if awk -v ratio="$HICACHE_RATIO" -v max="$DEFAULT_HICACHE_RATIO" 'BEGIN { exit !(ratio > max) }'; then
         echo "Error: HICACHE_RATIO=$HICACHE_RATIO exceeds configured limit $DEFAULT_HICACHE_RATIO" >&2
         exit 1
     fi
@@ -93,6 +97,7 @@ SGLANG_BACKEND_PORT="$PORT"
 ROUTER_LOG="$RESULT_DIR/router.log"
 if [ "$DP_ATTENTION" = "true" ]; then
     USE_SGLANG_ROUTER=true
+    ROUTER_POLICY_ARGS=()
     export AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID=true
     SGLANG_BACKEND_PORT=$((PORT + 1))
     SGLANG_ROUTER_METRICS_PORT=$((PORT + 10000))
@@ -103,15 +108,39 @@ PARALLEL_ARGS=(--tp "$TP")
 METRICS_ARGS=(--enable-metrics --enable-cache-report)
 CHUNKED_PREFILL_SIZE=8192
 SWA_FULL_TOKENS_RATIO=0.1
+MEM_FRACTION_STATIC=0.90
 if [ "$DP_ATTENTION" = "true" ]; then
     export SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS=1
     export SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND=1
     export SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK=8320
+
+    PREFILL_DECODE_INTERVAL=24
+    # SGLang divides this global budget by dp_size. Conc 64 retains the
+    # validated 8192-token per-rank budget.
+    CHUNKED_PREFILL_SIZE=$((8192 * TP))
+    SWA_FULL_TOKENS_RATIO=0.02
+
+    # Keep enough HBM workspace for the FP4 indexer at higher concurrency.
+    if [ "$CONC" -eq 96 ] || [ "$CONC" -eq 128 ] || [ "$CONC" -eq 160 ]; then
+        CHUNKED_PREFILL_SIZE=$((6144 * TP))
+    fi
+
+    # Conc 128/160 additionally use balanced DP admission and one-second load
+    # snapshots.
+    if [ "$CONC" -eq 128 ] || [ "$CONC" -eq 160 ]; then
+        PARALLEL_ARGS+=(--load-balance-method total_requests)
+        METRICS_ARGS+=(--load-snapshot-publish-interval 1)
+        export AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID=true
+    fi
+    if [ "$CONC" -eq 160 ]; then
+        PREFILL_DECODE_INTERVAL=20
+        ROUTER_POLICY_ARGS+=(--balance-abs-threshold 32)
+    fi
+
     PARALLEL_ARGS+=(
         --dp "$TP"
         --tokenizer-worker-num "$TP"
-        --enable-prefill-delayer
-        --prefill-decode-interval 10
+        --prefill-decode-interval "$PREFILL_DECODE_INTERVAL"
         --enable-dp-attention
         --enable-dp-attention-local-control-broadcast
         --incremental-streaming-output
@@ -123,10 +152,6 @@ if [ "$DP_ATTENTION" = "true" ]; then
         --disable-shared-experts-fusion
         --disable-flashinfer-autotune
     )
-    # SGLang divides this global budget by dp_size. Keep the tuned 8192-token
-    # per-rank budget for every DP-attention topology.
-    CHUNKED_PREFILL_SIZE=$((8192 * TP))
-    SWA_FULL_TOKENS_RATIO=0.02
 else
     PARALLEL_ARGS+=(
         --moe-runner-backend flashinfer_mxfp4
@@ -138,14 +163,16 @@ fi
 # The B200-specialized image deadlocks immediately after weight loading when
 # forced through the B300 compressed-attention/page-size overrides.
 # DeepGEMM's DSv4 indexer needs a multi-GiB temporary allocation at long
-# contexts. Leave the same HBM headroom used by the B300 recipe so a nearly
-# full GPU KV cache does not OOM while HiCache is spilling to host memory.
-MEM_FRACTION_STATIC=0.88
+# contexts. The selected fractions preserve the measured indexer and CUDA
+# graph headroom while HiCache spills to host memory.
 
 # AgentX concurrency counts live session trees, not individual requests.
 # Allow subagent fan-out to exceed CONC without clipping request bursts.
 MAX_RUNNING_REQUESTS=$((2 * CONC))
 CUDA_GRAPH_MAX_BS=$((2 * CONC))
+if [ "$DP_ATTENTION" = "true" ] && { [ "$CONC" -eq 96 ] || [ "$CONC" -eq 128 ] || [ "$CONC" -eq 160 ]; }; then
+    CUDA_GRAPH_MAX_BS=32
+fi
 CUDA_GRAPH_ARGS=(--cuda-graph-max-bs "$CUDA_GRAPH_MAX_BS")
 
 export PYTHONNOUSERSITE=1
@@ -205,6 +232,7 @@ SGLANG_CMD=(
     # across local ranks so post-load weight repacking reads from page cache
     # instead of issuing redundant fragmented mmap faults from every rank.
     --weight-loader-prefetch-checkpoints
+    --model-loader-extra-config '{"enable_multithread_load": true}'
     "${METRICS_ARGS[@]}"
     "${CACHE_ARGS[@]}"
 )
@@ -241,7 +269,8 @@ if [ "$USE_SGLANG_ROUTER" = "true" ]; then
     echo "Starting SGLang router on port $PORT for $TP DP ranks..."
     "${SGLANG_ROUTER_CMD[@]}" \
         --worker-urls "http://localhost:$SGLANG_BACKEND_PORT" \
-        --policy consistent_hashing \
+        --policy cache_aware \
+        "${ROUTER_POLICY_ARGS[@]}" \
         --request-id-headers x-correlation-id \
         --dp-aware \
         --host 0.0.0.0 \

configs/nvidia-master.yaml

@@ -941,8 +941,8 @@ dsv4-fp4-b200-sglang-agentic-hicache-mtp:
     - dram-utilization: 0.80
       search-space:
       - { tp: 8, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 3, 4, 5] }
-      - { tp: 8, kv-offloading: dram, kv-offload-backend: { name: hicache }, spec-decoding: mtp, conc-list: [8, 10, 16, 32] }
-      - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, spec-decoding: mtp, conc-list: [32, 64, 96], router: { name: sglang-router, version: "0.3.2" } }
+      - { tp: 8, kv-offloading: dram, kv-offload-backend: { name: hicache }, spec-decoding: mtp, conc-list: [8, 10, 16] }
+      - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, spec-decoding: mtp, conc-list: [64, 96, 128, 160], router: { name: sglang-router, version: "0.3.2" } }
 
 dsv4-fp4-b200-vllm:
   image: vllm/vllm-openai:v0.25.0

perf-changelog.yaml

@@ -6526,3 +6526,14 @@
     - "Pin throughput runs to the committed golden thinking_on acceptance length of 2.32 at three speculative tokens; eval-only runs keep real target verification."
     - "Keep the bfloat16 Mamba SSM state the cookbook specifies: SGLang requires it on SM100 or newer whenever the flashinfer linear-attention decode backend is selected."
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2758
+
+- config-keys:
+    - dsv4-fp4-b200-sglang-agentic-hicache-mtp
+  scenario-type:
+    - agentic-coding
+  description:
+    - "Refresh the B200 SGLang AgentX curve with validated TP8/DP8/EP8 HiCache profiles at DP-attention concurrencies 64, 96, 128, and 160."
+    - "Scope cache-aware routing and --prefill-decode-interval to the DP-attention profiles: use interval 24 at concurrency 64, 96, and 128, and interval 20 at concurrency 160."
+    - "Use mem-fraction-static 0.90 across every TP and DP profile. At concurrency 96, 128, and 160, use a 49152-token global chunked-prefill budget and CUDA graph maximum batch size 32 to preserve FP4-indexer workspace; additionally use total-requests DP balancing at concurrency 128 and 160, and set the cache-aware absolute balance threshold to 32 at concurrency 160."
+    - "Fit the HiCache tier to NScale host memory by using ratio 2.75 for replicated TP8 profiles and retaining ratio 8 for sharded DP8 profiles. Ratio 8 on TP8 requests about 351 GiB per rank and fails host-pool initialization on a 1.7 TiB NScale node; ratio 2.75 targets about 121 GiB per rank while leaving startup headroom."
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2718