[AgentX] DeepSeek-V4 B300 SGLang update

benchmarks/single_node/agentic/dsv4_fp4_b300_sglang_mtp.sh

@@ -65,14 +65,17 @@ export SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS=1
 CACHE_ARGS=()
 WARMUP_ARGS=()
 if require_agentic_kv_offload_backend hicache; then
-    # DeepSeek V4 HiCache currently rejects --hicache-size and supports
-    # capacity control only through a host/device token-capacity ratio.
-    # DSv4 exposes capacity as a host/device token ratio rather than bytes.
-    # Measurements put TP8 ratio=2 near 950 GB and TP4 ratio=8 near 1 TB,
-    # both below their configured capacities. The old TP4 ratio=16
-    # used roughly 2 TB and violated the half-node allocation rule.
+    # DeepSeek V4 HiCache rejects --hicache-size and controls capacity only
+    # through a host/device token ratio, so TOTAL_CPU_DRAM_GB cannot apply
+    # directly. Host capacity scales with BOTH the ratio and device KV, so it
+    # also grows with mem-fraction-static -- the two knobs multiply. Measured:
+    # TP8 ratio=2 at mem-fraction 0.835 gives 999 GB. ratio=4 at mem-fraction
+    # 0.93 overshoots: it left only 5.84 GB free on a 2,964 GB node and the
+    # V4 paged pool failed to allocate. ratio=3 keeps the tier near 2 TB with
+    # room for the paged pool, page cache, AIPerf and the router, while still
+    # well above the old half-node rule that pinned TP8 to ratio=2.
     if [ "$TP" -ge 8 ]; then
-        DEFAULT_HICACHE_RATIO=2
+        DEFAULT_HICACHE_RATIO=3
     else
         DEFAULT_HICACHE_RATIO=8
     fi
@@ -116,21 +119,48 @@ if [ "$DP_ATTENTION" = "true" ]; then
     PARALLEL_ARGS+=(
         --dp "$TP"
         --tokenizer-worker-num "$TP"
+        --enable-prefill-delayer
+        # TEMP(validation): 5 instead of 20 for the A/B against #2701.
+        --prefill-decode-interval 5
         --enable-dp-attention
         --enable-dp-attention-local-control-broadcast
         --incremental-streaming-output
         --stream-interval 20
         --dist-init-addr "127.0.0.1:$((PORT + 2000))"
         --ep-size "$EP_SIZE"
-        --moe-runner-backend flashinfer_mxfp4
+        --moe-a2a-backend megamoe
+        --enable-deepseek-v4-fp4-indexer
         --disable-flashinfer-autotune
     )
-    MEM_FRACTION_STATIC=0.95
-    if [ "$CONC" -ge 512 ]; then
-        # Leave room for FlashInfer's transient MoE workspace at the DEP8 tail.
-        MEM_FRACTION_STATIC=0.94
+    # DEP4 shards the model over half the node, so per-rank weights roughly
+    # double and the weights-only floor rises above 0.9 (the engine reports a
+    # minimum viable 0.9013 and refuses to start). Keep upstream's 0.95 there.
+    # DEP8 has room for the lower value, which leaves mega-MoE workspace
+    # headroom.
+    if [ "$TP" -ge 8 ]; then
+        # Mega-MoE's transient workspace lives OUTSIDE the static allocation and
+        # needs a single ~7 GB contiguous block, so headroom must grow with
+        # concurrency. Measured at conc 256: 0.835 (~42 GB free) runs; 0.93
+        # (~16 GB free) and 0.95 (~11 GB free) both die with a CUDA OOM on one
+        # DP rank, which then hangs the whole engine in the MLP-sync collective.
+        MEM_FRACTION_STATIC=0.93
+        if [ "$CONC" -ge 512 ]; then
+            # TEMP(validation): 0.86, matching #2701's DEP8 conc512/576 tier.
+            MEM_FRACTION_STATIC=0.86
+        elif [ "$CONC" -ge 384 ]; then
+            MEM_FRACTION_STATIC=0.89
+        elif [ "$CONC" -ge 256 ]; then
+            MEM_FRACTION_STATIC=0.9
+        fi
+    else
+        MEM_FRACTION_STATIC=0.95
     fi
-    CHUNKED_PREFILL_SIZE=16384
+    # --chunked-prefill-size is a GLOBAL budget: server_args.py divides it by
+    # dp_size, and dp_size is TP here. Scale it so every DEP shape gets the
+    # per-rank 8192 that was tuned, rather than 16384/rank at DEP4 -- which
+    # exceeds MegaMoE's per-rank token cap (a startup ValueError) and measured
+    # slower at DEP8 when tried directly.
+    CHUNKED_PREFILL_SIZE=$((8192 * TP))
 else
     PARALLEL_ARGS+=(
         --moe-runner-backend flashinfer_mxfp4
@@ -147,9 +177,24 @@ MODEL_ARGS=(
 # AgentX concurrency counts live session trees, not individual requests.
 # Allow subagent fan-out to exceed CONC without clipping request bursts.
 MAX_RUNNING_REQUESTS=$((2 * CONC))
-CUDA_GRAPH_MAX_BS=$CONC
+# Subagent fan-out means live requests exceed CONC (see MAX_RUNNING_REQUESTS
+# above), so sizing decode graphs at CONC would drop every larger batch to
+# eager decode. Capture past the fan-out; the runtime clamps this down to the
+# request pool size anyway.
+CUDA_GRAPH_MAX_BS=$((CONC * 4))
 [ "$CUDA_GRAPH_MAX_BS" -gt 64 ] && CUDA_GRAPH_MAX_BS=64
 
+# --cuda-graph-max-bs is an alias whose dest is cuda_graph_max_bs_decode, so the
+# two forms below are the same knob and must not both be passed.
+CUDA_GRAPH_ARGS=(--cuda-graph-max-bs "$CUDA_GRAPH_MAX_BS")
+SWA_FULL_TOKENS_RATIO=0.1
+if [ "$DP_ATTENTION" = "true" ]; then
+    # Decode graphs must cover the padded MTP batch across all DP ranks, which
+    # exceeds CONC; capping at 64 would fall back to eager decode.
+    CUDA_GRAPH_ARGS=(--cuda-graph-max-bs-decode 544)
+    SWA_FULL_TOKENS_RATIO=0.075
+fi
+
 export PYTHONNOUSERSITE=1
 export TORCH_CUDA_ARCH_LIST=10.0
 # Agentic warmup dispatches hundreds of large prompts at once. SGLang's
@@ -168,6 +213,17 @@ export SGLANG_OPT_USE_JIT_NORM=1
 export SGLANG_OPT_USE_JIT_INDEXER_METADATA=1
 export SGLANG_OPT_USE_TOPK_V2=1
 export SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2=1
+if [ "$DP_ATTENTION" = "true" ]; then
+    # MegaMoE's FP4/MXF4 activation path is opt-in -- both flags default False,
+    # so --moe-a2a-backend megamoe alone runs a different kernel than the one
+    # measured. DG_USE_FP4_ACTS / DG_USE_MXF4_KIND are forwarded to DeepGEMM
+    # automatically from these two.
+    export SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS=1
+    export SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND=1
+    # Must cover the per-rank prefill budget (8192) or startup raises; the
+    # extra 128 is headroom over the exact-fit boundary.
+    export SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK=8320
+fi
 if [ "${EVAL_ONLY}" != "true" ]; then
     export SGLANG_SIMULATE_ACC_LEN=2.49
     export SGLANG_SIMULATE_ACC_METHOD=match-expected
@@ -182,6 +238,23 @@ if [ -n "$TRITON_PTXAS_PATH" ]; then
     export TRITON_PTXAS_PATH
     echo "Using ptxas for Triton: $TRITON_PTXAS_PATH"
 fi
+# TEMP(validation): apply the unified-cache load-back multipin fix
+# (sgl-project/sglang#35880, cherry-pick of sgl-project/sglang#34975 onto the
+# dev-nightly-0820 base) onto the image's editable sglang source before the
+# server starts, so the conc-512 run does not die on the commit_load_back
+# single-pin assertion. Drop this once the image includes the fix.
+SGLANG_LOADBACK_PATCH="$SCRIPT_DIR/sglang-loadback-multipin.patch"
+if [ -f "$SGLANG_LOADBACK_PATCH" ] && [ -d /sgl-workspace/sglang ]; then
+    # The runtime image ships git but not patch(1); git apply works on a
+    # plain (non-repo) source tree, which is how the image ships sglang.
+    if git -C /sgl-workspace/sglang apply --check "$SGLANG_LOADBACK_PATCH" 2>/dev/null; then
+        git -C /sgl-workspace/sglang apply "$SGLANG_LOADBACK_PATCH" \
+            && echo "Applied sglang load-back multipin patch to /sgl-workspace/sglang" \
+            || echo "ERROR: failed to apply sglang load-back multipin patch"
+    else
+        echo "sglang load-back multipin patch not applicable (already applied?), skipping"
+    fi
+fi
 SGLANG_CMD=(
     "$SGLANG_PYTHON" -m sglang.launch_server
     --model-path "$MODEL_PATH"
@@ -191,9 +264,9 @@ SGLANG_CMD=(
     --trust-remote-code
     "${PARALLEL_ARGS[@]}"
     --mem-fraction-static "$MEM_FRACTION_STATIC"
-    --swa-full-tokens-ratio 0.1
+    --swa-full-tokens-ratio "$SWA_FULL_TOKENS_RATIO"
     --max-running-requests "$MAX_RUNNING_REQUESTS"
-    --cuda-graph-max-bs "$CUDA_GRAPH_MAX_BS"
+    "${CUDA_GRAPH_ARGS[@]}"
     --allow-auto-truncate
     --chunked-prefill-size "$CHUNKED_PREFILL_SIZE"
     --tool-call-parser deepseekv4

benchmarks/single_node/agentic/sglang-loadback-multipin.patch

@@ -0,0 +1,100 @@
+diff --git a/python/sglang/srt/mem_cache/unified_cache/unified_tree_core.py b/python/sglang/srt/mem_cache/unified_cache/unified_tree_core.py
+index faa14da3e..d2cfb422b 100644
+--- a/python/sglang/srt/mem_cache/unified_cache/unified_tree_core.py
++++ b/python/sglang/srt/mem_cache/unified_cache/unified_tree_core.py
+@@ -131,9 +131,14 @@ class UnifiedTreeNode:
+         self.id = UnifiedTreeNode.counter
+         UnifiedTreeNode.counter += 1
+         self.write_through_pending_id: Optional[int] = None
+-        # Anchor NodeId of an in-flight H->D load-back reading this node's
+-        # host slots; such host copies must not be reclaimed until the ack.
+-        self.load_back_pending_id: Optional[int] = None
++        # Anchor NodeIds of in-flight H->D load-backs reading this node's
++        # host slots; such host copies must not be reclaimed until every
++        # anchor has acked. Multiple live anchors can pin one node: a
++        # descendant's Full-KV chain covers this node while another request
++        # anchors here for its independently-evicted aux (e.g. mamba) state.
++        # Overlapping transfers only READ the shared host slots and write
++        # disjoint destinations, so concurrent pins are safe to track.
++        self.load_back_pending_ids: set[int] = set()
+
+     def component(self, component_type: ComponentType) -> ComponentData:
+         return self.component_data[component_type]
+@@ -1081,7 +1086,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
+         new_node.hit_count = child.hit_count
+         new_node.creation_time = child.creation_time
+         # Split fragments stay on the anchor's root path for the ack's walk.
+-        new_node.load_back_pending_id = child.load_back_pending_id
++        new_node.load_back_pending_ids = set(child.load_back_pending_ids)
+
+         self._for_each_component_lru(child, UnifiedLRUList.remove_node)
+
+@@ -1192,7 +1197,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
+             and cd.value is not None
+             and cd.host_value is not None
+             and node.write_through_pending_id is None
+-            and node.load_back_pending_id is None
++            and not node.load_back_pending_ids
+         )
+
+     def _for_each_component_lru(
+@@ -1430,7 +1435,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
+             return False
+         if (
+             node.write_through_pending_id is not None
+-            or node.load_back_pending_id is not None
++            or node.load_back_pending_ids
+         ):
+             return False
+         return cd.host_lock_ref == 0
+@@ -1988,13 +1993,10 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
+                 for xfer in xfers:
+                     for nid in xfer.nodes_to_load or ():
+                         pinned = self.node_by_id(nid)
+-                        # One live load-back per node; only the same anchor may
+-                        # re-pin (a node can sit in Full and aux transfer lists).
+-                        assert pinned.load_back_pending_id in (None, node_id), (
+-                            f"node {nid} pinned by load-back "
+-                            f"{pinned.load_back_pending_id}, new anchor {node_id}"
+-                        )
+-                        pinned.load_back_pending_id = node_id
++                        # Multiple live load-backs may pin one node (set.add is
++                        # also idempotent for a node sitting in both the Full and
++                        # an aux transfer list of the same anchor).
++                        pinned.load_back_pending_ids.add(node_id)
+         kv_xfer.device_indices = device_indices
+         self.components_by_type[BASE_COMPONENT_TYPE].commit_hicache_transfer(
+             node,
+@@ -2025,10 +2027,12 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
+         node = self.node_by_id(anchor_node_id)
+         while node is not None and node is not self.root_node:
+             if self.is_write_back:
+-                if node.load_back_pending_id != anchor_node_id:
++                if anchor_node_id not in node.load_back_pending_ids:
+                     node = node.parent
+                     continue
+-                node.load_back_pending_id = None
++                node.load_back_pending_ids.discard(anchor_node_id)
++                # The loaded copies become tracked duplicates only once the
++                # last in-flight load-back on this node acks.
+             self._update_duplicate_tracking(node)
+             node = node.parent
+
+@@ -2314,13 +2318,11 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
+         # mark would pin the node's host copy against reclaim forever.
+         ongoing_load_ids = {node_id for _, node_id in ongoing_load_back}
+         for node in all_nodes:
+-            if (
+-                node.load_back_pending_id is not None
+-                and node.load_back_pending_id not in ongoing_load_ids
+-            ):
++            stale_pins = node.load_back_pending_ids - ongoing_load_ids
++            if stale_pins:
+                 E(
+-                    f"[Ongoing] node {node.id} load_back_pending_id="
+-                    f"{node.load_back_pending_id} has no live load-back"
++                    f"[Ongoing] node {node.id} load_back_pending_ids="
++                    f"{sorted(stale_pins)} have no live load-back"
+                 )
+
+         if errors:

configs/nvidia-master.yaml

@@ -1156,7 +1156,7 @@ dsv4-fp4-b300-sglang:
       - { tp: 8, ep: 8, dp-attn: true, conc-start: 4096, conc-end: 4096 }
 
 dsv4-fp4-b300-sglang-agentic-hicache-mtp:
-  image: lmsysorg/sglang:v0.5.17-cu130
+  image: lmsysorg/sglang:dev-nightly-0820
   model: deepseek-ai/DeepSeek-V4-Pro
   model-prefix: dsv4
   runner: cluster:b300-nv
@@ -1165,21 +1165,22 @@ dsv4-fp4-b300-sglang-agentic-hicache-mtp:
   multinode: false
   scenarios:
     agentic-coding:
-    - dram-utilization: 0.80
+    - dram-utilization: 0.95
+      # TEMP(validation): sweep reduced to the single conc-512 point to validate
+      # the unified-cache load-back multipin patch applied by the recipe script.
       search-space:
-      - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 4, 8, 16, 20, 24, 32] }
-      - { tp: 8, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 4, 8, 16, 32, 40, 48, 52, 56, 60, 64, 72] }
-      - { tp: 4, ep: 4, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [8, 16, 24, 32, 40, 64], router: { name: sglang-router, version: "0.3.2" } }
-      - { tp: 4, ep: 4, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, spec-decoding: mtp, conc-list: [32, 40, 48, 56, 64, 72, 80, 88, 96, 128], router: { name: sglang-router, version: "0.3.2" } }
-      - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [52, 72, 100, 128, 144, 196, 512], router: { name: sglang-router, version: "0.3.2" } }
+      - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, spec-decoding: mtp, conc-list: [512], router: { name: sglang-router, version: "0.3.2" } }
 
   # DeepSeek-V4-Pro on B300 with EAGLE/MTP speculative decoding. Recipe is
   # selected inside benchmarks/single_node/dsv4_fp4_b300_sglang_mtp.sh by
   # DP_ATTENTION:
   #   dp-attn: false -> TP-only + flashinfer_mxfp4 + chunked-prefill 8192
-  #                     + EAGLE (3,1,4) + mem-fraction 0.90
-  #   dp-attn: true  -> DP-attn  + flashinfer_mxfp4 + chunked-prefill 32768
-  #                     + EAGLE (1,1,2) + mem-fraction 0.92 + max-running 256
+  #                     + mem-fraction 0.88 + swa-full-tokens-ratio 0.1
+  #   dp-attn: true  -> DP-attn  + megamoe + fp4 indexer
+  #                     + chunked-prefill 65536 + mem-fraction 0.90
+  #                     + swa-full-tokens-ratio 0.075
+  #                     + prefill-decode-interval 5
+  # Both paths share EAGLE (3,1,4) and max-running-requests 2*CONC.
 dsv4-fp4-b300-sglang-mtp:
   image: lmsysorg/sglang:nightly-dev-cu13-20260610-f332e526
   model: deepseek-ai/DeepSeek-V4-Pro