[AgentX] DeepSeek-V4-Pro-0813 llm-d on GB200

benchmarks/multi_node/dsv4_fp4_gb200_llmd-vllm-agg.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+set -eo pipefail
+
+export GPUS_PER_NODE=4 TIME_LIMIT=08:00:00 CONTAINER_IMAGE="$IMAGE"
+export PREFILL_WORKERS=1 DECODE_WORKERS=1
+
+cd "$GITHUB_WORKSPACE/benchmarks/multi_node/llm-d"
+exec bash ./submit.sh "$PREFILL_NODES" "$DECODE_NODES" \
+    "$ISL" "$OSL" "${CONC_LIST// /x}" inf "$RANDOM_RANGE_RATIO"

benchmarks/multi_node/dsv4_fp4_gb200_llmd-vllm-disagg.sh

@@ -1,60 +1,9 @@
 #!/usr/bin/env bash
-#
-# Wrapper for the DeepSeek-V4-Pro GB200 llmd-vllm P/D disagg benchmark
-# (mid-curve 1P1D and high-tpt 2P1D). Sibling of gptoss_fp4_h200_llmd-vllm.sh -
-# same shape, different topology (GB200 = 4 GPUs/node, role spans 2 nodes;
-# H200 = 8 GPUs/node, role on a single node). The runner resolves this script via
-#   SCRIPT_NAME="${EXP_NAME%%_*}_${PRECISION}_gb200_llmd-vllm-disagg.sh"
-# from launch_gb200-nv.sh.
+set -eo pipefail
 
-set -euo pipefail
+export GPUS_PER_NODE=4 TIME_LIMIT=08:00:00 CONTAINER_IMAGE="$IMAGE"
+export PREFILL_WORKERS="$PREFILL_NUM_WORKERS" DECODE_WORKERS="$DECODE_NUM_WORKERS"
 
-source "$(dirname "$0")/../benchmark_lib.sh"
-
-check_env_vars \
-    CONC_LIST \
-    ISL \
-    OSL \
-    IMAGE \
-    MODEL_PATH \
-    PREFILL_NODES \
-    DECODE_NODES \
-    RANDOM_RANGE_RATIO
-
-if [[ -n "${SLURM_JOB_ID:-}" ]]; then
-    echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME"
-fi
-
-set -x
-
-cd "$GITHUB_WORKSPACE/benchmarks/multi_node/llm-d" || exit 1
-
-# GB200 = 4 GPUs per node (Grace+Blackwell). The shared submit.sh
-# defaults GPUS_PER_NODE to 8, which is wrong for this SKU and would
-# overshoot DP_SIZE = nodes * 8.
-export GPUS_PER_NODE="${GPUS_PER_NODE:-4}"
-
-export TIME_LIMIT="${TIME_LIMIT:-08:00:00}"
-export MODEL_PATH=$MODEL_PATH
-export MODEL_NAME=$MODEL_NAME
-export CONTAINER_IMAGE=$IMAGE
-
-# Worker count per role (Option B multi-engine). Prefer an explicit
-# PREFILL_WORKERS/DECODE_WORKERS from the matrix additional-settings; else fall
-# back to the matrix num-worker fields (PREFILL_NUM_WORKERS/DECODE_NUM_WORKERS);
-# else 1 (single engine = unchanged 1P+1D / mid-curve). submit.sh reads these.
-export PREFILL_WORKERS="${PREFILL_WORKERS:-${PREFILL_NUM_WORKERS:-1}}"
-export DECODE_WORKERS="${DECODE_WORKERS:-${DECODE_NUM_WORKERS:-1}}"
-
-JOB_ID=$(bash ./submit.sh \
-    "$PREFILL_NODES" \
-    "$DECODE_NODES" \
-    "$ISL" "$OSL" "${CONC_LIST// /x}" inf \
-    "$RANDOM_RANGE_RATIO")
-
-if [[ -z "$JOB_ID" ]]; then
-    echo "Failed to submit job" >&2
-    exit 1
-fi
-
-echo "$JOB_ID"
+cd "$GITHUB_WORKSPACE/benchmarks/multi_node/llm-d"
+exec bash ./submit.sh "$PREFILL_NODES" "$DECODE_NODES" \
+    "$ISL" "$OSL" "${CONC_LIST// /x}" inf "$RANDOM_RANGE_RATIO"

benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-agentic.yaml

@@ -0,0 +1,70 @@
+# DeepSeek-V4-Pro-0813 (DSpark) FP4 GB200, aggregated DEP8, 2 nodes.
+# DSpark speculative decoding: 5 tokens. No Mooncake.
+apiVersion: llm-d.ai/v1alpha1
+kind: EndpointPickerConfig
+
+plugins:
+  - name: file-disc
+    type: file-discovery
+    parameters:
+      path: /tmp/endpoints.yaml
+      watchFile: false
+
+  - type: prefill-filter
+  - type: queue-scorer
+  - type: kv-cache-utilization-scorer
+  - type: active-request-scorer
+  - type: prefix-cache-scorer
+
+schedulingProfiles:
+  - name: default
+    plugins:
+      - pluginRef: prefill-filter
+      - pluginRef: queue-scorer
+        weight: 2
+      - pluginRef: kv-cache-utilization-scorer
+        weight: 2
+      - pluginRef: active-request-scorer
+        weight: 1
+      - pluginRef: prefix-cache-scorer
+        weight: 3
+
+dataLayer:
+  discovery:
+    pluginRef: file-disc
+
+prefill:
+  tp: 1
+  enable-expert-parallel: true
+  extra-args: >-
+    --kv-cache-dtype fp8
+    --gpu-memory-utilization 0.92
+    --max-num-batched-tokens 8192
+    --block-size 256
+    --tokenizer-mode deepseek_v4
+    --tool-call-parser deepseek_v4
+    --enable-auto-tool-choice
+    --reasoning-parser deepseek_v4
+    --moe-backend deep_gemm_mega_moe
+    --enable-ep-weight-filter
+    --enable-cumem-allocator
+    --no-disable-hybrid-kv-cache-manager
+    --no-enable-flashinfer-autotune
+    --numa-bind
+    --compilation-config {"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}
+    --speculative-config {"method":"dspark","num_speculative_tokens":5,"draft_sample_method":"probabilistic","attention_backend":"FLASHINFER_MLA_SPARSE_DSV4","enable_adaptive_verification":false}
+    --attention-config {"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}
+  env:
+    VLLM_USE_RUST_FRONTEND: "1"
+    VLLM_SERVER_DEV_MODE: "1"
+    VLLM_ENGINE_READY_TIMEOUT_S: "3600"
+    VLLM_RPC_TIMEOUT: "600000"
+    VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768"
+    VLLM_DSV4_MEGA_FP8_COMBINE: "1"
+    VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800"
+    VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1"
+    TILELANG_CLEANUP_TEMP_FILES: "1"
+    VLLM_USE_NCCL_SYMM_MEM: "1"
+
+slurm:
+  time_limit: "08:00:00"

benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-mooncake-agentic.yaml

@@ -0,0 +1,89 @@
+# DeepSeek-V4-Pro-0813 (DSpark) FP4 GB200, aggregated DEP8 with Mooncake prefix-cache.
+# Sibling of agg-gb200-dep8-dspark-agentic.yaml; adds Mooncake (P2PHANDSHAKE embedded RDMA)
+# so server.sh wires MultiConnector (NixlConnector + MooncakeStoreConnector, kv_both role).
+apiVersion: llm-d.ai/v1alpha1
+kind: EndpointPickerConfig
+
+plugins:
+  - name: file-disc
+    type: file-discovery
+    parameters:
+      path: /tmp/endpoints.yaml
+      watchFile: false
+
+  - type: prefill-filter
+  - type: queue-scorer
+  - type: kv-cache-utilization-scorer
+  - type: active-request-scorer
+  - type: prefix-cache-scorer
+
+schedulingProfiles:
+  - name: default
+    plugins:
+      - pluginRef: prefill-filter
+      - pluginRef: queue-scorer
+        weight: 2
+      - pluginRef: kv-cache-utilization-scorer
+        weight: 2
+      - pluginRef: active-request-scorer
+        weight: 1
+      - pluginRef: prefix-cache-scorer
+        weight: 3
+
+dataLayer:
+  discovery:
+    pluginRef: file-disc
+
+# ---- Per-role vLLM flags ----
+prefill:
+  tp: 1
+  enable-expert-parallel: true
+  extra-args: >-
+    --kv-cache-dtype fp8
+    --gpu-memory-utilization 0.92
+    --max-num-batched-tokens 8192
+    --block-size 256
+    --tokenizer-mode deepseek_v4
+    --tool-call-parser deepseek_v4
+    --enable-auto-tool-choice
+    --reasoning-parser deepseek_v4
+    --moe-backend deep_gemm_mega_moe
+    --enable-ep-weight-filter
+    --enable-cumem-allocator
+    --no-disable-hybrid-kv-cache-manager
+    --no-enable-flashinfer-autotune
+    --numa-bind
+    --compilation-config {"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}
+    --speculative-config {"method":"dspark","num_speculative_tokens":5,"draft_sample_method":"probabilistic","attention_backend":"FLASHINFER_MLA_SPARSE_DSV4","enable_adaptive_verification":false}
+    --attention-config {"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}
+  env:
+    PYTHONHASHSEED: "0"
+    VLLM_USE_RUST_FRONTEND: "1"
+    VLLM_SERVER_DEV_MODE: "1"
+    VLLM_ENGINE_READY_TIMEOUT_S: "3600"
+    VLLM_RPC_TIMEOUT: "600000"
+    VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768"
+    VLLM_DSV4_MEGA_FP8_COMBINE: "1"
+    VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800"
+    VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1"
+    TILELANG_CLEANUP_TEMP_FILES: "1"
+    VLLM_USE_NCCL_SYMM_MEM: "1"
+    VLLM_MOONCAKE_LOAD_RECV_THREADS: "20"
+    MC_ENABLE_DEST_DEVICE_AFFINITY: "1"
+    MC_STORE_CLIENT_METRIC: "1"
+    MC_STORE_CLIENT_METRIC_INTERVAL: "5"
+    MC_TE_METRIC: "0"
+
+# ---- Mooncake KV store config ----
+mooncake:
+  store_config:
+    metadata_server: "P2PHANDSHAKE"
+    local_buffer_size: "4GB"
+    protocol: "rdma"
+    device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3"
+    mode: "embedded"
+    enable_offload: false  # SSD only; the embedded DRAM pool is still enabled.
+
+# ---- SLURM resource directives ----
+slurm:
+  time_limit: "08:00:00"

benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-tp8-dspark-agentic.yaml

@@ -0,0 +1,73 @@
+# DeepSeek-V4-Pro-0813 (DSpark) FP4 GB200, aggregated TP8, 2 nodes.
+# DSpark speculative decoding: 5 tokens. No Mooncake (two-node TP8).
+apiVersion: llm-d.ai/v1alpha1
+kind: EndpointPickerConfig
+
+plugins:
+  - name: file-disc
+    type: file-discovery
+    parameters:
+      path: /tmp/endpoints.yaml
+      watchFile: false
+
+  - type: prefill-filter
+  - type: queue-scorer
+  - type: kv-cache-utilization-scorer
+  - type: active-request-scorer
+  - type: prefix-cache-scorer
+
+schedulingProfiles:
+  - name: default
+    plugins:
+      - pluginRef: prefill-filter
+      - pluginRef: queue-scorer
+        weight: 2
+      - pluginRef: kv-cache-utilization-scorer
+        weight: 2
+      - pluginRef: active-request-scorer
+        weight: 1
+      - pluginRef: prefix-cache-scorer
+        weight: 3
+
+dataLayer:
+  discovery:
+    pluginRef: file-disc
+
+prefill:
+  tp: 8
+  # Must be explicit false: server.sh defaults enable-expert-parallel to true.
+  enable-expert-parallel: false
+  extra-args: >-
+    --kv-cache-dtype fp8
+    --gpu-memory-utilization 0.92
+    --max-num-batched-tokens 8192
+    --block-size 256
+    --tokenizer-mode deepseek_v4
+    --tool-call-parser deepseek_v4
+    --enable-auto-tool-choice
+    --reasoning-parser deepseek_v4
+    --disable-custom-all-reduce
+    --enable-cumem-allocator
+    --no-disable-hybrid-kv-cache-manager
+    --no-enable-flashinfer-autotune
+    --compilation-config {"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}
+    --speculative-config {"method":"dspark","num_speculative_tokens":5,"draft_sample_method":"probabilistic","attention_backend":"FLASHINFER_MLA_SPARSE_DSV4","enable_adaptive_verification":false}
+    --attention-config {"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}
+  env:
+    VLLM_USE_RUST_FRONTEND: "1"
+    VLLM_SERVER_DEV_MODE: "1"
+    VLLM_ENGINE_READY_TIMEOUT_S: "3600"
+    VLLM_RPC_TIMEOUT: "600000"
+    VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768"
+    VLLM_DSV4_MEGA_FP8_COMBINE: "1"
+    VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800"
+    VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024"
+    TILELANG_CLEANUP_TEMP_FILES: "1"
+    VLLM_USE_NCCL_SYMM_MEM: "0"  # TP8 intra-node allreduce; DEP8 recipe uses "1"
+    VLLM_ALLREDUCE_USE_SYMM_MEM: "0"
+    VLLM_ALLREDUCE_USE_FLASHINFER: "1"
+    VLLM_FLASHINFER_ALLREDUCE_BACKEND: "auto"
+    NCCL_P2P_LEVEL: "NVL"
+
+slurm:
+  time_limit: "08:00:00"