[AgentX] DeepSeek-v4-pro llm-d NVL72-B200

benchmarks/llm-d/binaries-b200-v0.10.0.env

@@ -0,0 +1,33 @@
+# B200-only binary pins for the agentX-ported EPP config used by
+# dsv4-fp4-b200-agentx-1p1d-dep8-dep8.yaml (disagg-profile-handler
+# `deciders:` shape + inflight-load-producer / approx-prefix-cache-producer /
+# prefix-cache-affinity-filter / token-load-scorer plugins), which require
+# the v0.10.0 EPP/pd-sidecar router build. Kept as a sibling file (not a
+# change to binaries.env) so GB200's shared v0.9.0 pins and bin path are
+# completely untouched.
+#
+# Pull with extract-binaries.sh instead of binaries.env's defaults:
+#   BINARIES_ENV_FILE=binaries-b200-v0.10.0.env \
+#     benchmarks/llm-d/extract-binaries.sh
+# Run once (or whenever these pins change) on a host with docker + registry
+# access and the /home/sa-shared filesystem mounted.
+
+# --- Endpoint Picker (EPP / inference scheduler) ---
+EPP_FROM_IMAGE="ghcr.io/llm-d/llm-d-router-endpoint-picker:v0.10.0"
+EPP_BIN_PATH="/app/epp"
+
+# --- P/D routing sidecar (pd-sidecar) ---
+ROUTING_SIDECAR_IMAGE="ghcr.io/llm-d/llm-d-router-disagg-sidecar:v0.10.0"
+ROUTING_SIDECAR_BIN_PATH="/app/pd-sidecar"
+
+# --- Envoy front proxy ---
+# Unchanged from binaries.env; re-declared here so this file is
+# self-contained (extract-binaries.sh always extracts all three).
+ENVOY_FROM_IMAGE="envoyproxy/envoy:distroless-v1.33.2"
+ENVOY_BIN_PATH="/usr/local/bin/envoy"
+
+# B200 is x86_64 (unlike GB200's Grace/arm64, binaries.env's default).
+LLMD_BIN_PLATFORM="${LLMD_BIN_PLATFORM:-linux/amd64}"
+
+# Dedicated path so this never collides with GB200's shared bin dir.
+LLMD_BIN_DIR="${LLMD_BIN_DIR:-/home/sa-shared/llm-d-bins-v0.10.0}"

benchmarks/llm-d/extract-binaries.sh

@@ -13,12 +13,19 @@
 #   ./extract-binaries.sh                 # uses binaries.env defaults
 #   LLMD_BIN_DIR=/some/dir ./extract-binaries.sh
 #   LLMD_BIN_PLATFORM=linux/amd64 ./extract-binaries.sh   # for an x86 test
+#   BINARIES_ENV_FILE=binaries-b200-v0.10.0.env ./extract-binaries.sh
+#     # pull a different set of image pins (e.g. a router-version bump
+#     # scoped to one cluster) without touching the shared binaries.env
 
 set -euo pipefail
 
 HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+BINARIES_ENV_FILE="${BINARIES_ENV_FILE:-$HERE/binaries.env}"
+# Resolve a bare filename against this script's directory, so
+# BINARIES_ENV_FILE=foo.env works regardless of the caller's cwd.
+[[ "$BINARIES_ENV_FILE" != /* ]] && BINARIES_ENV_FILE="$HERE/$BINARIES_ENV_FILE"
 # shellcheck source=/dev/null
-source "$HERE/binaries.env"
+source "$BINARIES_ENV_FILE"
 
 echo "Extracting llm-d binaries -> $LLMD_BIN_DIR (platform $LLMD_BIN_PLATFORM)"
 mkdir -p "$LLMD_BIN_DIR"

benchmarks/multi_node/dsv4_fp4_b200_llmd-vllm-agg.sh

@@ -0,0 +1,64 @@
+#!/usr/bin/env bash
+#
+# Wrapper for the DeepSeek-V4-Pro B200 llmd-vllm aggregated benchmark
+# (agentX-flavored TP8 / DEP8, one engine does both prefill and decode).
+# Sibling of dsv4_fp4_b200_llmd-vllm-disagg.sh - same shape, but always
+# submits with DECODE_NODES=0 (no decode role at all; see the aggregated
+# recipes' header comments and server.sh's IS_AGGREGATED handling). The
+# runner resolves this script via
+#   SCRIPT_NAME="${EXP_NAME%%_*}_${PRECISION}_b200_llmd-vllm-agg.sh"
+# from launch_b200-dgxc-slurm.sh when DISAGG=false.
+
+set -euo pipefail
+
+source "$(dirname "$0")/../benchmark_lib.sh"
+
+check_env_vars \
+    CONC_LIST \
+    ISL \
+    OSL \
+    IMAGE \
+    MODEL_PATH \
+    PREFILL_NODES \
+    DECODE_NODES \
+    RANDOM_RANGE_RATIO
+
+if [[ -n "${SLURM_JOB_ID:-}" ]]; then
+    echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME"
+fi
+
+if [[ "$DECODE_NODES" != "0" ]]; then
+    echo "Error: dsv4_fp4_b200_llmd-vllm-agg.sh requires DECODE_NODES=0 (got $DECODE_NODES); aggregated mode has no decode role" >&2
+    exit 1
+fi
+
+set -x
+
+cd "$GITHUB_WORKSPACE/benchmarks/multi_node/llm-d" || exit 1
+
+# B200 DGX = 8 GPUs per node (submit.sh defaults to 8, explicit for clarity).
+export GPUS_PER_NODE="${GPUS_PER_NODE:-8}"
+
+export TIME_LIMIT="${TIME_LIMIT:-08:00:00}"
+export MODEL_PATH=$MODEL_PATH
+export MODEL_NAME=$MODEL_NAME
+export CONTAINER_IMAGE=$IMAGE
+
+# Aggregated is always a single engine (no multi-engine high-tpt split), so
+# PREFILL_WORKERS is always 1. DECODE_WORKERS is unused (DECODE_NODES=0) but
+# still exported since submit.sh/server.sh read it unconditionally.
+export PREFILL_WORKERS="${PREFILL_WORKERS:-1}"
+export DECODE_WORKERS="${DECODE_WORKERS:-1}"
+
+JOB_ID=$(bash ./submit.sh \
+    "$PREFILL_NODES" \
+    "$DECODE_NODES" \
+    "$ISL" "$OSL" "${CONC_LIST// /x}" inf \
+    "$RANDOM_RANGE_RATIO")
+
+if [[ -z "$JOB_ID" ]]; then
+    echo "Failed to submit job" >&2
+    exit 1
+fi
+
+echo "$JOB_ID"

benchmarks/multi_node/dsv4_fp4_b200_llmd-vllm-disagg.sh

@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+#
+# Wrapper for the DeepSeek-V4-Pro B200 llmd-vllm P/D disagg benchmark
+# (agentX-flavored 1P-DEP8/1D-DEP8). Sibling of
+# dsv4_fp4_gb200_llmd-vllm-disagg.sh - same shape, different topology
+# (B200 = 8 GPUs/node, so each DEP8 role fits on ONE node; GB200 = 4
+# GPUs/node, role spans 2 nodes). The runner resolves this script via
+#   SCRIPT_NAME="${EXP_NAME%%_*}_${PRECISION}_b200_llmd-vllm-disagg.sh"
+# from launch_b200-dgxc-slurm.sh.
+
+set -euo pipefail
+
+source "$(dirname "$0")/../benchmark_lib.sh"
+
+check_env_vars \
+    CONC_LIST \
+    ISL \
+    OSL \
+    IMAGE \
+    MODEL_PATH \
+    PREFILL_NODES \
+    DECODE_NODES \
+    RANDOM_RANGE_RATIO
+
+if [[ -n "${SLURM_JOB_ID:-}" ]]; then
+    echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME"
+fi
+
+set -x
+
+cd "$GITHUB_WORKSPACE/benchmarks/multi_node/llm-d" || exit 1
+
+# B200 DGX = 8 GPUs per node (submit.sh defaults to 8, explicit for clarity).
+export GPUS_PER_NODE="${GPUS_PER_NODE:-8}"
+
+export TIME_LIMIT="${TIME_LIMIT:-08:00:00}"
+export MODEL_PATH=$MODEL_PATH
+export MODEL_NAME=$MODEL_NAME
+export CONTAINER_IMAGE=$IMAGE
+
+# Worker count per role (Option B multi-engine). Prefer an explicit
+# PREFILL_WORKERS/DECODE_WORKERS from the matrix additional-settings; else
+# fall back to the matrix num-worker fields (PREFILL_NUM_WORKERS/
+# DECODE_NUM_WORKERS); else 1 (single engine = unchanged 1P+1D). submit.sh
+# reads these.
+export PREFILL_WORKERS="${PREFILL_WORKERS:-${PREFILL_NUM_WORKERS:-1}}"
+export DECODE_WORKERS="${DECODE_WORKERS:-${DECODE_NUM_WORKERS:-1}}"
+
+JOB_ID=$(bash ./submit.sh \
+    "$PREFILL_NODES" \
+    "$DECODE_NODES" \
+    "$ISL" "$OSL" "${CONC_LIST// /x}" inf \
+    "$RANDOM_RANGE_RATIO")
+
+if [[ -z "$JOB_ID" ]]; then
+    echo "Failed to submit job" >&2
+    exit 1
+fi
+
+echo "$JOB_ID"

benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-agentic.yaml

@@ -0,0 +1,70 @@
+# DeepSeek-V4-Pro-0813 (DSpark) FP4 B200, aggregated DEP8, 1 node.
+# DSpark speculative decoding: 5 tokens. No Mooncake.
+apiVersion: llm-d.ai/v1alpha1
+kind: EndpointPickerConfig
+
+plugins:
+  - name: file-disc
+    type: file-discovery
+    parameters:
+      path: /tmp/endpoints.yaml
+      watchFile: false
+
+  - type: prefill-filter
+  - type: queue-scorer
+  - type: kv-cache-utilization-scorer
+  - type: active-request-scorer
+  - type: prefix-cache-scorer
+
+schedulingProfiles:
+  - name: default
+    plugins:
+      - pluginRef: prefill-filter
+      - pluginRef: queue-scorer
+        weight: 2
+      - pluginRef: kv-cache-utilization-scorer
+        weight: 2
+      - pluginRef: active-request-scorer
+        weight: 1
+      - pluginRef: prefix-cache-scorer
+        weight: 3
+
+dataLayer:
+  discovery:
+    pluginRef: file-disc
+
+prefill:
+  tp: 1
+  enable-expert-parallel: true
+  extra-args: >-
+    --kv-cache-dtype fp8
+    --gpu-memory-utilization 0.92
+    --max-model-len 9280
+    --max-num-seqs 64
+    --max-num-batched-tokens 8192
+    --max-cudagraph-capture-size 64
+    --block-size 256
+    --tokenizer-mode deepseek_v4
+    --moe-backend deep_gemm_mega_moe
+    --enable-ep-weight-filter
+    --enable-cumem-allocator
+    --no-disable-hybrid-kv-cache-manager
+    --no-enable-flashinfer-autotune
+    --numa-bind
+    --compilation-config {"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}
+    --speculative-config {"method":"dspark","num_speculative_tokens":5,"draft_sample_method":"probabilistic","attention_backend":"FLASHINFER_MLA_SPARSE_DSV4","enable_adaptive_verification":true}
+    --attention-config {"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}
+  env:
+    VLLM_USE_RUST_FRONTEND: "1"
+    VLLM_SERVER_DEV_MODE: "1"
+    VLLM_ENGINE_READY_TIMEOUT_S: "3600"
+    VLLM_RPC_TIMEOUT: "600000"
+    VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768"
+    VLLM_DSV4_MEGA_FP8_COMBINE: "1"
+    VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800"
+    VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1"
+    TILELANG_CLEANUP_TEMP_FILES: "1"
+    VLLM_USE_NCCL_SYMM_MEM: "1"
+
+slurm:
+  time_limit: "08:00:00"

benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-mooncake-agentic.yaml

@@ -0,0 +1,89 @@
+# DeepSeek-V4-Pro-0813 (DSpark) FP4 B200, aggregated DEP8 with Mooncake prefix-cache.
+# Sibling of agg-gb200-dep8-dspark-agentic.yaml; adds Mooncake (P2PHANDSHAKE embedded RDMA)
+# so server.sh wires MultiConnector (NixlConnector + MooncakeStoreConnector, kv_both role).
+apiVersion: llm-d.ai/v1alpha1
+kind: EndpointPickerConfig
+
+plugins:
+  - name: file-disc
+    type: file-discovery
+    parameters:
+      path: /tmp/endpoints.yaml
+      watchFile: false
+
+  - type: prefill-filter
+  - type: queue-scorer
+  - type: kv-cache-utilization-scorer
+  - type: active-request-scorer
+  - type: prefix-cache-scorer
+
+schedulingProfiles:
+  - name: default
+    plugins:
+      - pluginRef: prefill-filter
+      - pluginRef: queue-scorer
+        weight: 2
+      - pluginRef: kv-cache-utilization-scorer
+        weight: 2
+      - pluginRef: active-request-scorer
+        weight: 1
+      - pluginRef: prefix-cache-scorer
+        weight: 3
+
+dataLayer:
+  discovery:
+    pluginRef: file-disc
+
+# ---- Per-role vLLM flags ----
+prefill:
+  tp: 1
+  enable-expert-parallel: true
+  extra-args: >-
+    --kv-cache-dtype fp8
+    --gpu-memory-utilization 0.92
+    --max-model-len 9280
+    --max-num-seqs 64
+    --max-num-batched-tokens 8192
+    --max-cudagraph-capture-size 64
+    --block-size 256
+    --tokenizer-mode deepseek_v4
+    --moe-backend deep_gemm_mega_moe
+    --enable-ep-weight-filter
+    --enable-cumem-allocator
+    --no-disable-hybrid-kv-cache-manager
+    --no-enable-flashinfer-autotune
+    --numa-bind
+    --compilation-config {"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}
+    --speculative-config {"method":"dspark","num_speculative_tokens":5,"draft_sample_method":"probabilistic","attention_backend":"FLASHINFER_MLA_SPARSE_DSV4","enable_adaptive_verification":true}
+    --attention-config {"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}
+  env:
+    VLLM_USE_RUST_FRONTEND: "1"
+    VLLM_SERVER_DEV_MODE: "1"
+    VLLM_ENGINE_READY_TIMEOUT_S: "3600"
+    VLLM_RPC_TIMEOUT: "600000"
+    VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768"
+    VLLM_DSV4_MEGA_FP8_COMBINE: "1"
+    VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800"
+    VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1"
+    TILELANG_CLEANUP_TEMP_FILES: "1"
+    VLLM_USE_NCCL_SYMM_MEM: "1"
+    VLLM_MOONCAKE_LOAD_RECV_THREADS: "20"
+    MC_ENABLE_DEST_DEVICE_AFFINITY: "1"
+    MC_STORE_CLIENT_METRIC: "1"
+    MC_STORE_CLIENT_METRIC_INTERVAL: "5"
+    MC_TE_METRIC: "0"
+
+# ---- Mooncake KV store config ----
+mooncake:
+  store_config:
+    metadata_server: "P2PHANDSHAKE"
+    global_segment_size: "140GB"
+    local_buffer_size: "4GB"
+    protocol: "rdma"
+    device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3"
+    mode: "embedded"
+    enable_offload: false
+
+# ---- SLURM resource directives ----
+slurm:
+  time_limit: "08:00:00"