[AgentX] DeepSeek-v4-pro llm-d NVL72-B200

benchmarks/llm-d/binaries-b200-v0.10.0.env

@@ -0,0 +1,33 @@
+# B200-only binary pins for the agentX-ported EPP config used by
+# dsv4-fp4-b200-agentx-1p1d-dep8-dep8.yaml (disagg-profile-handler
+# `deciders:` shape + inflight-load-producer / approx-prefix-cache-producer /
+# prefix-cache-affinity-filter / token-load-scorer plugins), which require
+# the v0.10.0 EPP/pd-sidecar router build. Kept as a sibling file (not a
+# change to binaries.env) so GB200's shared v0.9.0 pins and bin path are
+# completely untouched.
+#
+# Pull with extract-binaries.sh instead of binaries.env's defaults:
+#   BINARIES_ENV_FILE=binaries-b200-v0.10.0.env \
+#     benchmarks/llm-d/extract-binaries.sh
+# Run once (or whenever these pins change) on a host with docker + registry
+# access and the /home/sa-shared filesystem mounted.
+
+# --- Endpoint Picker (EPP / inference scheduler) ---
+EPP_FROM_IMAGE="ghcr.io/llm-d/llm-d-router-endpoint-picker:v0.10.0"
+EPP_BIN_PATH="/app/epp"
+
+# --- P/D routing sidecar (pd-sidecar) ---
+ROUTING_SIDECAR_IMAGE="ghcr.io/llm-d/llm-d-router-disagg-sidecar:v0.10.0"
+ROUTING_SIDECAR_BIN_PATH="/app/pd-sidecar"
+
+# --- Envoy front proxy ---
+# Unchanged from binaries.env; re-declared here so this file is
+# self-contained (extract-binaries.sh always extracts all three).
+ENVOY_FROM_IMAGE="envoyproxy/envoy:distroless-v1.33.2"
+ENVOY_BIN_PATH="/usr/local/bin/envoy"
+
+# B200 is x86_64 (unlike GB200's Grace/arm64, binaries.env's default).
+LLMD_BIN_PLATFORM="${LLMD_BIN_PLATFORM:-linux/amd64}"
+
+# Dedicated path so this never collides with GB200's shared bin dir.
+LLMD_BIN_DIR="${LLMD_BIN_DIR:-/home/sa-shared/llm-d-bins-v0.10.0}"

benchmarks/llm-d/extract-binaries.sh

@@ -13,12 +13,19 @@
 #   ./extract-binaries.sh                 # uses binaries.env defaults
 #   LLMD_BIN_DIR=/some/dir ./extract-binaries.sh
 #   LLMD_BIN_PLATFORM=linux/amd64 ./extract-binaries.sh   # for an x86 test
+#   BINARIES_ENV_FILE=binaries-b200-v0.10.0.env ./extract-binaries.sh
+#     # pull a different set of image pins (e.g. a router-version bump
+#     # scoped to one cluster) without touching the shared binaries.env
 
 set -euo pipefail
 
 HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+BINARIES_ENV_FILE="${BINARIES_ENV_FILE:-$HERE/binaries.env}"
+# Resolve a bare filename against this script's directory, so
+# BINARIES_ENV_FILE=foo.env works regardless of the caller's cwd.
+[[ "$BINARIES_ENV_FILE" != /* ]] && BINARIES_ENV_FILE="$HERE/$BINARIES_ENV_FILE"
 # shellcheck source=/dev/null
-source "$HERE/binaries.env"
+source "$BINARIES_ENV_FILE"
 
 echo "Extracting llm-d binaries -> $LLMD_BIN_DIR (platform $LLMD_BIN_PLATFORM)"
 mkdir -p "$LLMD_BIN_DIR"

benchmarks/multi_node/dsv4_fp4_b200_llmd-vllm-agg.sh

@@ -0,0 +1,64 @@
+#!/usr/bin/env bash
+#
+# Wrapper for the DeepSeek-V4-Pro B200 llmd-vllm aggregated benchmark
+# (agentX-flavored TP8 / DEP8, one engine does both prefill and decode).
+# Sibling of dsv4_fp4_b200_llmd-vllm-disagg.sh - same shape, but always
+# submits with DECODE_NODES=0 (no decode role at all; see the aggregated
+# recipes' header comments and server.sh's IS_AGGREGATED handling). The
+# runner resolves this script via
+#   SCRIPT_NAME="${EXP_NAME%%_*}_${PRECISION}_b200_llmd-vllm-agg.sh"
+# from launch_b200-dgxc-slurm.sh when DISAGG=false.
+
+set -euo pipefail
+
+source "$(dirname "$0")/../benchmark_lib.sh"
+
+check_env_vars \
+    CONC_LIST \
+    ISL \
+    OSL \
+    IMAGE \
+    MODEL_PATH \
+    PREFILL_NODES \
+    DECODE_NODES \
+    RANDOM_RANGE_RATIO
+
+if [[ -n "${SLURM_JOB_ID:-}" ]]; then
+    echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME"
+fi
+
+if [[ "$DECODE_NODES" != "0" ]]; then
+    echo "Error: dsv4_fp4_b200_llmd-vllm-agg.sh requires DECODE_NODES=0 (got $DECODE_NODES); aggregated mode has no decode role" >&2
+    exit 1
+fi
+
+set -x
+
+cd "$GITHUB_WORKSPACE/benchmarks/multi_node/llm-d" || exit 1
+
+# B200 DGX = 8 GPUs per node (submit.sh defaults to 8, explicit for clarity).
+export GPUS_PER_NODE="${GPUS_PER_NODE:-8}"
+
+export TIME_LIMIT="${TIME_LIMIT:-08:00:00}"
+export MODEL_PATH=$MODEL_PATH
+export MODEL_NAME=$MODEL_NAME
+export CONTAINER_IMAGE=$IMAGE
+
+# Aggregated is always a single engine (no multi-engine high-tpt split), so
+# PREFILL_WORKERS is always 1. DECODE_WORKERS is unused (DECODE_NODES=0) but
+# still exported since submit.sh/server.sh read it unconditionally.
+export PREFILL_WORKERS="${PREFILL_WORKERS:-1}"
+export DECODE_WORKERS="${DECODE_WORKERS:-1}"
+
+JOB_ID=$(bash ./submit.sh \
+    "$PREFILL_NODES" \
+    "$DECODE_NODES" \
+    "$ISL" "$OSL" "${CONC_LIST// /x}" inf \
+    "$RANDOM_RANGE_RATIO")
+
+if [[ -z "$JOB_ID" ]]; then
+    echo "Failed to submit job" >&2
+    exit 1
+fi
+
+echo "$JOB_ID"

benchmarks/multi_node/dsv4_fp4_b200_llmd-vllm-disagg.sh

@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+#
+# Wrapper for the DeepSeek-V4-Pro B200 llmd-vllm P/D disagg benchmark
+# (agentX-flavored 1P-DEP8/1D-DEP8). Sibling of
+# dsv4_fp4_gb200_llmd-vllm-disagg.sh - same shape, different topology
+# (B200 = 8 GPUs/node, so each DEP8 role fits on ONE node; GB200 = 4
+# GPUs/node, role spans 2 nodes). The runner resolves this script via
+#   SCRIPT_NAME="${EXP_NAME%%_*}_${PRECISION}_b200_llmd-vllm-disagg.sh"
+# from launch_b200-dgxc-slurm.sh.
+
+set -euo pipefail
+
+source "$(dirname "$0")/../benchmark_lib.sh"
+
+check_env_vars \
+    CONC_LIST \
+    ISL \
+    OSL \
+    IMAGE \
+    MODEL_PATH \
+    PREFILL_NODES \
+    DECODE_NODES \
+    RANDOM_RANGE_RATIO
+
+if [[ -n "${SLURM_JOB_ID:-}" ]]; then
+    echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME"
+fi
+
+set -x
+
+cd "$GITHUB_WORKSPACE/benchmarks/multi_node/llm-d" || exit 1
+
+# B200 DGX = 8 GPUs per node (submit.sh defaults to 8, explicit for clarity).
+export GPUS_PER_NODE="${GPUS_PER_NODE:-8}"
+
+export TIME_LIMIT="${TIME_LIMIT:-08:00:00}"
+export MODEL_PATH=$MODEL_PATH
+export MODEL_NAME=$MODEL_NAME
+export CONTAINER_IMAGE=$IMAGE
+
+# Worker count per role (Option B multi-engine). Prefer an explicit
+# PREFILL_WORKERS/DECODE_WORKERS from the matrix additional-settings; else
+# fall back to the matrix num-worker fields (PREFILL_NUM_WORKERS/
+# DECODE_NUM_WORKERS); else 1 (single engine = unchanged 1P+1D). submit.sh
+# reads these.
+export PREFILL_WORKERS="${PREFILL_WORKERS:-${PREFILL_NUM_WORKERS:-1}}"
+export DECODE_WORKERS="${DECODE_WORKERS:-${DECODE_NUM_WORKERS:-1}}"
+
+JOB_ID=$(bash ./submit.sh \
+    "$PREFILL_NODES" \
+    "$DECODE_NODES" \
+    "$ISL" "$OSL" "${CONC_LIST// /x}" inf \
+    "$RANDOM_RANGE_RATIO")
+
+if [[ -z "$JOB_ID" ]]; then
+    echo "Failed to submit job" >&2
+    exit 1
+fi
+
+echo "$JOB_ID"

benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-agentic.yaml

@@ -0,0 +1,114 @@
+# DeepSeek-V4-Pro (FP4) on B200, aggregated DEP8 (one engine does both
+# prefill and decode) via llmd-vllm. agentX-flavored sibling of
+# dsv4-fp4-b200-agentx-agg-tp8.yaml (same aggregated wiring, DP+EP instead
+# of pure TP).
+#
+# Topology: B200 = 8 GPUs/node, so DEP8 (DP=8, EP on) fits on exactly ONE
+# node (unlike GB200's 4 GPUs/node, where agentX's ix-agg-dep-base.yaml/
+# agg-dep8.yaml needs `lws: {size: 2, replicas: 1}` spanning 2 nodes via
+# hybrid-lb). PREFILL_NODES=1 DECODE_NODES=0 GPUS_PER_NODE=8 -> 1 node / 8
+# GPUs total. Since the role never spans >1 node, server.sh's
+# LWS_GROUP_SIZE is always 1 here, so the cross-node --data-parallel-hybrid-lb
+# / NVSHMEM-ibgda code paths never activate - vLLM runs its DP=8 ranks
+# in-process on the one node.
+#
+# Aggregated-mode wiring: see the header comment in
+# dsv4-fp4-b200-agentx-agg-tp8.yaml (no decode role / no pd-sidecar / no
+# kv_transfer_config, engine mapped to the "prefill" role, rank 0 doubles
+# as EPP/Envoy/bench coordinator). The EPP config below is identical in
+# shape to that recipe's (same rationale: agentX's aggregated K8s config has
+# no llm-d EPP schema to port, so this is a purpose-built minimal
+# single-profile config, no v0.10.0 router bump needed).
+#
+# Source (agentX): llm-manifesto/models/deepseek-v4-ix/ix-agg-dep-base.yaml
+# + agg-dep8.yaml. Dropped entirely (no InferenceX equivalent needed):
+# the `mooncake:` sidecar block and the MultiConnector
+# NixlConnector+SimpleCPUOffloadConnector+MooncakeStoreConnector KV chain
+# (aggregated has no P/D KV transfer at all, see above), and the
+# Mooncake/UCX-specific env (VLLM_MOONCAKE_LOAD_RECV_THREADS,
+# MC_ENABLE_DEST_DEVICE_AFFINITY, UCX_TLS=rc,cuda_copy) that existed only to
+# support that KV chain. max_model_len trimmed to 9280 (ISL 8192 + OSL 1024
+# + headroom) instead of agentX's general-purpose 1048576, matching every
+# other recipe in this directory.
+apiVersion: llm-d.ai/v1alpha1
+kind: EndpointPickerConfig
+
+plugins:
+  - name: file-disc
+    type: file-discovery
+    parameters:
+      path: /tmp/endpoints.yaml
+      watchFile: false
+
+  - type: prefill-filter
+  - type: queue-scorer
+  - type: kv-cache-utilization-scorer
+  - type: active-request-scorer
+  - type: prefix-cache-scorer
+
+schedulingProfiles:
+  - name: default
+    plugins:
+      - pluginRef: prefill-filter
+      - pluginRef: queue-scorer
+        weight: 2
+      - pluginRef: kv-cache-utilization-scorer
+        weight: 2
+      - pluginRef: active-request-scorer
+        weight: 1
+      - pluginRef: prefix-cache-scorer
+        weight: 3
+
+dataLayer:
+  discovery:
+    pluginRef: file-disc
+
+# ---- Per-role vLLM flags ----
+# Ported from agentX's ix-agg-dep-base.yaml (roles[0], extended by
+# agg-dep8.yaml). DEP8 (TP=1, DP=8, EP on), FLASHINFER_MLA_SPARSE_DSV4
+# attention, deep_gemm_mega_moe MoE backend, EP weight filter (keeps only
+# served experts resident - required for DEP8 to avoid OOM), matching the
+# disagg recipe's DEP8 roles. tp/enable-expert-parallel are explicit here
+# even though they match server.sh's defaults, for self-documentation.
+#
+# No `decode:` block: server.sh only ever reads the section named after the
+# node's own ROLE, which is always "prefill" for this recipe (DECODE_NODES=0).
+prefill:
+  tp: 1
+  enable-expert-parallel: true
+  extra-args: >-
+    --kv-cache-dtype fp8
+    --gpu-memory-utilization 0.92
+    --max-model-len 9280
+    --max-num-seqs 64
+    --max-num-batched-tokens 8192
+    --max-cudagraph-capture-size 64
+    --block-size 256
+    --tokenizer-mode deepseek_v4
+    --moe-backend deep_gemm_mega_moe
+    --enable-ep-weight-filter
+    --enable-cumem-allocator
+    --no-disable-hybrid-kv-cache-manager
+    --no-enable-flashinfer-autotune
+    --numa-bind
+    --compilation-config {"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}
+    --speculative-config {"method":"dspark","num_speculative_tokens":5,"draft_sample_method":"probabilistic","attention_backend":"FLASHINFER_MLA_SPARSE_DSV4","enable_adaptive_verification":true}
+    --attention-config {"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}
+  env:
+    VLLM_USE_RUST_FRONTEND: "1"
+    VLLM_SERVER_DEV_MODE: "1"
+    VLLM_ENGINE_READY_TIMEOUT_S: "3600"
+    VLLM_RPC_TIMEOUT: "600000"
+    VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768"
+    VLLM_DSV4_MEGA_FP8_COMBINE: "1"
+    VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800"
+    VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1"
+    TILELANG_CLEANUP_TEMP_FILES: "1"
+    # Differs from agg-tp8.yaml's "0" - ported as-is from
+    # ix-agg-dep-base.yaml (DEP8's DP-ranks-as-processes path apparently
+    # wants NCCL symmetric memory enabled, unlike pure TP8's allreduce path).
+    VLLM_USE_NCCL_SYMM_MEM: "1"
+
+# ---- SLURM resource directives ----
+slurm:
+  time_limit: "08:00:00"

benchmarks/multi_node/llm-d-recipes/agentic/agg-gb200-dep8-dspark-mooncake-agentic.yaml

@@ -0,0 +1,96 @@
+# DeepSeek-V4-Pro (FP4) on B200, aggregated DEP8 with Mooncake prefix-cache
+# KV store enabled. Sibling of agg-gb200-dep8-dspark-agentic.yaml; adds a
+# `mooncake:` section so server.sh wires a MultiConnector
+# (NixlConnector + MooncakeStoreConnector) for cross-run prefix-cache sharing
+# via RDMA between DP ranks. See server.sh's aggregated+Mooncake branch for
+# the kv_both role assignment.
+#
+# EPP config and per-role vLLM flags are identical to the base dep8 recipe.
+# Mooncake store config (P2PHANDSHAKE embedded mode, 140 GB segment) matches
+# the disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml recipe.
+apiVersion: llm-d.ai/v1alpha1
+kind: EndpointPickerConfig
+
+plugins:
+  - name: file-disc
+    type: file-discovery
+    parameters:
+      path: /tmp/endpoints.yaml
+      watchFile: false
+
+  - type: prefill-filter
+  - type: queue-scorer
+  - type: kv-cache-utilization-scorer
+  - type: active-request-scorer
+  - type: prefix-cache-scorer
+
+schedulingProfiles:
+  - name: default
+    plugins:
+      - pluginRef: prefill-filter
+      - pluginRef: queue-scorer
+        weight: 2
+      - pluginRef: kv-cache-utilization-scorer
+        weight: 2
+      - pluginRef: active-request-scorer
+        weight: 1
+      - pluginRef: prefix-cache-scorer
+        weight: 3
+
+dataLayer:
+  discovery:
+    pluginRef: file-disc
+
+# ---- Per-role vLLM flags ----
+prefill:
+  tp: 1
+  enable-expert-parallel: true
+  extra-args: >-
+    --kv-cache-dtype fp8
+    --gpu-memory-utilization 0.92
+    --max-model-len 9280
+    --max-num-seqs 64
+    --max-num-batched-tokens 8192
+    --max-cudagraph-capture-size 64
+    --block-size 256
+    --tokenizer-mode deepseek_v4
+    --moe-backend deep_gemm_mega_moe
+    --enable-ep-weight-filter
+    --enable-cumem-allocator
+    --no-disable-hybrid-kv-cache-manager
+    --no-enable-flashinfer-autotune
+    --numa-bind
+    --compilation-config {"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}
+    --speculative-config {"method":"dspark","num_speculative_tokens":5,"draft_sample_method":"probabilistic","attention_backend":"FLASHINFER_MLA_SPARSE_DSV4","enable_adaptive_verification":true}
+    --attention-config {"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}
+  env:
+    VLLM_USE_RUST_FRONTEND: "1"
+    VLLM_SERVER_DEV_MODE: "1"
+    VLLM_ENGINE_READY_TIMEOUT_S: "3600"
+    VLLM_RPC_TIMEOUT: "600000"
+    VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768"
+    VLLM_DSV4_MEGA_FP8_COMBINE: "1"
+    VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800"
+    VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1"
+    TILELANG_CLEANUP_TEMP_FILES: "1"
+    VLLM_USE_NCCL_SYMM_MEM: "1"
+    VLLM_MOONCAKE_LOAD_RECV_THREADS: "20"
+    MC_ENABLE_DEST_DEVICE_AFFINITY: "1"
+    MC_STORE_CLIENT_METRIC: "1"
+    MC_STORE_CLIENT_METRIC_INTERVAL: "5"
+    MC_TE_METRIC: "0"
+
+# ---- Mooncake KV store config ----
+mooncake:
+  store_config:
+    metadata_server: "P2PHANDSHAKE"
+    global_segment_size: "140GB"
+    local_buffer_size: "4GB"
+    protocol: "rdma"
+    device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3"
+    mode: "embedded"
+    enable_offload: false
+
+# ---- SLURM resource directives ----
+slurm:
+  time_limit: "08:00:00"