diff --git a/benchmarks/single_node/agentic/qwen3.5_fp4_mi355x_sglang_mtp.sh b/benchmarks/single_node/agentic/qwen3.5_fp4_mi355x_sglang_mtp.sh index 2cdcaa8cf..e9e3d5eb5 100644 --- a/benchmarks/single_node/agentic/qwen3.5_fp4_mi355x_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/qwen3.5_fp4_mi355x_sglang_mtp.sh @@ -82,14 +82,15 @@ if [ "$TP" -ge 4 ]; then fi MAX_RUNNING_REQUESTS=$((2 * CONC)) -CUDA_GRAPH_MAX_BS="$CONC" -[ "$CUDA_GRAPH_MAX_BS" -gt 64 ] && CUDA_GRAPH_MAX_BS=64 +CUDA_GRAPH_MAX_BS=$MAX_RUNNING_REQUESTS +[ "$CUDA_GRAPH_MAX_BS" -gt 128 ] && CUDA_GRAPH_MAX_BS=128 export PYTHONNOUSERSITE=1 export SGLANG_USE_AITER=1 export SGLANG_USE_AITER_UNIFIED_ATTN=1 export AITER_FLYDSL_FORCE=1 export SGLANG_MAMBA_SSM_DTYPE=bfloat16 +export ROCM_QUICK_REDUCE_QUANTIZATION=INT8 export SGLANG_TIMEOUT_KEEP_ALIVE=1800 if [ "${EVAL_ONLY:-false}" != "true" ]; then @@ -114,8 +115,8 @@ SGLANG_CMD=( --kv-cache-dtype fp8_e4m3 --cuda-graph-max-bs "$CUDA_GRAPH_MAX_BS" --max-running-requests "$MAX_RUNNING_REQUESTS" - --max-prefill-tokens 32768 - --chunked-prefill-size 32768 + --max-prefill-tokens 16384 + --chunked-prefill-size 16384 --scheduler-recv-interval "$SCHEDULER_RECV_INTERVAL" --stream-interval 50 "${TOKENIZER_ARGS[@]}" diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 200096d5c..03dc2b11f 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -361,7 +361,7 @@ qwen3.5-fp4-mi355x-sglang-mtp: - { tp: 4, conc-start: 4, conc-end: 16, spec-decoding: mtp } qwen3.5-fp4-mi355x-sglang-agentic-mtp: - image: lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260818 + image: lmsysorg/sglang-rocm:v0.5.18-rocm720-mi35x-20260829 model: amd/Qwen3.5-397B-A17B-MXFP4 model-prefix: qwen3.5 runner: cluster:mi355x-amds diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 813aa3fb0..ead08729a 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6492,7 +6492,6 @@ - "Add HiCache host-DRAM KV tier arms at TP4 concurrency 40, 48, 56, and 64 and TP2 concurrency 20, 24, 28, and 32, using hicache ratio 1.5 with write_through, direct io, and page_first_direct layout." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2693 - - config-keys: - kimik2.6-fp4-b200-dynamo-vllm - dsv4-fp4-b200-dynamo-vllm @@ -6594,4 +6593,14 @@ description: - "Update the vLLM B200 Kimi-K3 AgentX configs." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2776 - + +- config-keys: + - qwen3.5-fp4-mi355x-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Bump image from lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260818 to lmsysorg/sglang-rocm:v0.5.18-rocm720-mi35x-20260829" + - "Route multi-GPU collectives through INT8-quantized ROCm quick all-reduce (ROCM_QUICK_REDUCE_QUANTIZATION=INT8), matching the published SGLang cookbook recipe for MXFP4 on MI355X (https://docs.sglang.io/cookbook/autoregressive/Qwen/Qwen3.5). This arm already omits --enable-aiter-allreduce-fusion, so the quick-reduce regime applies with no other launch change: the two paths are mutually exclusive and collectives only fall back to custom all-reduce, where the regime is read, when the fusion flag is off." + - "Halve the prefill budget, --max-prefill-tokens and --chunked-prefill-size from 32768 to 16384, matching the B200 sibling recipe qwen3.5_fp4_b200_sglang_mtp.sh." + - "Capture the decode CUDA graph to min(2*CONC, 128) instead of min(CONC, 64). The replay keeps --max-running-requests 2*CONC in flight, so a CONC-sized graph dropped every decode batch above CONC onto the eager path. The 128 cap follows the sibling MI355X AgentX recipe dsv4_fp4_mi355x_sglang_mtp.sh." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2737