diff --git a/benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_mi355x.sh b/benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_mi355x.sh index 87809f54b6..f31cb41b14 100644 --- a/benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_mi355x.sh +++ b/benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_mi355x.sh @@ -21,6 +21,7 @@ export SGLANG_USE_AITER=1 export SGLANG_USE_AITER_UNIFIED_ATTN=1 export AITER_FLYDSL_FORCE=1 export SGLANG_MAMBA_SSM_DTYPE=bfloat16 +export SGLANG_USE_AITER_FP8_PER_TOKEN=1 SERVER_LOG=/workspace/server.log MEM_FRAC_STATIC=${MEM_FRAC_STATIC:-0.8} diff --git a/benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_mi355x_mtp.sh b/benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_mi355x_mtp.sh index 16e86e5443..3c3ef258a4 100755 --- a/benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_mi355x_mtp.sh +++ b/benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_mi355x_mtp.sh @@ -21,6 +21,7 @@ export SGLANG_USE_AITER=1 export SGLANG_USE_AITER_UNIFIED_ATTN=1 export AITER_FLYDSL_FORCE=1 export SGLANG_MAMBA_SSM_DTYPE=bfloat16 +export SGLANG_USE_AITER_FP8_PER_TOKEN=1 SERVER_LOG=/workspace/server.log MEM_FRAC_STATIC=${MEM_FRAC_STATIC:-0.8} diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 2195bad7e2..cd19b52a38 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -314,7 +314,7 @@ qwen3.5-fp8-mi355x-sglang-disagg: qwen3.5-fp4-mi355x-sglang: image: lmsysorg/sglang-rocm:v0.5.16-rocm720-mi35x-20260730 - model: amd/Qwen3.5-397B-A17B-MXFP4 + model: amd/Qwen3.5-397B-A17B-MXFP4-AttnFP8-V2 model-prefix: qwen3.5 runner: mi355x precision: fp4 @@ -346,7 +346,7 @@ qwen3.5-fp4-mi355x-atom: qwen3.5-fp4-mi355x-sglang-mtp: image: lmsysorg/sglang-rocm:v0.5.16-rocm720-mi35x-20260730 - model: amd/Qwen3.5-397B-A17B-MXFP4 + model: amd/Qwen3.5-397B-A17B-MXFP4-AttnFP8-V2 model-prefix: qwen3.5 runner: mi355x precision: fp4 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index d90a270383..1497ac0bb9 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6409,3 +6409,13 @@ - "Require dcgm-power telemetry on the GLM-5.2 H200 disaggregated AgentX recipe." - "Enable dcgm-power telemetry on the DSV4 H200 aggregated AgentX recipe." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2684 + +- config-keys: + - qwen3.5-fp4-mi355x-sglang + - qwen3.5-fp4-mi355x-sglang-mtp + scenario-type: + - fixed-seq-len + description: + - "Switch the served checkpoint from amd/Qwen3.5-397B-A17B-MXFP4 to amd/Qwen3.5-397B-A17B-MXFP4-AttnFP8-V2 on both arms." + - "Export SGLANG_USE_AITER_FP8_PER_TOKEN=1 on both arms, as the checkpoint's published SGLang recipe requires." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2695