diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx11_gen1_dep8_batch128_eplb0_mtp3_1229.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx11_gen1_dep8_batch128_eplb0_mtp3_1229.yaml new file mode 100644 index 0000000000..2c31d7e28b --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx11_gen1_dep8_batch128_eplb0_mtp3_1229.yaml @@ -0,0 +1,169 @@ +name: disagg-gb300-11p1d-dep1-dep8-c1229-mtp +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + precision: fp4 + +dynamo: + install: true + wheel: 1.4.0.dev20260807 + +identity: + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + frameworks: + tensorrt_llm: 1.3.0rc24 +slurm: + time_limit: 04:00:00 +health_check: + max_attempts: 540 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 3 + prefill_workers: 11 + gpus_per_prefill: 1 + decode_nodes: 2 + decode_workers: 1 + gpus_per_decode: 8 +backend: + type: trtllm + sequential_node_start: 2 + prefill_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' + CTX_LOAD_STAGGER_S: '180' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + decode_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + trtllm_config: + prefill: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 16384 + transceiver_runtime: PYTHON + cuda_graph_config: null + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.7 + max_batch_size: 2 + max_num_tokens: 16896 + max_seq_len: 8448 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 1 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + tensor_parallel_size: 1 + trust_remote_code: true + decode: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 16384 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 12 + - 16 + - 20 + - 24 + - 28 + - 32 + - 36 + - 40 + - 44 + - 48 + - 52 + - 56 + - 60 + - 64 + - 68 + - 72 + - 76 + - 80 + - 84 + - 88 + - 92 + - 96 + - 100 + - 104 + - 108 + - 112 + - 116 + - 120 + - 124 + - 128 + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + max_batch_size: 128 + max_num_tokens: 512 + max_seq_len: 9472 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + stream_interval: 100 + tensor_parallel_size: 8 + trust_remote_code: true +frontend: + type: dynamo + enable_multiple_frontends: true + env: + ETCD_LEASE_TTL: '120' + DYN_TOKENIZER: "fastokens" + +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: + - 1229 + req_rate: inf + num_prompts_mult: 16 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx17_gen2_dep8_batch64_eplb0_mtp3_1126.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx17_gen2_dep8_batch64_eplb0_mtp3_1126.yaml new file mode 100644 index 0000000000..4cc846e1bf --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx17_gen2_dep8_batch64_eplb0_mtp3_1126.yaml @@ -0,0 +1,151 @@ +name: disagg-gb300-17p2d-dep1-dep8-c1126-mtp +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + precision: fp4 + +dynamo: + install: true + wheel: 1.4.0.dev20260807 + +identity: + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + frameworks: + tensorrt_llm: 1.3.0rc24 +slurm: + time_limit: 04:00:00 +health_check: + max_attempts: 540 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 5 + prefill_workers: 17 + gpus_per_prefill: 1 + decode_nodes: 4 + decode_workers: 2 + gpus_per_decode: 8 +backend: + type: trtllm + sequential_node_start: 2 + prefill_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' + CTX_LOAD_STAGGER_S: '180' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + decode_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + trtllm_config: + prefill: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 16384 + cuda_graph_config: null + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.7 + max_batch_size: 2 + max_num_tokens: 16896 + max_seq_len: 8448 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 1 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + tensor_parallel_size: 1 + trust_remote_code: true + decode: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 16384 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 12 + - 16 + - 20 + - 24 + - 28 + - 32 + - 36 + - 40 + - 44 + - 48 + - 52 + - 56 + - 60 + - 64 + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + max_batch_size: 64 + max_num_tokens: 256 + max_seq_len: 9472 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + stream_interval: 100 + tensor_parallel_size: 8 + trust_remote_code: true +frontend: + type: dynamo + enable_multiple_frontends: true + env: + ETCD_LEASE_TTL: '120' + DYN_TOKENIZER: "fastokens" + +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: + - 1126 + req_rate: inf + num_prompts_mult: 16 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx1_gen2_tep8_batch8_eplb0_mtp3_20.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx1_gen2_tep8_batch8_eplb0_mtp3_20.yaml new file mode 100644 index 0000000000..b595a5ff87 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx1_gen2_tep8_batch8_eplb0_mtp3_20.yaml @@ -0,0 +1,134 @@ +name: disagg-gb300-1p2d-dep1-tep8-c20-mtp +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + precision: fp4 + +dynamo: + install: true + wheel: 1.4.0.dev20260807 + +identity: + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 +slurm: + time_limit: 04:00:00 +health_check: + max_attempts: 540 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 1 + decode_nodes: 4 + decode_workers: 2 + gpus_per_decode: 8 +backend: + type: trtllm + prefill_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' + CTX_LOAD_STAGGER_S: '180' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + decode_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + trtllm_config: + prefill: + cache_transceiver_config: + backend: UCX + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 16384 + cuda_graph_config: null + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + max_batch_size: 32 + max_num_tokens: 16896 + max_seq_len: 8448 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 1 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + tensor_parallel_size: 1 + trust_remote_code: true + decode: + cache_transceiver_config: + backend: UCX + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 16384 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + enable_padding: true + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + max_batch_size: 8 + max_num_tokens: 32 + max_seq_len: 9472 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + stream_interval: 100 + tensor_parallel_size: 8 + trust_remote_code: true +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: '120' + DYN_TOKENIZER: "fastokens" + +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: + - 20 + req_rate: inf + num_prompts_mult: 16 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx1dep2_gen4tp16_mtp3_c4.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx1_gen4_tep8_batch1_eplb0_mtp3_12.yaml similarity index 50% rename from benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx1dep2_gen4tp16_mtp3_c4.yaml rename to benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx1_gen4_tep8_batch1_eplb0_mtp3_12.yaml index c1fda5499a..f13e5392f4 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx1dep2_gen4tp16_mtp3_c4.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx1_gen4_tep8_batch1_eplb0_mtp3_12.yaml @@ -1,156 +1,133 @@ -# Derived from config #01 for one-request-per-decode-worker minimum-TPOT measurement. -# Reconstructed from successful source result: /lustre/fsw/coreai_comparch_infbench/minma/sa_qwen_disagg/exp3_8k1k_mtp3_0609_tot/results/e2e-clean/1ctxDEP2c128_4genTP16c4_E2E_C16/conc16 -# Variant runtime: srt-slurm f6eb42aee4664207dcf2ec601e3bd57bd527efd6 with official TensorRT-LLM 1.3.0rc18 container. -# Unlisted model, topology, resource, frontend, and engine fields remain identical to original-configs. -name: ctx1dep2_gen4tp16_mtp3_c4 -identity: - model: - repo: nvidia/Qwen3.5-397B-A17B-NVFP4 - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 - frameworks: - tensorrt_llm: 1.3.0rc18 +name: disagg-gb300-1p4d-dep1-tep8-c12-mtp model: path: qwen3.5-fp4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 precision: fp4 + +dynamo: + install: true + wheel: 1.4.0.dev20260807 + +identity: + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 +slurm: + time_limit: 04:00:00 +health_check: + max_attempts: 540 + interval_seconds: 10 resources: gpu_type: gb300 gpus_per_node: 4 prefill_nodes: 1 prefill_workers: 1 - gpus_per_prefill: 2 - decode_nodes: 16 + gpus_per_prefill: 1 + decode_nodes: 8 decode_workers: 4 - gpus_per_decode: 16 - het_jobs: false + gpus_per_decode: 8 backend: type: trtllm prefill_environment: + TLLM_LOG_LEVEL: INFO TRTLLM_SERVER_DISABLE_GC: '1' TRTLLM_WORKER_DISABLE_GC: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' TRTLLM_ENABLE_PDL: '1' - TLLM_LOG_LEVEL: INFO - TLLM_PROFILE_LOG_RANKS: all - TLLM_USE_FLASHINFER_GDN_PREFILL: '1' - HF_HUB_OFFLINE: '1' - PYTHONNOUSERSITE: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' + CTX_LOAD_STAGGER_S: '180' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp decode_environment: + TLLM_LOG_LEVEL: INFO TRTLLM_SERVER_DISABLE_GC: '1' TRTLLM_WORKER_DISABLE_GC: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' TRTLLM_ENABLE_PDL: '1' - TLLM_LOG_LEVEL: INFO - TLLM_PROFILE_LOG_RANKS: all - TLLM_USE_FLASHINFER_GDN_PREFILL: '0' - HF_HUB_OFFLINE: '1' - PYTHONNOUSERSITE: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp trtllm_config: prefill: - trust_remote_code: true - max_batch_size: 64 - max_num_tokens: 32768 - max_seq_len: 8395 - tensor_parallel_size: 2 - moe_expert_parallel_size: 2 - enable_attention_dp: true - pipeline_parallel_size: 1 - print_iter_log: true - enable_iter_perf_stats: true + cache_transceiver_config: + backend: UCX + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 16384 cuda_graph_config: null disable_overlap_scheduler: true - enable_chunked_prefill: true + enable_attention_dp: true kv_cache_config: + dtype: fp8 enable_block_reuse: false free_gpu_memory_fraction: 0.85 - dtype: fp8 + max_batch_size: 32 + max_num_tokens: 16896 + max_seq_len: 8448 moe_config: backend: CUTEDSL - use_low_precision_moe_combine: true + moe_expert_parallel_size: 1 nvfp4_gemm_config: allowed_backends: - cutlass - cublaslt - cutedsl - cuda_core - cache_transceiver_config: - max_tokens_in_buffer: 8192 - backend: UCX - # Match NVIDIA's 8k/1k high-concurrency disagg deadline and avoid rc18's unsafe 60s cancellation path. - kv_transfer_timeout_ms: 600000 - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - context_chunking_policy: FIRST_COME_FIRST_SERVED + pipeline_parallel_size: 1 + print_iter_log: true speculative_config: decoding_type: MTP max_draft_len: 3 - use_mtp_vanilla: false - decode: + tensor_parallel_size: 1 trust_remote_code: true - tensor_parallel_size: 16 - moe_expert_parallel_size: 1 - enable_attention_dp: false - enable_lm_head_tp_in_adp: false - pipeline_parallel_size: 1 - max_batch_size: 1 - max_num_tokens: 4 - max_seq_len: 9419 + decode: + cache_transceiver_config: + backend: UCX + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 16384 cuda_graph_config: - enable_padding: true batch_sizes: - 1 - print_iter_log: true - enable_iter_perf_stats: true + - 2 + - 4 + enable_padding: true + enable_attention_dp: false + enable_lm_head_tp_in_adp: false kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.8 dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + max_batch_size: 1 + max_num_tokens: 4 + max_seq_len: 9472 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 nvfp4_gemm_config: allowed_backends: - cutlass - cublaslt - cutedsl - cuda_core - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - cache_transceiver_config: - max_tokens_in_buffer: 8192 - backend: UCX - # Match NVIDIA's 8k/1k high-concurrency disagg deadline and avoid rc18's unsafe 60s cancellation path. - kv_transfer_timeout_ms: 600000 - stream_interval: 100 - num_postprocess_workers: 4 - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - context_chunking_policy: FIRST_COME_FIRST_SERVED + pipeline_parallel_size: 1 + print_iter_log: true speculative_config: decoding_type: MTP max_draft_len: 3 - use_mtp_vanilla: false + stream_interval: 100 + tensor_parallel_size: 8 + trust_remote_code: true frontend: type: dynamo enable_multiple_frontends: false env: ETCD_LEASE_TTL: '120' DYN_TOKENIZER: "fastokens" + benchmark: - type: "sa-bench" + type: sa-bench isl: 8192 osl: 1024 - concurrencies: "4" - req_rate: "inf" + concurrencies: + - 12 + req_rate: inf num_prompts_mult: 16 -dynamo: - install: true - version: 1.3.0.dev20260708 - request_plane: tcp -infra: - etcd_nats_dedicated_node: false -profiling: - type: none -health_check: - max_attempts: 720 - interval_seconds: 10 -enable_config_dump: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx1_gen4_tep8_batch2_eplb0_mtp3_8.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx1_gen4_tep8_batch2_eplb0_mtp3_8.yaml new file mode 100644 index 0000000000..58ece185ab --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx1_gen4_tep8_batch2_eplb0_mtp3_8.yaml @@ -0,0 +1,135 @@ +name: disagg-gb300-1p4d-dep1-tep8-c8-mtp +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + precision: fp4 + +dynamo: + install: true + wheel: 1.4.0.dev20260807 + +identity: + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + frameworks: + tensorrt_llm: 1.3.0rc24 +slurm: + time_limit: 04:00:00 +health_check: + max_attempts: 540 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 1 + decode_nodes: 8 + decode_workers: 4 + gpus_per_decode: 8 +backend: + type: trtllm + prefill_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' + CTX_LOAD_STAGGER_S: '180' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + decode_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + trtllm_config: + prefill: + cache_transceiver_config: + backend: UCX + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 16384 + cuda_graph_config: null + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + max_batch_size: 32 + max_num_tokens: 16896 + max_seq_len: 8448 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 1 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + tensor_parallel_size: 1 + trust_remote_code: true + decode: + cache_transceiver_config: + backend: UCX + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 16384 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + enable_padding: true + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + max_batch_size: 2 + max_num_tokens: 8 + max_seq_len: 9472 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + stream_interval: 100 + tensor_parallel_size: 8 + trust_remote_code: true +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: '120' + DYN_TOKENIZER: "fastokens" + +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: + - 8 + req_rate: inf + num_prompts_mult: 16 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx1dep2_gen1tep16_mtp3_c48.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx1dep2_gen1tep16_mtp3_c48.yaml deleted file mode 100644 index 9438243e28..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx1dep2_gen1tep16_mtp3_c48.yaml +++ /dev/null @@ -1,167 +0,0 @@ -# Reconstructed from successful source result: /lustre/fsw/coreai_comparch_infbench/minma/sa_qwen_disagg/exp3_8k1k_mtp3_0609_tot/results/e2e-clean/1ctxDEP2c128_1genTEP16c32_E2E_C32/conc32 -# Variant runtime: srt-slurm f6eb42aee4664207dcf2ec601e3bd57bd527efd6 with official TensorRT-LLM 1.3.0rc18 container. -# Unlisted model, topology, resource, frontend, and engine fields remain identical to original-configs. -name: ctx1dep2_gen1tep16_mtp3_c48 -identity: - model: - repo: nvidia/Qwen3.5-397B-A17B-NVFP4 - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 - frameworks: - tensorrt_llm: 1.3.0rc18 -model: - path: qwen3.5-fp4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 - precision: fp4 -resources: - gpu_type: gb300 - gpus_per_node: 4 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 2 - decode_nodes: 4 - decode_workers: 1 - gpus_per_decode: 16 - het_jobs: false -backend: - type: trtllm - prefill_environment: - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TRTLLM_ENABLE_PDL: '1' - TLLM_LOG_LEVEL: INFO - TLLM_PROFILE_LOG_RANKS: all - TLLM_USE_FLASHINFER_GDN_PREFILL: '1' - HF_HUB_OFFLINE: '1' - PYTHONNOUSERSITE: '1' - decode_environment: - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TRTLLM_ENABLE_PDL: '1' - TLLM_LOG_LEVEL: INFO - TLLM_PROFILE_LOG_RANKS: all - TLLM_USE_FLASHINFER_GDN_PREFILL: '0' - HF_HUB_OFFLINE: '1' - PYTHONNOUSERSITE: '1' - trtllm_config: - prefill: - trust_remote_code: true - max_batch_size: 64 - max_num_tokens: 32768 - max_seq_len: 8395 - tensor_parallel_size: 2 - moe_expert_parallel_size: 2 - enable_attention_dp: true - pipeline_parallel_size: 1 - print_iter_log: true - enable_iter_perf_stats: true - cuda_graph_config: null - disable_overlap_scheduler: true - enable_chunked_prefill: true - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - dtype: fp8 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - cache_transceiver_config: - max_tokens_in_buffer: 8192 - backend: UCX - # Match NVIDIA's 8k/1k high-concurrency disagg deadline and avoid rc18's unsafe 60s cancellation path. - kv_transfer_timeout_ms: 600000 - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - context_chunking_policy: FIRST_COME_FIRST_SERVED - speculative_config: - decoding_type: MTP - max_draft_len: 3 - use_mtp_vanilla: false - decode: - trust_remote_code: true - tensor_parallel_size: 16 - moe_expert_parallel_size: 16 - enable_attention_dp: false - enable_lm_head_tp_in_adp: false - pipeline_parallel_size: 1 - max_batch_size: 40 - max_num_tokens: 4096 - max_seq_len: 9419 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 16 - - 20 - - 24 - - 32 - - 64 - - 128 - - 256 - - 384 - - 512 - print_iter_log: true - enable_iter_perf_stats: true - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.8 - dtype: fp8 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - cache_transceiver_config: - max_tokens_in_buffer: 8192 - backend: UCX - # Match NVIDIA's 8k/1k high-concurrency disagg deadline and avoid rc18's unsafe 60s cancellation path. - kv_transfer_timeout_ms: 600000 - stream_interval: 100 - num_postprocess_workers: 4 - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - context_chunking_policy: FIRST_COME_FIRST_SERVED - speculative_config: - decoding_type: MTP - max_draft_len: 3 - use_mtp_vanilla: false -frontend: - type: dynamo - enable_multiple_frontends: false - env: - ETCD_LEASE_TTL: '120' - DYN_TOKENIZER: "fastokens" -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "48" - req_rate: "inf" - num_prompts_mult: 16 -dynamo: - install: true - version: 1.3.0.dev20260708 - request_plane: tcp -infra: - etcd_nats_dedicated_node: false -profiling: - type: none -health_check: - max_attempts: 720 - interval_seconds: 10 -enable_config_dump: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx1dep2_gen7tep8_mtp3_c42.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx1dep2_gen7tep8_mtp3_c42.yaml deleted file mode 100644 index f32dbc95a0..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx1dep2_gen7tep8_mtp3_c42.yaml +++ /dev/null @@ -1,168 +0,0 @@ -# Reconstructed from successful source result: /lustre/fsw/coreai_comparch_infbench/minma/sa_qwen_disagg/exp3_8k1k_mtp3_0609_tot/results/e2e-clean/1ctxDEP2c128_7genTEP8c4_E2E_C28/conc28 -# Variant runtime: srt-slurm f6eb42aee4664207dcf2ec601e3bd57bd527efd6 with official TensorRT-LLM 1.3.0rc18 container. -# Unlisted model, topology, resource, frontend, and engine fields remain identical to original-configs. -name: ctx1dep2_gen7tep8_mtp3_c42 -identity: - model: - repo: nvidia/Qwen3.5-397B-A17B-NVFP4 - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 - frameworks: - tensorrt_llm: 1.3.0rc18 -model: - path: qwen3.5-fp4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 - precision: fp4 -resources: - gpu_type: gb300 - gpus_per_node: 4 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 2 - decode_nodes: 14 - decode_workers: 7 - gpus_per_decode: 8 - het_jobs: false -backend: - type: trtllm - prefill_environment: - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TRTLLM_ENABLE_PDL: '1' - TLLM_LOG_LEVEL: INFO - TLLM_PROFILE_LOG_RANKS: all - TLLM_USE_FLASHINFER_GDN_PREFILL: '1' - HF_HUB_OFFLINE: '1' - PYTHONNOUSERSITE: '1' - decode_environment: - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TRTLLM_ENABLE_PDL: '1' - TLLM_LOG_LEVEL: INFO - TLLM_PROFILE_LOG_RANKS: all - TLLM_USE_FLASHINFER_GDN_PREFILL: '0' - HF_HUB_OFFLINE: '1' - PYTHONNOUSERSITE: '1' - trtllm_config: - prefill: - trust_remote_code: true - max_batch_size: 64 - max_num_tokens: 32768 - max_seq_len: 8395 - tensor_parallel_size: 2 - moe_expert_parallel_size: 2 - enable_attention_dp: true - pipeline_parallel_size: 1 - print_iter_log: true - enable_iter_perf_stats: true - cuda_graph_config: null - disable_overlap_scheduler: true - enable_chunked_prefill: true - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - dtype: fp8 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - cache_transceiver_config: - max_tokens_in_buffer: 8192 - backend: UCX - # Match NVIDIA's 8k/1k high-concurrency disagg deadline and avoid rc18's unsafe 60s cancellation path. - kv_transfer_timeout_ms: 600000 - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - context_chunking_policy: FIRST_COME_FIRST_SERVED - speculative_config: - decoding_type: MTP - max_draft_len: 3 - use_mtp_vanilla: false - decode: - trust_remote_code: true - tensor_parallel_size: 8 - moe_expert_parallel_size: 8 - enable_attention_dp: false - enable_lm_head_tp_in_adp: false - pipeline_parallel_size: 1 - max_batch_size: 5 - max_num_tokens: 4096 - max_seq_len: 9419 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 16 - - 20 - - 24 - - 32 - - 64 - - 128 - - 256 - - 384 - - 512 - print_iter_log: true - enable_iter_perf_stats: true - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.8 - dtype: fp8 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - cache_transceiver_config: - max_tokens_in_buffer: 8192 - backend: UCX - # Match NVIDIA's 8k/1k high-concurrency disagg deadline and avoid rc18's unsafe 60s cancellation path. - kv_transfer_timeout_ms: 600000 - stream_interval: 100 - num_postprocess_workers: 4 - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - context_chunking_policy: FIRST_COME_FIRST_SERVED - allreduce_strategy: MNNVL - speculative_config: - decoding_type: MTP - max_draft_len: 3 - use_mtp_vanilla: false -frontend: - type: dynamo - enable_multiple_frontends: false - env: - ETCD_LEASE_TTL: '120' - DYN_TOKENIZER: "fastokens" -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "42" - req_rate: "inf" - num_prompts_mult: 16 -dynamo: - install: true - version: 1.3.0.dev20260708 - request_plane: tcp -infra: - etcd_nats_dedicated_node: false -profiling: - type: none -health_check: - max_attempts: 720 - interval_seconds: 10 -enable_config_dump: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx24_gen1_dep16_batch512_eplb0_mtp0_8192.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx24_gen1_dep16_batch512_eplb0_mtp0_8192.yaml new file mode 100644 index 0000000000..5b3e044150 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx24_gen1_dep16_batch512_eplb0_mtp0_8192.yaml @@ -0,0 +1,194 @@ +name: disagg-gb300-24p1d-dep1-dep16-c8192-stp +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + precision: fp4 + +dynamo: + install: true + wheel: 1.4.0.dev20260807 + +identity: + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + frameworks: + tensorrt_llm: 1.3.0rc24 +slurm: + time_limit: 04:00:00 +health_check: + max_attempts: 540 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 6 + prefill_workers: 24 + gpus_per_prefill: 1 + decode_nodes: 4 + decode_workers: 1 + gpus_per_decode: 16 +backend: + type: trtllm + sequential_node_start: 2 + prefill_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' + CTX_LOAD_STAGGER_S: '180' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + decode_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + trtllm_config: + prefill: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 16384 + cuda_graph_config: null + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.7 + max_batch_size: 2 + max_num_tokens: 16896 + max_seq_len: 8448 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 1 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + tensor_parallel_size: 1 + trust_remote_code: true + decode: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 16384 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + - 40 + - 48 + - 56 + - 64 + - 72 + - 80 + - 88 + - 96 + - 104 + - 112 + - 120 + - 128 + - 136 + - 144 + - 152 + - 160 + - 168 + - 176 + - 184 + - 192 + - 200 + - 208 + - 216 + - 224 + - 232 + - 240 + - 248 + - 256 + - 264 + - 272 + - 280 + - 288 + - 296 + - 304 + - 312 + - 320 + - 328 + - 336 + - 344 + - 352 + - 360 + - 368 + - 376 + - 384 + - 392 + - 400 + - 408 + - 416 + - 424 + - 432 + - 440 + - 448 + - 456 + - 464 + - 472 + - 480 + - 488 + - 496 + - 504 + - 512 + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.75 + max_batch_size: 512 + max_num_tokens: 512 + max_seq_len: 9472 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + moe_expert_parallel_size: 16 + num_postprocess_workers: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + stream_interval: 100 + tensor_parallel_size: 16 + trust_remote_code: true +frontend: + type: dynamo + enable_multiple_frontends: true + env: + ETCD_LEASE_TTL: '120' + DYN_TOKENIZER: "fastokens" + +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: + - 8192 + req_rate: inf + num_prompts_mult: 16 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx26_gen1_dep16_batch256_eplb0_mtp2_4301.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx26_gen1_dep16_batch256_eplb0_mtp2_4301.yaml new file mode 100644 index 0000000000..e6915c5ef7 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx26_gen1_dep16_batch256_eplb0_mtp2_4301.yaml @@ -0,0 +1,201 @@ +name: disagg-gb300-26p1d-dep1-dep16-c4301-mtp +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + precision: fp4 + +dynamo: + install: true + wheel: 1.4.0.dev20260807 + +identity: + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + frameworks: + tensorrt_llm: 1.3.0rc24 +slurm: + time_limit: 04:00:00 +health_check: + max_attempts: 540 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 7 + prefill_workers: 26 + gpus_per_prefill: 1 + decode_nodes: 4 + decode_workers: 1 + gpus_per_decode: 16 +backend: + type: trtllm + sequential_node_start: 2 + prefill_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' + CTX_LOAD_STAGGER_S: '180' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + decode_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + trtllm_config: + prefill: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 16384 + transceiver_runtime: PYTHON + cuda_graph_config: null + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.7 + max_batch_size: 2 + max_num_tokens: 16896 + max_seq_len: 8448 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 1 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + max_draft_len: 2 + tensor_parallel_size: 1 + trust_remote_code: true + decode: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 16384 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 12 + - 16 + - 20 + - 24 + - 28 + - 32 + - 36 + - 40 + - 44 + - 48 + - 52 + - 56 + - 60 + - 64 + - 68 + - 72 + - 76 + - 80 + - 84 + - 88 + - 92 + - 96 + - 100 + - 104 + - 108 + - 112 + - 116 + - 120 + - 124 + - 128 + - 132 + - 136 + - 140 + - 144 + - 148 + - 152 + - 156 + - 160 + - 164 + - 168 + - 172 + - 176 + - 180 + - 184 + - 188 + - 192 + - 196 + - 200 + - 204 + - 208 + - 212 + - 216 + - 220 + - 224 + - 228 + - 232 + - 236 + - 240 + - 244 + - 248 + - 252 + - 256 + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.75 + max_batch_size: 256 + max_num_tokens: 768 + max_seq_len: 9472 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + moe_expert_parallel_size: 16 + num_postprocess_workers: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + max_draft_len: 2 + stream_interval: 100 + tensor_parallel_size: 16 + trust_remote_code: true +frontend: + type: dynamo + enable_multiple_frontends: true + env: + ETCD_LEASE_TTL: '120' + DYN_TOKENIZER: "fastokens" + +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: + - 4301 + req_rate: inf + num_prompts_mult: 16 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx2dep2_gen15tp4_mtp3_c90.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx2dep2_gen15tp4_mtp3_c90.yaml deleted file mode 100644 index 7cb1d46b14..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx2dep2_gen15tp4_mtp3_c90.yaml +++ /dev/null @@ -1,167 +0,0 @@ -# Reconstructed from successful source result: /lustre/fsw/coreai_comparch_infbench/minma/sa_qwen_disagg/exp3_8k1k_mtp3_0609_tot/results/e2e-clean/2ctxDEP2c128_15genTP4c4_E2E_C60/conc60 -# Variant runtime: srt-slurm f6eb42aee4664207dcf2ec601e3bd57bd527efd6 with official TensorRT-LLM 1.3.0rc18 container. -# Unlisted model, topology, resource, frontend, and engine fields remain identical to original-configs. -name: ctx2dep2_gen15tp4_mtp3_c90 -identity: - model: - repo: nvidia/Qwen3.5-397B-A17B-NVFP4 - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 - frameworks: - tensorrt_llm: 1.3.0rc18 -model: - path: qwen3.5-fp4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 - precision: fp4 -resources: - gpu_type: gb300 - gpus_per_node: 4 - prefill_nodes: 1 - prefill_workers: 2 - gpus_per_prefill: 2 - decode_nodes: 15 - decode_workers: 15 - gpus_per_decode: 4 - het_jobs: false -backend: - type: trtllm - prefill_environment: - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TRTLLM_ENABLE_PDL: '1' - TLLM_LOG_LEVEL: INFO - TLLM_PROFILE_LOG_RANKS: all - TLLM_USE_FLASHINFER_GDN_PREFILL: '1' - HF_HUB_OFFLINE: '1' - PYTHONNOUSERSITE: '1' - decode_environment: - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TRTLLM_ENABLE_PDL: '1' - TLLM_LOG_LEVEL: INFO - TLLM_PROFILE_LOG_RANKS: all - TLLM_USE_FLASHINFER_GDN_PREFILL: '0' - HF_HUB_OFFLINE: '1' - PYTHONNOUSERSITE: '1' - trtllm_config: - prefill: - trust_remote_code: true - max_batch_size: 64 - max_num_tokens: 32768 - max_seq_len: 8395 - tensor_parallel_size: 2 - moe_expert_parallel_size: 2 - enable_attention_dp: true - pipeline_parallel_size: 1 - print_iter_log: true - enable_iter_perf_stats: true - cuda_graph_config: null - disable_overlap_scheduler: true - enable_chunked_prefill: true - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - dtype: fp8 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - cache_transceiver_config: - max_tokens_in_buffer: 8192 - backend: UCX - # Match NVIDIA's 8k/1k high-concurrency disagg deadline and avoid rc18's unsafe 60s cancellation path. - kv_transfer_timeout_ms: 600000 - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - context_chunking_policy: FIRST_COME_FIRST_SERVED - speculative_config: - decoding_type: MTP - max_draft_len: 3 - use_mtp_vanilla: false - decode: - trust_remote_code: true - tensor_parallel_size: 4 - moe_expert_parallel_size: 1 - enable_attention_dp: false - enable_lm_head_tp_in_adp: false - pipeline_parallel_size: 1 - max_batch_size: 5 - max_num_tokens: 4096 - max_seq_len: 9419 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 16 - - 20 - - 24 - - 32 - - 64 - - 128 - - 256 - - 384 - - 512 - print_iter_log: true - enable_iter_perf_stats: true - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.8 - dtype: fp8 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - cache_transceiver_config: - max_tokens_in_buffer: 8192 - backend: UCX - # Match NVIDIA's 8k/1k high-concurrency disagg deadline and avoid rc18's unsafe 60s cancellation path. - kv_transfer_timeout_ms: 600000 - stream_interval: 100 - num_postprocess_workers: 4 - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - context_chunking_policy: FIRST_COME_FIRST_SERVED - speculative_config: - decoding_type: MTP - max_draft_len: 3 - use_mtp_vanilla: false -frontend: - type: dynamo - enable_multiple_frontends: false - env: - ETCD_LEASE_TTL: '120' - DYN_TOKENIZER: "fastokens" -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "90" - req_rate: "inf" - num_prompts_mult: 16 -dynamo: - install: true - version: 1.3.0.dev20260708 - request_plane: tcp -infra: - etcd_nats_dedicated_node: false -profiling: - type: none -health_check: - max_attempts: 720 - interval_seconds: 10 -enable_config_dump: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx3dep2_gen4tep8_mtp3_c192.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx3dep2_gen4tep8_mtp3_c192.yaml deleted file mode 100644 index 2a088dc3b8..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx3dep2_gen4tep8_mtp3_c192.yaml +++ /dev/null @@ -1,168 +0,0 @@ -# Reconstructed from successful source result: /lustre/fsw/coreai_comparch_infbench/minma/sa_qwen_disagg/exp3_8k1k_mtp3_0609_tot/results/e2e-clean/3ctxDEP2c128_4genTEP8c32_E2E_C128/conc128 -# Variant runtime: srt-slurm f6eb42aee4664207dcf2ec601e3bd57bd527efd6 with official TensorRT-LLM 1.3.0rc18 container. -# Unlisted model, topology, resource, frontend, and engine fields remain identical to original-configs. -name: ctx3dep2_gen4tep8_mtp3_c192 -identity: - model: - repo: nvidia/Qwen3.5-397B-A17B-NVFP4 - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 - frameworks: - tensorrt_llm: 1.3.0rc18 -model: - path: qwen3.5-fp4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 - precision: fp4 -resources: - gpu_type: gb300 - gpus_per_node: 4 - prefill_nodes: 2 - prefill_workers: 3 - gpus_per_prefill: 2 - decode_nodes: 8 - decode_workers: 4 - gpus_per_decode: 8 - het_jobs: false -backend: - type: trtllm - prefill_environment: - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TRTLLM_ENABLE_PDL: '1' - TLLM_LOG_LEVEL: INFO - TLLM_PROFILE_LOG_RANKS: all - TLLM_USE_FLASHINFER_GDN_PREFILL: '1' - HF_HUB_OFFLINE: '1' - PYTHONNOUSERSITE: '1' - decode_environment: - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TRTLLM_ENABLE_PDL: '1' - TLLM_LOG_LEVEL: INFO - TLLM_PROFILE_LOG_RANKS: all - TLLM_USE_FLASHINFER_GDN_PREFILL: '0' - HF_HUB_OFFLINE: '1' - PYTHONNOUSERSITE: '1' - trtllm_config: - prefill: - trust_remote_code: true - max_batch_size: 64 - max_num_tokens: 32768 - max_seq_len: 8395 - tensor_parallel_size: 2 - moe_expert_parallel_size: 2 - enable_attention_dp: true - pipeline_parallel_size: 1 - print_iter_log: true - enable_iter_perf_stats: true - cuda_graph_config: null - disable_overlap_scheduler: true - enable_chunked_prefill: true - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - dtype: fp8 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - cache_transceiver_config: - max_tokens_in_buffer: 8192 - backend: UCX - # Match NVIDIA's 8k/1k high-concurrency disagg deadline and avoid rc18's unsafe 60s cancellation path. - kv_transfer_timeout_ms: 600000 - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - context_chunking_policy: FIRST_COME_FIRST_SERVED - speculative_config: - decoding_type: MTP - max_draft_len: 3 - use_mtp_vanilla: false - decode: - trust_remote_code: true - tensor_parallel_size: 8 - moe_expert_parallel_size: 8 - enable_attention_dp: false - enable_lm_head_tp_in_adp: false - pipeline_parallel_size: 1 - max_batch_size: 40 - max_num_tokens: 4096 - max_seq_len: 9419 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 16 - - 20 - - 24 - - 32 - - 64 - - 128 - - 256 - - 384 - - 512 - print_iter_log: true - enable_iter_perf_stats: true - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.8 - dtype: fp8 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - cache_transceiver_config: - max_tokens_in_buffer: 8192 - backend: UCX - # Match NVIDIA's 8k/1k high-concurrency disagg deadline and avoid rc18's unsafe 60s cancellation path. - kv_transfer_timeout_ms: 600000 - stream_interval: 100 - num_postprocess_workers: 4 - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - context_chunking_policy: FIRST_COME_FIRST_SERVED - allreduce_strategy: MNNVL - speculative_config: - decoding_type: MTP - max_draft_len: 3 - use_mtp_vanilla: false -frontend: - type: dynamo - enable_multiple_frontends: false - env: - ETCD_LEASE_TTL: '120' - DYN_TOKENIZER: "fastokens" -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "192" - req_rate: "inf" - num_prompts_mult: 16 -dynamo: - install: true - version: 1.3.0.dev20260708 - request_plane: tcp -infra: - etcd_nats_dedicated_node: false -profiling: - type: none -health_check: - max_attempts: 720 - interval_seconds: 10 -enable_config_dump: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx3dep2_gen7tep8_mtp3_c168.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx3dep2_gen7tep8_mtp3_c168.yaml deleted file mode 100644 index 12b6138e6a..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx3dep2_gen7tep8_mtp3_c168.yaml +++ /dev/null @@ -1,168 +0,0 @@ -# Reconstructed from successful source result: /lustre/fsw/coreai_comparch_infbench/minma/sa_qwen_disagg/exp3_8k1k_mtp3_0609_tot/results/e2e-clean/3ctxDEP2c128_7genTEP8c16_E2E_C112/conc112 -# Variant runtime: srt-slurm f6eb42aee4664207dcf2ec601e3bd57bd527efd6 with official TensorRT-LLM 1.3.0rc18 container. -# Unlisted model, topology, resource, frontend, and engine fields remain identical to original-configs. -name: ctx3dep2_gen7tep8_mtp3_c168 -identity: - model: - repo: nvidia/Qwen3.5-397B-A17B-NVFP4 - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 - frameworks: - tensorrt_llm: 1.3.0rc18 -model: - path: qwen3.5-fp4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 - precision: fp4 -resources: - gpu_type: gb300 - gpus_per_node: 4 - prefill_nodes: 2 - prefill_workers: 3 - gpus_per_prefill: 2 - decode_nodes: 14 - decode_workers: 7 - gpus_per_decode: 8 - het_jobs: false -backend: - type: trtllm - prefill_environment: - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TRTLLM_ENABLE_PDL: '1' - TLLM_LOG_LEVEL: INFO - TLLM_PROFILE_LOG_RANKS: all - TLLM_USE_FLASHINFER_GDN_PREFILL: '1' - HF_HUB_OFFLINE: '1' - PYTHONNOUSERSITE: '1' - decode_environment: - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TRTLLM_ENABLE_PDL: '1' - TLLM_LOG_LEVEL: INFO - TLLM_PROFILE_LOG_RANKS: all - TLLM_USE_FLASHINFER_GDN_PREFILL: '0' - HF_HUB_OFFLINE: '1' - PYTHONNOUSERSITE: '1' - trtllm_config: - prefill: - trust_remote_code: true - max_batch_size: 64 - max_num_tokens: 32768 - max_seq_len: 8395 - tensor_parallel_size: 2 - moe_expert_parallel_size: 2 - enable_attention_dp: true - pipeline_parallel_size: 1 - print_iter_log: true - enable_iter_perf_stats: true - cuda_graph_config: null - disable_overlap_scheduler: true - enable_chunked_prefill: true - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - dtype: fp8 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - cache_transceiver_config: - max_tokens_in_buffer: 8192 - backend: UCX - # Match NVIDIA's 8k/1k high-concurrency disagg deadline and avoid rc18's unsafe 60s cancellation path. - kv_transfer_timeout_ms: 600000 - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - context_chunking_policy: FIRST_COME_FIRST_SERVED - speculative_config: - decoding_type: MTP - max_draft_len: 3 - use_mtp_vanilla: false - decode: - trust_remote_code: true - tensor_parallel_size: 8 - moe_expert_parallel_size: 8 - enable_attention_dp: false - enable_lm_head_tp_in_adp: false - pipeline_parallel_size: 1 - max_batch_size: 20 - max_num_tokens: 4096 - max_seq_len: 9419 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 16 - - 20 - - 24 - - 32 - - 64 - - 128 - - 256 - - 384 - - 512 - print_iter_log: true - enable_iter_perf_stats: true - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.8 - dtype: fp8 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - cache_transceiver_config: - max_tokens_in_buffer: 8192 - backend: UCX - # Match NVIDIA's 8k/1k high-concurrency disagg deadline and avoid rc18's unsafe 60s cancellation path. - kv_transfer_timeout_ms: 600000 - stream_interval: 100 - num_postprocess_workers: 4 - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - context_chunking_policy: FIRST_COME_FIRST_SERVED - allreduce_strategy: MNNVL - speculative_config: - decoding_type: MTP - max_draft_len: 3 - use_mtp_vanilla: false -frontend: - type: dynamo - enable_multiple_frontends: false - env: - ETCD_LEASE_TTL: '120' - DYN_TOKENIZER: "fastokens" -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "168" - req_rate: "inf" - num_prompts_mult: 16 -dynamo: - install: true - version: 1.3.0.dev20260708 - request_plane: tcp -infra: - etcd_nats_dedicated_node: false -profiling: - type: none -health_check: - max_attempts: 720 - interval_seconds: 10 -enable_config_dump: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx2dep2_gen9tp4_mtp0_c216.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx4_gen3_tep8_batch32_eplb0_mtp3_126.yaml similarity index 50% rename from benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx2dep2_gen9tp4_mtp0_c216.yaml rename to benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx4_gen3_tep8_batch32_eplb0_mtp3_126.yaml index 595666bd0b..c10ba86c45 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx2dep2_gen9tp4_mtp0_c216.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx4_gen3_tep8_batch32_eplb0_mtp3_126.yaml @@ -1,154 +1,141 @@ -# Source: /lustre/fsw/coreai_comparch_infbench/minma/sa_qwen_disagg/exp3_8k1k_mtp0_0609_tot/e2e_submission_1/configs-variant1/2ctxDEP2c512_9genTP4c16_E2E_C144.yaml -# 5x provenance result (server settings only; target concurrency differs): /lustre/fsw/coreai_comparch_infbench/minma/sa_qwen_disagg/exp3_8k1k_mtp0_0609_tot/e2e_submission_1/results-variant1/2ctxDEP2c512_9genTP4c16_E2E_C144__v1_sysc720_genbs20_chunk1_ctxgdn1_gengdn0_buf8192/conc720 -name: ctx2dep2_gen9tp4_mtp0_c216 -identity: - model: - repo: nvidia/Qwen3.5-397B-A17B-NVFP4 - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 - frameworks: - tensorrt_llm: 1.3.0rc18 +name: disagg-gb300-4p3d-dep1-tep8-c126-mtp model: path: qwen3.5-fp4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 precision: fp4 + +dynamo: + install: true + wheel: 1.4.0.dev20260807 + +identity: + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 +slurm: + time_limit: 04:00:00 +health_check: + max_attempts: 540 + interval_seconds: 10 resources: gpu_type: gb300 gpus_per_node: 4 prefill_nodes: 1 - prefill_workers: 2 - gpus_per_prefill: 2 - decode_nodes: 9 - decode_workers: 9 - gpus_per_decode: 4 - het_jobs: false + prefill_workers: 4 + gpus_per_prefill: 1 + decode_nodes: 6 + decode_workers: 3 + gpus_per_decode: 8 backend: type: trtllm + sequential_node_start: 2 prefill_environment: + TLLM_LOG_LEVEL: INFO TRTLLM_SERVER_DISABLE_GC: '1' TRTLLM_WORKER_DISABLE_GC: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' TRTLLM_ENABLE_PDL: '1' - TLLM_LOG_LEVEL: INFO - TLLM_PROFILE_LOG_RANKS: all - HF_HUB_OFFLINE: '1' - PYTHONNOUSERSITE: '1' - TLLM_USE_FLASHINFER_GDN_PREFILL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' + CTX_LOAD_STAGGER_S: '180' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp decode_environment: + TLLM_LOG_LEVEL: INFO TRTLLM_SERVER_DISABLE_GC: '1' TRTLLM_WORKER_DISABLE_GC: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' TRTLLM_ENABLE_PDL: '1' - TLLM_LOG_LEVEL: INFO - TLLM_PROFILE_LOG_RANKS: all - HF_HUB_OFFLINE: '1' - PYTHONNOUSERSITE: '1' - TLLM_USE_FLASHINFER_GDN_PREFILL: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp trtllm_config: prefill: - trust_remote_code: true - max_batch_size: 256 - max_num_tokens: 32768 - max_seq_len: 8395 - tensor_parallel_size: 2 - moe_expert_parallel_size: 2 - enable_attention_dp: true - pipeline_parallel_size: 1 - print_iter_log: true - enable_iter_perf_stats: true + cache_transceiver_config: + backend: UCX + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 16384 cuda_graph_config: null disable_overlap_scheduler: true + enable_attention_dp: true kv_cache_config: + dtype: fp8 enable_block_reuse: false free_gpu_memory_fraction: 0.85 - dtype: fp8 + max_batch_size: 32 + max_num_tokens: 16896 + max_seq_len: 8448 moe_config: backend: CUTEDSL - use_low_precision_moe_combine: true + moe_expert_parallel_size: 1 nvfp4_gemm_config: allowed_backends: - cutlass - cublaslt - cutedsl - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + tensor_parallel_size: 1 + trust_remote_code: true + decode: cache_transceiver_config: - max_tokens_in_buffer: 8192 backend: UCX - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - context_chunking_policy: FIRST_COME_FIRST_SERVED - enable_chunked_prefill: true - decode: - trust_remote_code: true - tensor_parallel_size: 4 - moe_expert_parallel_size: 1 - enable_attention_dp: false - enable_lm_head_tp_in_adp: false - pipeline_parallel_size: 1 - max_batch_size: 20 - max_num_tokens: 16384 - max_seq_len: 9419 + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 16384 cuda_graph_config: - enable_padding: true batch_sizes: - 1 - 2 - 4 - 8 + - 12 - 16 - 20 - 24 + - 28 - 32 - - 64 - - 128 - - 256 - - 384 - - 512 - print_iter_log: true - enable_iter_perf_stats: true + enable_padding: true + enable_attention_dp: false + enable_lm_head_tp_in_adp: false kv_cache_config: + dtype: fp8 enable_block_reuse: false free_gpu_memory_fraction: 0.9 - dtype: fp8 + max_batch_size: 32 + max_num_tokens: 128 + max_seq_len: 9472 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 nvfp4_gemm_config: allowed_backends: - cutlass - cublaslt - cutedsl - cuda_core - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - cache_transceiver_config: - max_tokens_in_buffer: 8192 - backend: UCX + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 stream_interval: 100 - num_postprocess_workers: 4 - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - context_chunking_policy: FIRST_COME_FIRST_SERVED + tensor_parallel_size: 8 + trust_remote_code: true frontend: type: dynamo enable_multiple_frontends: false env: ETCD_LEASE_TTL: '120' DYN_TOKENIZER: "fastokens" + benchmark: - type: "sa-bench" + type: sa-bench isl: 8192 osl: 1024 - concurrencies: "216" - req_rate: "inf" + concurrencies: + - 126 + req_rate: inf num_prompts_mult: 16 -dynamo: - install: true - version: 1.3.0.dev20260708 - request_plane: tcp -infra: - etcd_nats_dedicated_node: false -profiling: - type: none -health_check: - max_attempts: 720 - interval_seconds: 10 -enable_config_dump: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx5_gen1_dep16_batch8_eplb0_mtp3_154.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx5_gen1_dep16_batch8_eplb0_mtp3_154.yaml new file mode 100644 index 0000000000..8299757f52 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx5_gen1_dep16_batch8_eplb0_mtp3_154.yaml @@ -0,0 +1,139 @@ +name: disagg-gb300-5p1d-dep1-dep16-c154-mtp +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + precision: fp4 + +dynamo: + install: true + wheel: 1.4.0.dev20260807 + +identity: + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + frameworks: + tensorrt_llm: 1.3.0rc24 +slurm: + time_limit: 04:00:00 +health_check: + max_attempts: 540 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 2 + prefill_workers: 5 + gpus_per_prefill: 1 + decode_nodes: 4 + decode_workers: 1 + gpus_per_decode: 16 +backend: + type: trtllm + sequential_node_start: 2 + prefill_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' + CTX_LOAD_STAGGER_S: '180' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + decode_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + trtllm_config: + prefill: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 16384 + transceiver_runtime: PYTHON + cuda_graph_config: null + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.7 + max_batch_size: 2 + max_num_tokens: 16896 + max_seq_len: 8448 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 1 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + tensor_parallel_size: 1 + trust_remote_code: true + decode: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 16384 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.75 + max_batch_size: 8 + max_num_tokens: 32 + max_seq_len: 9472 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + moe_expert_parallel_size: 16 + num_postprocess_workers: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + stream_interval: 100 + tensor_parallel_size: 16 + trust_remote_code: true +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: '120' + DYN_TOKENIZER: "fastokens" + +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: + - 154 + req_rate: inf + num_prompts_mult: 16 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx6dep2_gen5tep8_mtp3_c480.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx6dep2_gen5tep8_mtp3_c480.yaml deleted file mode 100644 index 14ff198a06..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx6dep2_gen5tep8_mtp3_c480.yaml +++ /dev/null @@ -1,168 +0,0 @@ -# Reconstructed from successful source result: /lustre/fsw/coreai_comparch_infbench/minma/sa_qwen_disagg/exp3_8k1k_mtp3_0609_tot/results/e2e-clean/6ctxDEP2c128_5genTEP8c64_E2E_C320/conc320 -# Variant runtime: srt-slurm f6eb42aee4664207dcf2ec601e3bd57bd527efd6 with official TensorRT-LLM 1.3.0rc18 container. -# Unlisted model, topology, resource, frontend, and engine fields remain identical to original-configs. -name: ctx6dep2_gen5tep8_mtp3_c480 -identity: - model: - repo: nvidia/Qwen3.5-397B-A17B-NVFP4 - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 - frameworks: - tensorrt_llm: 1.3.0rc18 -model: - path: qwen3.5-fp4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 - precision: fp4 -resources: - gpu_type: gb300 - gpus_per_node: 4 - prefill_nodes: 3 - prefill_workers: 6 - gpus_per_prefill: 2 - decode_nodes: 10 - decode_workers: 5 - gpus_per_decode: 8 - het_jobs: false -backend: - type: trtllm - prefill_environment: - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TRTLLM_ENABLE_PDL: '1' - TLLM_LOG_LEVEL: INFO - TLLM_PROFILE_LOG_RANKS: all - TLLM_USE_FLASHINFER_GDN_PREFILL: '1' - HF_HUB_OFFLINE: '1' - PYTHONNOUSERSITE: '1' - decode_environment: - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TRTLLM_ENABLE_PDL: '1' - TLLM_LOG_LEVEL: INFO - TLLM_PROFILE_LOG_RANKS: all - TLLM_USE_FLASHINFER_GDN_PREFILL: '0' - HF_HUB_OFFLINE: '1' - PYTHONNOUSERSITE: '1' - trtllm_config: - prefill: - trust_remote_code: true - max_batch_size: 64 - max_num_tokens: 32768 - max_seq_len: 8395 - tensor_parallel_size: 2 - moe_expert_parallel_size: 2 - enable_attention_dp: true - pipeline_parallel_size: 1 - print_iter_log: true - enable_iter_perf_stats: true - cuda_graph_config: null - disable_overlap_scheduler: true - enable_chunked_prefill: true - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - dtype: fp8 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - cache_transceiver_config: - max_tokens_in_buffer: 8192 - backend: UCX - # Match NVIDIA's 8k/1k high-concurrency disagg deadline and avoid rc18's unsafe 60s cancellation path. - kv_transfer_timeout_ms: 600000 - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - context_chunking_policy: FIRST_COME_FIRST_SERVED - speculative_config: - decoding_type: MTP - max_draft_len: 3 - use_mtp_vanilla: false - decode: - trust_remote_code: true - tensor_parallel_size: 8 - moe_expert_parallel_size: 8 - enable_attention_dp: false - enable_lm_head_tp_in_adp: false - pipeline_parallel_size: 1 - max_batch_size: 80 - max_num_tokens: 4096 - max_seq_len: 9419 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 16 - - 20 - - 24 - - 32 - - 64 - - 128 - - 256 - - 384 - - 512 - print_iter_log: true - enable_iter_perf_stats: true - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.8 - dtype: fp8 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - cache_transceiver_config: - max_tokens_in_buffer: 8192 - backend: UCX - # Match NVIDIA's 8k/1k high-concurrency disagg deadline and avoid rc18's unsafe 60s cancellation path. - kv_transfer_timeout_ms: 600000 - stream_interval: 100 - num_postprocess_workers: 4 - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - context_chunking_policy: FIRST_COME_FIRST_SERVED - allreduce_strategy: MNNVL - speculative_config: - decoding_type: MTP - max_draft_len: 3 - use_mtp_vanilla: false -frontend: - type: dynamo - enable_multiple_frontends: true - env: - ETCD_LEASE_TTL: '120' - DYN_TOKENIZER: "fastokens" -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "480" - req_rate: "inf" - num_prompts_mult: 16 -dynamo: - install: true - version: 1.3.0.dev20260708 - request_plane: tcp -infra: - etcd_nats_dedicated_node: false -profiling: - type: none -health_check: - max_attempts: 720 - interval_seconds: 10 -enable_config_dump: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx4dep2_gen1dep16_mtp3_c384.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx7_gen1_dep16_batch32_eplb0_mtp3_666.yaml similarity index 52% rename from benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx4dep2_gen1dep16_mtp3_c384.yaml rename to benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx7_gen1_dep16_batch32_eplb0_mtp3_666.yaml index 6a3802ef47..ac83aa5d78 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx4dep2_gen1dep16_mtp3_c384.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx7_gen1_dep16_batch32_eplb0_mtp3_666.yaml @@ -1,167 +1,139 @@ -# Reconstructed from successful source result: /lustre/fsw/coreai_comparch_infbench/minma/sa_qwen_disagg/exp3_8k1k_mtp3_0609_tot/results/e2e-clean/4ctxDEP2c128_1genDEP16c256_E2E_C256/conc256 -# Variant runtime: srt-slurm f6eb42aee4664207dcf2ec601e3bd57bd527efd6 with official TensorRT-LLM 1.3.0rc18 container. -# Unlisted model, topology, resource, frontend, and engine fields remain identical to original-configs. -name: ctx4dep2_gen1dep16_mtp3_c384 -identity: - model: - repo: nvidia/Qwen3.5-397B-A17B-NVFP4 - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 - frameworks: - tensorrt_llm: 1.3.0rc18 +name: disagg-gb300-7p1d-dep2-dep16-c666-mtp model: path: qwen3.5-fp4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 precision: fp4 + +dynamo: + install: true + wheel: 1.4.0.dev20260807 + +identity: + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 +slurm: + time_limit: 04:00:00 +health_check: + max_attempts: 540 + interval_seconds: 10 resources: gpu_type: gb300 gpus_per_node: 4 - prefill_nodes: 2 - prefill_workers: 4 + prefill_nodes: 4 + prefill_workers: 7 gpus_per_prefill: 2 decode_nodes: 4 decode_workers: 1 gpus_per_decode: 16 - het_jobs: false backend: type: trtllm + sequential_node_start: 2 prefill_environment: + TLLM_LOG_LEVEL: INFO TRTLLM_SERVER_DISABLE_GC: '1' TRTLLM_WORKER_DISABLE_GC: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' TRTLLM_ENABLE_PDL: '1' - TLLM_LOG_LEVEL: INFO - TLLM_PROFILE_LOG_RANKS: all - TLLM_USE_FLASHINFER_GDN_PREFILL: '1' - HF_HUB_OFFLINE: '1' - PYTHONNOUSERSITE: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp decode_environment: + TLLM_LOG_LEVEL: INFO TRTLLM_SERVER_DISABLE_GC: '1' TRTLLM_WORKER_DISABLE_GC: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' TRTLLM_ENABLE_PDL: '1' - TLLM_LOG_LEVEL: INFO - TLLM_PROFILE_LOG_RANKS: all - TLLM_USE_FLASHINFER_GDN_PREFILL: '0' - HF_HUB_OFFLINE: '1' - PYTHONNOUSERSITE: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp trtllm_config: prefill: - trust_remote_code: true - max_batch_size: 64 - max_num_tokens: 32768 - max_seq_len: 8395 - tensor_parallel_size: 2 - moe_expert_parallel_size: 2 - enable_attention_dp: true - pipeline_parallel_size: 1 - print_iter_log: true - enable_iter_perf_stats: true + cache_transceiver_config: + backend: UCX + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 16384 cuda_graph_config: null disable_overlap_scheduler: true - enable_chunked_prefill: true + enable_attention_dp: true kv_cache_config: + dtype: fp8 enable_block_reuse: false free_gpu_memory_fraction: 0.85 - dtype: fp8 + max_batch_size: 32 + max_num_tokens: 16896 + max_seq_len: 8448 moe_config: backend: CUTEDSL - use_low_precision_moe_combine: true + moe_expert_parallel_size: 2 nvfp4_gemm_config: allowed_backends: - cutlass - cublaslt - cutedsl - cuda_core - cache_transceiver_config: - max_tokens_in_buffer: 8192 - backend: UCX - # Match NVIDIA's 8k/1k high-concurrency disagg deadline and avoid rc18's unsafe 60s cancellation path. - kv_transfer_timeout_ms: 600000 - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - context_chunking_policy: FIRST_COME_FIRST_SERVED + pipeline_parallel_size: 1 + print_iter_log: true speculative_config: decoding_type: MTP max_draft_len: 3 - use_mtp_vanilla: false - decode: + tensor_parallel_size: 2 trust_remote_code: true - tensor_parallel_size: 16 - moe_expert_parallel_size: 16 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - pipeline_parallel_size: 1 - max_batch_size: 20 - max_num_tokens: 4096 - max_seq_len: 9419 + decode: + cache_transceiver_config: + backend: UCX + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 16384 cuda_graph_config: - enable_padding: true batch_sizes: - 1 - 2 - 4 - 8 + - 12 - 16 - 20 - 24 + - 28 - 32 - - 64 - - 128 - - 256 - - 384 - - 512 - print_iter_log: true - enable_iter_perf_stats: true + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: true kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.8 dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.75 + max_batch_size: 32 + max_num_tokens: 128 + max_seq_len: 9472 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + moe_expert_parallel_size: 16 + num_postprocess_workers: 4 nvfp4_gemm_config: allowed_backends: - cutlass - cublaslt - cutedsl - cuda_core - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - cache_transceiver_config: - max_tokens_in_buffer: 8192 - backend: UCX - # Match NVIDIA's 8k/1k high-concurrency disagg deadline and avoid rc18's unsafe 60s cancellation path. - kv_transfer_timeout_ms: 600000 - stream_interval: 100 - num_postprocess_workers: 4 - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - context_chunking_policy: FIRST_COME_FIRST_SERVED + pipeline_parallel_size: 1 + print_iter_log: true speculative_config: decoding_type: MTP max_draft_len: 3 - use_mtp_vanilla: false + stream_interval: 100 + tensor_parallel_size: 16 + trust_remote_code: true frontend: type: dynamo enable_multiple_frontends: true env: ETCD_LEASE_TTL: '120' DYN_TOKENIZER: "fastokens" + benchmark: - type: "sa-bench" + type: sa-bench isl: 8192 osl: 1024 - concurrencies: "384" - req_rate: "inf" + concurrencies: + - 666 + req_rate: inf num_prompts_mult: 16 -dynamo: - install: true - version: 1.3.0.dev20260708 - request_plane: tcp -infra: - etcd_nats_dedicated_node: false -profiling: - type: none -health_check: - max_attempts: 720 - interval_seconds: 10 -enable_config_dump: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx9dep2_gen1dep16_mtp3_c1536.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx9dep2_gen1dep16_mtp3_c1536.yaml deleted file mode 100644 index 6e0195f2da..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx9dep2_gen1dep16_mtp3_c1536.yaml +++ /dev/null @@ -1,167 +0,0 @@ -# Reconstructed from successful source result: /lustre/fsw/coreai_comparch_infbench/minma/sa_qwen_disagg/exp3_8k1k_mtp3_0609_tot/results/e2e-clean/9ctxDEP2c2048_1genDEP16c1024_E2E_C1024/conc1024 -# Variant runtime: srt-slurm f6eb42aee4664207dcf2ec601e3bd57bd527efd6 with official TensorRT-LLM 1.3.0rc18 container. -# Unlisted model, topology, resource, frontend, and engine fields remain identical to original-configs. -name: ctx9dep2_gen1dep16_mtp3_c1536 -identity: - model: - repo: nvidia/Qwen3.5-397B-A17B-NVFP4 - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 - frameworks: - tensorrt_llm: 1.3.0rc18 -model: - path: qwen3.5-fp4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 - precision: fp4 -resources: - gpu_type: gb300 - gpus_per_node: 4 - prefill_nodes: 5 - prefill_workers: 9 - gpus_per_prefill: 2 - decode_nodes: 4 - decode_workers: 1 - gpus_per_decode: 16 - het_jobs: false -backend: - type: trtllm - prefill_environment: - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TRTLLM_ENABLE_PDL: '1' - TLLM_LOG_LEVEL: INFO - TLLM_PROFILE_LOG_RANKS: all - TLLM_USE_FLASHINFER_GDN_PREFILL: '1' - HF_HUB_OFFLINE: '1' - PYTHONNOUSERSITE: '1' - decode_environment: - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TRTLLM_ENABLE_PDL: '1' - TLLM_LOG_LEVEL: INFO - TLLM_PROFILE_LOG_RANKS: all - TLLM_USE_FLASHINFER_GDN_PREFILL: '0' - HF_HUB_OFFLINE: '1' - PYTHONNOUSERSITE: '1' - trtllm_config: - prefill: - trust_remote_code: true - max_batch_size: 128 - max_num_tokens: 32768 - max_seq_len: 8395 - tensor_parallel_size: 2 - moe_expert_parallel_size: 2 - enable_attention_dp: true - pipeline_parallel_size: 1 - print_iter_log: true - enable_iter_perf_stats: true - cuda_graph_config: null - disable_overlap_scheduler: true - enable_chunked_prefill: true - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - dtype: fp8 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - cache_transceiver_config: - max_tokens_in_buffer: 8192 - backend: UCX - # Match NVIDIA's 8k/1k high-concurrency disagg deadline and avoid rc18's unsafe 60s cancellation path. - kv_transfer_timeout_ms: 600000 - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - context_chunking_policy: FIRST_COME_FIRST_SERVED - speculative_config: - decoding_type: MTP - max_draft_len: 3 - use_mtp_vanilla: false - decode: - trust_remote_code: true - tensor_parallel_size: 16 - moe_expert_parallel_size: 16 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - pipeline_parallel_size: 1 - max_batch_size: 80 - max_num_tokens: 4096 - max_seq_len: 9419 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 16 - - 20 - - 24 - - 32 - - 64 - - 128 - - 256 - - 384 - - 512 - print_iter_log: true - enable_iter_perf_stats: true - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.8 - dtype: fp8 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - cache_transceiver_config: - max_tokens_in_buffer: 8192 - backend: UCX - # Match NVIDIA's 8k/1k high-concurrency disagg deadline and avoid rc18's unsafe 60s cancellation path. - kv_transfer_timeout_ms: 600000 - stream_interval: 100 - num_postprocess_workers: 4 - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - context_chunking_policy: FIRST_COME_FIRST_SERVED - speculative_config: - decoding_type: MTP - max_draft_len: 3 - use_mtp_vanilla: false -frontend: - type: dynamo - enable_multiple_frontends: true - env: - ETCD_LEASE_TTL: '120' - DYN_TOKENIZER: "fastokens" -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "1536" - req_rate: "inf" - num_prompts_mult: 16 -dynamo: - install: true - version: 1.3.0.dev20260708 - request_plane: tcp -infra: - etcd_nats_dedicated_node: false -profiling: - type: none -health_check: - max_attempts: 720 - interval_seconds: 10 -enable_config_dump: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx10dep2_gen3dep8_mtp0_c4608.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx10_gen1_dep8_batch256_eplb0_mtp0_2150.yaml similarity index 50% rename from benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx10dep2_gen3dep8_mtp0_c4608.yaml rename to benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx10_gen1_dep8_batch256_eplb0_mtp0_2150.yaml index 917db683ef..856dcfa539 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx10dep2_gen3dep8_mtp0_c4608.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx10_gen1_dep8_batch256_eplb0_mtp0_2150.yaml @@ -1,154 +1,162 @@ -# Source: /lustre/fsw/coreai_comparch_infbench/minma/sa_qwen_disagg/exp3_8k1k_mtp0_0609_tot/e2e_submission_1/configs-variant1/10ctxDEP2c512_3genDEP8c1024_E2E_C3072.yaml -# 5x provenance result (server settings only; target concurrency differs): /lustre/fsw/coreai_comparch_infbench/minma/sa_qwen_disagg/exp3_8k1k_mtp0_0609_tot/e2e_submission_1/results-variant1/10ctxDEP2c512_3genDEP8c1024_E2E_C3072__v1_sysc15360_genbs160_chunk1_ctxgdn1_gengdn0_buf8192/conc15360 -name: ctx10dep2_gen3dep8_mtp0_c4608 -identity: - model: - repo: nvidia/Qwen3.5-397B-A17B-NVFP4 - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 - frameworks: - tensorrt_llm: 1.3.0rc18 +name: disagg-gb300-10p1d-dep1-dep8-c2150-stp model: path: qwen3.5-fp4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 precision: fp4 + +dynamo: + install: true + wheel: 1.4.0.dev20260807 + +identity: + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + frameworks: + tensorrt_llm: 1.3.0rc24 +slurm: + time_limit: 04:00:00 +health_check: + max_attempts: 540 + interval_seconds: 10 resources: gpu_type: gb300 gpus_per_node: 4 - prefill_nodes: 5 + prefill_nodes: 3 prefill_workers: 10 - gpus_per_prefill: 2 - decode_nodes: 6 - decode_workers: 3 + gpus_per_prefill: 1 + decode_nodes: 2 + decode_workers: 1 gpus_per_decode: 8 - het_jobs: false backend: type: trtllm + sequential_node_start: 2 prefill_environment: + TLLM_LOG_LEVEL: INFO TRTLLM_SERVER_DISABLE_GC: '1' TRTLLM_WORKER_DISABLE_GC: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' TRTLLM_ENABLE_PDL: '1' - TLLM_LOG_LEVEL: INFO - TLLM_PROFILE_LOG_RANKS: all - HF_HUB_OFFLINE: '1' - PYTHONNOUSERSITE: '1' - TLLM_USE_FLASHINFER_GDN_PREFILL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' + CTX_LOAD_STAGGER_S: '180' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp decode_environment: + TLLM_LOG_LEVEL: INFO TRTLLM_SERVER_DISABLE_GC: '1' TRTLLM_WORKER_DISABLE_GC: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' TRTLLM_ENABLE_PDL: '1' - TLLM_LOG_LEVEL: INFO - TLLM_PROFILE_LOG_RANKS: all - HF_HUB_OFFLINE: '1' - PYTHONNOUSERSITE: '1' - TLLM_USE_FLASHINFER_GDN_PREFILL: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp trtllm_config: prefill: - trust_remote_code: true - max_batch_size: 256 - max_num_tokens: 32768 - max_seq_len: 8395 - tensor_parallel_size: 2 - moe_expert_parallel_size: 2 - enable_attention_dp: true - pipeline_parallel_size: 1 - print_iter_log: true - enable_iter_perf_stats: true + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 16384 cuda_graph_config: null disable_overlap_scheduler: true + enable_attention_dp: true kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.7 + max_batch_size: 2 + max_num_tokens: 16896 + max_seq_len: 8448 moe_config: backend: CUTEDSL - use_low_precision_moe_combine: true + moe_expert_parallel_size: 1 nvfp4_gemm_config: allowed_backends: - cutlass - cublaslt - cutedsl - cuda_core - cache_transceiver_config: - max_tokens_in_buffer: 8192 - backend: UCX - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - context_chunking_policy: FIRST_COME_FIRST_SERVED - enable_chunked_prefill: true - decode: - trust_remote_code: true - tensor_parallel_size: 8 - moe_expert_parallel_size: 8 - enable_attention_dp: true - enable_lm_head_tp_in_adp: false pipeline_parallel_size: 1 - max_batch_size: 160 - max_num_tokens: 16384 - max_seq_len: 9419 + print_iter_log: true + tensor_parallel_size: 1 + trust_remote_code: true + decode: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 16384 cuda_graph_config: - enable_padding: true batch_sizes: - 1 - 2 - 4 - 8 - 16 - - 20 - 24 - 32 + - 40 + - 48 + - 56 - 64 + - 72 + - 80 + - 88 + - 96 + - 104 + - 112 + - 120 - 128 + - 136 + - 144 + - 152 + - 160 + - 168 + - 176 + - 184 + - 192 + - 200 + - 208 + - 216 + - 224 + - 232 + - 240 + - 248 - 256 - - 384 - - 512 - print_iter_log: true - enable_iter_perf_stats: true + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: false kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.9 dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + max_batch_size: 256 + max_num_tokens: 256 + max_seq_len: 9472 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 nvfp4_gemm_config: allowed_backends: - cutlass - cublaslt - cutedsl - cuda_core - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - cache_transceiver_config: - max_tokens_in_buffer: 8192 - backend: UCX + pipeline_parallel_size: 1 + print_iter_log: true stream_interval: 100 - num_postprocess_workers: 4 - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - context_chunking_policy: FIRST_COME_FIRST_SERVED + tensor_parallel_size: 8 + trust_remote_code: true frontend: type: dynamo enable_multiple_frontends: true env: ETCD_LEASE_TTL: '120' DYN_TOKENIZER: "fastokens" + benchmark: - type: "sa-bench" + type: sa-bench isl: 8192 osl: 1024 - concurrencies: "4608" - req_rate: "inf" + concurrencies: + - 2150 + req_rate: inf num_prompts_mult: 16 -dynamo: - install: true - version: 1.3.0.dev20260708 - request_plane: tcp -infra: - etcd_nats_dedicated_node: false -profiling: - type: none -health_check: - max_attempts: 720 - interval_seconds: 10 -enable_config_dump: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx11_gen1_dep16_batch64_eplb0_mtp0_1076.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx11_gen1_dep16_batch64_eplb0_mtp0_1076.yaml new file mode 100644 index 0000000000..eb81e30ff9 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx11_gen1_dep16_batch64_eplb0_mtp0_1076.yaml @@ -0,0 +1,140 @@ +name: disagg-gb300-11p1d-dep1-dep16-c1076-stp +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + precision: fp4 + +dynamo: + install: true + wheel: 1.4.0.dev20260807 + +identity: + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + frameworks: + tensorrt_llm: 1.3.0rc24 +slurm: + time_limit: 04:00:00 +health_check: + max_attempts: 540 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 3 + prefill_workers: 11 + gpus_per_prefill: 1 + decode_nodes: 4 + decode_workers: 1 + gpus_per_decode: 16 +backend: + type: trtllm + sequential_node_start: 2 + prefill_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' + CTX_LOAD_STAGGER_S: '180' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + decode_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + trtllm_config: + prefill: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 16384 + transceiver_runtime: PYTHON + cuda_graph_config: null + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.7 + max_batch_size: 2 + max_num_tokens: 16896 + max_seq_len: 8448 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 1 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + tensor_parallel_size: 1 + trust_remote_code: true + decode: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 16384 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + - 40 + - 48 + - 56 + - 64 + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.75 + max_batch_size: 64 + max_num_tokens: 64 + max_seq_len: 9472 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + moe_expert_parallel_size: 16 + num_postprocess_workers: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + stream_interval: 100 + tensor_parallel_size: 16 + trust_remote_code: true +frontend: + type: dynamo + enable_multiple_frontends: true + env: + ETCD_LEASE_TTL: '120' + DYN_TOKENIZER: "fastokens" + +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: + - 1076 + req_rate: inf + num_prompts_mult: 16 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx16_gen1_dep16_batch128_eplb0_mtp0_2253.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx16_gen1_dep16_batch128_eplb0_mtp0_2253.yaml new file mode 100644 index 0000000000..5034c39d1e --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx16_gen1_dep16_batch128_eplb0_mtp0_2253.yaml @@ -0,0 +1,148 @@ +name: disagg-gb300-16p1d-dep1-dep16-c2253-stp +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + precision: fp4 + +dynamo: + install: true + wheel: 1.4.0.dev20260807 + +identity: + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + frameworks: + tensorrt_llm: 1.3.0rc24 +slurm: + time_limit: 04:00:00 +health_check: + max_attempts: 540 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 4 + prefill_workers: 16 + gpus_per_prefill: 1 + decode_nodes: 4 + decode_workers: 1 + gpus_per_decode: 16 +backend: + type: trtllm + sequential_node_start: 2 + prefill_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' + CTX_LOAD_STAGGER_S: '180' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + decode_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + trtllm_config: + prefill: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 16384 + transceiver_runtime: PYTHON + cuda_graph_config: null + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.7 + max_batch_size: 2 + max_num_tokens: 16896 + max_seq_len: 8448 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 1 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + tensor_parallel_size: 1 + trust_remote_code: true + decode: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 16384 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + - 40 + - 48 + - 56 + - 64 + - 72 + - 80 + - 88 + - 96 + - 104 + - 112 + - 120 + - 128 + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.75 + max_batch_size: 128 + max_num_tokens: 128 + max_seq_len: 9472 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + moe_expert_parallel_size: 16 + num_postprocess_workers: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + stream_interval: 100 + tensor_parallel_size: 16 + trust_remote_code: true +frontend: + type: dynamo + enable_multiple_frontends: true + env: + ETCD_LEASE_TTL: '120' + DYN_TOKENIZER: "fastokens" + +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: + - 2253 + req_rate: inf + num_prompts_mult: 16 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx1_gen2_tep8_batch16_eplb0_mtp0_42.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx1_gen2_tep8_batch16_eplb0_mtp0_42.yaml new file mode 100644 index 0000000000..084010e55d --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx1_gen2_tep8_batch16_eplb0_mtp0_42.yaml @@ -0,0 +1,129 @@ +name: disagg-gb300-1p2d-dep1-tep8-c42-stp +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + precision: fp4 + +dynamo: + install: true + wheel: 1.4.0.dev20260807 + +identity: + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 +slurm: + time_limit: 04:00:00 +health_check: + max_attempts: 540 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 1 + decode_nodes: 4 + decode_workers: 2 + gpus_per_decode: 8 +backend: + type: trtllm + prefill_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' + CTX_LOAD_STAGGER_S: '180' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + decode_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + trtllm_config: + prefill: + cache_transceiver_config: + backend: UCX + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 16384 + cuda_graph_config: null + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + max_batch_size: 32 + max_num_tokens: 16896 + max_seq_len: 8448 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 1 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + tensor_parallel_size: 1 + trust_remote_code: true + decode: + cache_transceiver_config: + backend: UCX + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 16384 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + enable_padding: true + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + max_batch_size: 16 + max_num_tokens: 16 + max_seq_len: 9472 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + stream_interval: 100 + tensor_parallel_size: 8 + trust_remote_code: true +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: '120' + DYN_TOKENIZER: "fastokens" + +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: + - 42 + req_rate: inf + num_prompts_mult: 16 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx1dep2_gen8tep8_mtp0_c48.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx1_gen4_tep8_batch1_eplb0_mtp0_8.yaml similarity index 52% rename from benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx1dep2_gen8tep8_mtp0_c48.yaml rename to benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx1_gen4_tep8_batch1_eplb0_mtp0_8.yaml index fe33250b79..4317340be6 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx1dep2_gen8tep8_mtp0_c48.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx1_gen4_tep8_batch1_eplb0_mtp0_8.yaml @@ -1,155 +1,125 @@ -# Source: /lustre/fsw/coreai_comparch_infbench/minma/sa_qwen_disagg/exp3_8k1k_mtp0_0609_tot/e2e_submission_1/configs-variant1/1ctxDEP2c128_8genTEP8c4_E2E_C32.yaml -# 5x provenance result (server settings only; target concurrency differs): /lustre/fsw/coreai_comparch_infbench/minma/sa_qwen_disagg/exp3_8k1k_mtp0_0609_tot/e2e_submission_1/results-variant1/1ctxDEP2c128_8genTEP8c4_E2E_C32__v1_sysc160_genbs5_chunk1_ctxgdn1_gengdn0_buf8192/conc160 -name: ctx1dep2_gen8tep8_mtp0_c48 -identity: - model: - repo: nvidia/Qwen3.5-397B-A17B-NVFP4 - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 - frameworks: - tensorrt_llm: 1.3.0rc18 +name: disagg-gb300-1p4d-dep2-tep8-c8-stp model: path: qwen3.5-fp4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 precision: fp4 + +dynamo: + install: true + wheel: 1.4.0.dev20260807 + +identity: + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 +slurm: + time_limit: 04:00:00 +health_check: + max_attempts: 540 + interval_seconds: 10 resources: gpu_type: gb300 gpus_per_node: 4 prefill_nodes: 1 prefill_workers: 1 gpus_per_prefill: 2 - decode_nodes: 16 - decode_workers: 8 + decode_nodes: 8 + decode_workers: 4 gpus_per_decode: 8 - het_jobs: false backend: type: trtllm prefill_environment: + TLLM_LOG_LEVEL: INFO TRTLLM_SERVER_DISABLE_GC: '1' TRTLLM_WORKER_DISABLE_GC: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' TRTLLM_ENABLE_PDL: '1' - TLLM_LOG_LEVEL: INFO - TLLM_PROFILE_LOG_RANKS: all - HF_HUB_OFFLINE: '1' - PYTHONNOUSERSITE: '1' - TLLM_USE_FLASHINFER_GDN_PREFILL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp decode_environment: + TLLM_LOG_LEVEL: INFO TRTLLM_SERVER_DISABLE_GC: '1' TRTLLM_WORKER_DISABLE_GC: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' TRTLLM_ENABLE_PDL: '1' - TLLM_LOG_LEVEL: INFO - TLLM_PROFILE_LOG_RANKS: all - HF_HUB_OFFLINE: '1' - PYTHONNOUSERSITE: '1' - TLLM_USE_FLASHINFER_GDN_PREFILL: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp trtllm_config: prefill: - trust_remote_code: true - max_batch_size: 64 - max_num_tokens: 32768 - max_seq_len: 8395 - tensor_parallel_size: 2 - moe_expert_parallel_size: 2 - enable_attention_dp: true - pipeline_parallel_size: 1 - print_iter_log: true - enable_iter_perf_stats: true + cache_transceiver_config: + backend: UCX + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 16384 cuda_graph_config: null disable_overlap_scheduler: true + enable_attention_dp: true kv_cache_config: + dtype: fp8 enable_block_reuse: false free_gpu_memory_fraction: 0.85 - dtype: fp8 + max_batch_size: 32 + max_num_tokens: 16896 + max_seq_len: 8448 moe_config: backend: CUTEDSL - use_low_precision_moe_combine: true + moe_expert_parallel_size: 2 nvfp4_gemm_config: allowed_backends: - cutlass - cublaslt - cutedsl - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + tensor_parallel_size: 2 + trust_remote_code: true + decode: cache_transceiver_config: - max_tokens_in_buffer: 8192 backend: UCX - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - context_chunking_policy: FIRST_COME_FIRST_SERVED - enable_chunked_prefill: true - decode: - trust_remote_code: true - tensor_parallel_size: 8 - moe_expert_parallel_size: 8 - enable_attention_dp: false - enable_lm_head_tp_in_adp: false - pipeline_parallel_size: 1 - max_batch_size: 5 - max_num_tokens: 16384 - max_seq_len: 9419 + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 16384 cuda_graph_config: - enable_padding: true batch_sizes: - 1 - 2 - 4 - - 8 - - 16 - - 20 - - 24 - - 32 - - 64 - - 128 - - 256 - - 384 - - 512 - print_iter_log: true - enable_iter_perf_stats: true + enable_padding: true + enable_attention_dp: false + enable_lm_head_tp_in_adp: false kv_cache_config: + dtype: fp8 enable_block_reuse: false free_gpu_memory_fraction: 0.9 - dtype: fp8 + max_batch_size: 1 + max_num_tokens: 1 + max_seq_len: 9472 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 nvfp4_gemm_config: allowed_backends: - cutlass - cublaslt - cutedsl - cuda_core - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - cache_transceiver_config: - max_tokens_in_buffer: 8192 - backend: UCX + pipeline_parallel_size: 1 + print_iter_log: true stream_interval: 100 - num_postprocess_workers: 4 - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - context_chunking_policy: FIRST_COME_FIRST_SERVED - allreduce_strategy: MNNVL + tensor_parallel_size: 8 + trust_remote_code: true frontend: type: dynamo enable_multiple_frontends: false env: ETCD_LEASE_TTL: '120' DYN_TOKENIZER: "fastokens" + benchmark: - type: "sa-bench" + type: sa-bench isl: 8192 osl: 1024 - concurrencies: "48" - req_rate: "inf" + concurrencies: + - 8 + req_rate: inf num_prompts_mult: 16 -dynamo: - install: true - version: 1.3.0.dev20260708 - request_plane: tcp -infra: - etcd_nats_dedicated_node: false -profiling: - type: none -health_check: - max_attempts: 720 - interval_seconds: 10 -enable_config_dump: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx1_gen4_tep8_batch4_eplb0_mtp0_24.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx1_gen4_tep8_batch4_eplb0_mtp0_24.yaml new file mode 100644 index 0000000000..1972efbff4 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx1_gen4_tep8_batch4_eplb0_mtp0_24.yaml @@ -0,0 +1,127 @@ +name: disagg-gb300-1p4d-dep1-tep8-c24-stp +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + precision: fp4 + +dynamo: + install: true + wheel: 1.4.0.dev20260807 + +identity: + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 +slurm: + time_limit: 04:00:00 +health_check: + max_attempts: 540 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 1 + decode_nodes: 8 + decode_workers: 4 + gpus_per_decode: 8 +backend: + type: trtllm + prefill_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' + CTX_LOAD_STAGGER_S: '180' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + decode_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + trtllm_config: + prefill: + cache_transceiver_config: + backend: UCX + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 16384 + cuda_graph_config: null + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + max_batch_size: 32 + max_num_tokens: 16896 + max_seq_len: 8448 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 1 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + tensor_parallel_size: 1 + trust_remote_code: true + decode: + cache_transceiver_config: + backend: UCX + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 16384 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + enable_padding: true + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + max_batch_size: 4 + max_num_tokens: 4 + max_seq_len: 9472 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + stream_interval: 100 + tensor_parallel_size: 8 + trust_remote_code: true +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: '120' + DYN_TOKENIZER: "fastokens" + +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: + - 24 + req_rate: inf + num_prompts_mult: 16 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx1dep2_gen4tep16_mtp0_c24.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx1dep2_gen4tep16_mtp0_c24.yaml deleted file mode 100644 index 9ea0b58b1e..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx1dep2_gen4tep16_mtp0_c24.yaml +++ /dev/null @@ -1,154 +0,0 @@ -# Source: /lustre/fsw/coreai_comparch_infbench/minma/sa_qwen_disagg/exp3_8k1k_mtp0_0609_tot/e2e_submission_1/configs-variant1/1ctxDEP2c128_4genTEP16c4_E2E_C16.yaml -# 5x provenance result (server settings only; target concurrency differs): /lustre/fsw/coreai_comparch_infbench/minma/sa_qwen_disagg/exp3_8k1k_mtp0_0609_tot/e2e_submission_1/results-variant1/1ctxDEP2c128_4genTEP16c4_E2E_C16__v1_sysc80_genbs5_chunk1_ctxgdn1_gengdn0_buf8192/conc80 -name: ctx1dep2_gen4tep16_mtp0_c24 -identity: - model: - repo: nvidia/Qwen3.5-397B-A17B-NVFP4 - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 - frameworks: - tensorrt_llm: 1.3.0rc18 -model: - path: qwen3.5-fp4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 - precision: fp4 -resources: - gpu_type: gb300 - gpus_per_node: 4 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 2 - decode_nodes: 16 - decode_workers: 4 - gpus_per_decode: 16 - het_jobs: false -backend: - type: trtllm - prefill_environment: - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TRTLLM_ENABLE_PDL: '1' - TLLM_LOG_LEVEL: INFO - TLLM_PROFILE_LOG_RANKS: all - HF_HUB_OFFLINE: '1' - PYTHONNOUSERSITE: '1' - TLLM_USE_FLASHINFER_GDN_PREFILL: '1' - decode_environment: - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TRTLLM_ENABLE_PDL: '1' - TLLM_LOG_LEVEL: INFO - TLLM_PROFILE_LOG_RANKS: all - HF_HUB_OFFLINE: '1' - PYTHONNOUSERSITE: '1' - TLLM_USE_FLASHINFER_GDN_PREFILL: '0' - trtllm_config: - prefill: - trust_remote_code: true - max_batch_size: 64 - max_num_tokens: 32768 - max_seq_len: 8395 - tensor_parallel_size: 2 - moe_expert_parallel_size: 2 - enable_attention_dp: true - pipeline_parallel_size: 1 - print_iter_log: true - enable_iter_perf_stats: true - cuda_graph_config: null - disable_overlap_scheduler: true - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - dtype: fp8 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - cache_transceiver_config: - max_tokens_in_buffer: 8192 - backend: UCX - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - context_chunking_policy: FIRST_COME_FIRST_SERVED - enable_chunked_prefill: true - decode: - trust_remote_code: true - tensor_parallel_size: 16 - moe_expert_parallel_size: 16 - enable_attention_dp: false - enable_lm_head_tp_in_adp: false - pipeline_parallel_size: 1 - max_batch_size: 5 - max_num_tokens: 16384 - max_seq_len: 9419 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 16 - - 20 - - 24 - - 32 - - 64 - - 128 - - 256 - - 384 - - 512 - print_iter_log: true - enable_iter_perf_stats: true - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.9 - dtype: fp8 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - cache_transceiver_config: - max_tokens_in_buffer: 8192 - backend: UCX - stream_interval: 100 - num_postprocess_workers: 4 - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - context_chunking_policy: FIRST_COME_FIRST_SERVED -frontend: - type: dynamo - enable_multiple_frontends: false - env: - ETCD_LEASE_TTL: '120' - DYN_TOKENIZER: "fastokens" -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "24" - req_rate: "inf" - num_prompts_mult: 16 -dynamo: - install: true - version: 1.3.0.dev20260708 - request_plane: tcp -infra: - etcd_nats_dedicated_node: false -profiling: - type: none -health_check: - max_attempts: 720 - interval_seconds: 10 -enable_config_dump: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx24_gen1_dep16_batch512_eplb0_mtp0_8192.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx24_gen1_dep16_batch512_eplb0_mtp0_8192.yaml new file mode 100644 index 0000000000..5b3e044150 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx24_gen1_dep16_batch512_eplb0_mtp0_8192.yaml @@ -0,0 +1,194 @@ +name: disagg-gb300-24p1d-dep1-dep16-c8192-stp +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + precision: fp4 + +dynamo: + install: true + wheel: 1.4.0.dev20260807 + +identity: + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + frameworks: + tensorrt_llm: 1.3.0rc24 +slurm: + time_limit: 04:00:00 +health_check: + max_attempts: 540 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 6 + prefill_workers: 24 + gpus_per_prefill: 1 + decode_nodes: 4 + decode_workers: 1 + gpus_per_decode: 16 +backend: + type: trtllm + sequential_node_start: 2 + prefill_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' + CTX_LOAD_STAGGER_S: '180' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + decode_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + trtllm_config: + prefill: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 16384 + cuda_graph_config: null + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.7 + max_batch_size: 2 + max_num_tokens: 16896 + max_seq_len: 8448 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 1 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + tensor_parallel_size: 1 + trust_remote_code: true + decode: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 16384 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + - 40 + - 48 + - 56 + - 64 + - 72 + - 80 + - 88 + - 96 + - 104 + - 112 + - 120 + - 128 + - 136 + - 144 + - 152 + - 160 + - 168 + - 176 + - 184 + - 192 + - 200 + - 208 + - 216 + - 224 + - 232 + - 240 + - 248 + - 256 + - 264 + - 272 + - 280 + - 288 + - 296 + - 304 + - 312 + - 320 + - 328 + - 336 + - 344 + - 352 + - 360 + - 368 + - 376 + - 384 + - 392 + - 400 + - 408 + - 416 + - 424 + - 432 + - 440 + - 448 + - 456 + - 464 + - 472 + - 480 + - 488 + - 496 + - 504 + - 512 + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.75 + max_batch_size: 512 + max_num_tokens: 512 + max_seq_len: 9472 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + moe_expert_parallel_size: 16 + num_postprocess_workers: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + stream_interval: 100 + tensor_parallel_size: 16 + trust_remote_code: true +frontend: + type: dynamo + enable_multiple_frontends: true + env: + ETCD_LEASE_TTL: '120' + DYN_TOKENIZER: "fastokens" + +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: + - 8192 + req_rate: inf + num_prompts_mult: 16 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx2dep2_gen15tp4_mtp0_c180.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx2dep2_gen15tp4_mtp0_c180.yaml deleted file mode 100644 index b100c7f7c9..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx2dep2_gen15tp4_mtp0_c180.yaml +++ /dev/null @@ -1,154 +0,0 @@ -# Source: /lustre/fsw/coreai_comparch_infbench/minma/sa_qwen_disagg/exp3_8k1k_mtp0_0609_tot/e2e_submission_1/configs-variant1/2ctxDEP2c128_15genTP4c8_E2E_C120.yaml -# 5x provenance result (server settings only; target concurrency differs): /lustre/fsw/coreai_comparch_infbench/minma/sa_qwen_disagg/exp3_8k1k_mtp0_0609_tot/e2e_submission_1/results-variant1/2ctxDEP2c128_15genTP4c8_E2E_C120__v1_sysc600_genbs10_chunk1_ctxgdn1_gengdn0_buf8192/conc600 -name: ctx2dep2_gen15tp4_mtp0_c180 -identity: - model: - repo: nvidia/Qwen3.5-397B-A17B-NVFP4 - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 - frameworks: - tensorrt_llm: 1.3.0rc18 -model: - path: qwen3.5-fp4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 - precision: fp4 -resources: - gpu_type: gb300 - gpus_per_node: 4 - prefill_nodes: 1 - prefill_workers: 2 - gpus_per_prefill: 2 - decode_nodes: 15 - decode_workers: 15 - gpus_per_decode: 4 - het_jobs: false -backend: - type: trtllm - prefill_environment: - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TRTLLM_ENABLE_PDL: '1' - TLLM_LOG_LEVEL: INFO - TLLM_PROFILE_LOG_RANKS: all - HF_HUB_OFFLINE: '1' - PYTHONNOUSERSITE: '1' - TLLM_USE_FLASHINFER_GDN_PREFILL: '1' - decode_environment: - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TRTLLM_ENABLE_PDL: '1' - TLLM_LOG_LEVEL: INFO - TLLM_PROFILE_LOG_RANKS: all - HF_HUB_OFFLINE: '1' - PYTHONNOUSERSITE: '1' - TLLM_USE_FLASHINFER_GDN_PREFILL: '0' - trtllm_config: - prefill: - trust_remote_code: true - max_batch_size: 64 - max_num_tokens: 32768 - max_seq_len: 8395 - tensor_parallel_size: 2 - moe_expert_parallel_size: 2 - enable_attention_dp: true - pipeline_parallel_size: 1 - print_iter_log: true - enable_iter_perf_stats: true - cuda_graph_config: null - disable_overlap_scheduler: true - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - dtype: fp8 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - cache_transceiver_config: - max_tokens_in_buffer: 8192 - backend: UCX - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - context_chunking_policy: FIRST_COME_FIRST_SERVED - enable_chunked_prefill: true - decode: - trust_remote_code: true - tensor_parallel_size: 4 - moe_expert_parallel_size: 1 - enable_attention_dp: false - enable_lm_head_tp_in_adp: false - pipeline_parallel_size: 1 - max_batch_size: 10 - max_num_tokens: 16384 - max_seq_len: 9419 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 16 - - 20 - - 24 - - 32 - - 64 - - 128 - - 256 - - 384 - - 512 - print_iter_log: true - enable_iter_perf_stats: true - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.9 - dtype: fp8 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - cache_transceiver_config: - max_tokens_in_buffer: 8192 - backend: UCX - stream_interval: 100 - num_postprocess_workers: 4 - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - context_chunking_policy: FIRST_COME_FIRST_SERVED -frontend: - type: dynamo - enable_multiple_frontends: false - env: - ETCD_LEASE_TTL: '120' - DYN_TOKENIZER: "fastokens" -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "180" - req_rate: "inf" - num_prompts_mult: 16 -dynamo: - install: true - version: 1.3.0.dev20260708 - request_plane: tcp -infra: - etcd_nats_dedicated_node: false -profiling: - type: none -health_check: - max_attempts: 720 - interval_seconds: 10 -enable_config_dump: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx3dep2_gen1dep4_mtp0_c1536.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx3dep2_gen1dep4_mtp0_c1536.yaml deleted file mode 100644 index 2beea78e73..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx3dep2_gen1dep4_mtp0_c1536.yaml +++ /dev/null @@ -1,155 +0,0 @@ -# Source: /lustre/fsw/coreai_comparch_infbench/minma/sa_qwen_disagg/exp3_8k1k_mtp0_0609_tot/e2e_submission_1/configs-variant1/3ctxDEP2c512_1genDEP4c1024_E2E_C1024.yaml -# 5x provenance result (server settings only; target concurrency differs): /lustre/fsw/coreai_comparch_infbench/minma/sa_qwen_disagg/exp3_8k1k_mtp0_0609_tot/e2e_submission_1/results-variant1/3ctxDEP2c512_1genDEP4c1024_E2E_C1024__v1_sysc5120_genbs320_chunk1_ctxgdn1_gengdn0_buf8192/conc5120 -# 1.5x performance reference only (not config-identical: GEN BS256, buffer7168, GEN GDN1): /lustre/fsw/coreai_comparch_infbench/minma/sa_qwen_disagg/exp3_8k1k_mtp0_0609_tot/ablation_3xDEP2_1xDEP4_max_tpt/results/sys_conc1536__ctx_gdn1_chunked1__ctx_nt32768__gen_bs256__cgdefault__s1r1__buf7168__allranklog1/conc1536 -name: ctx3dep2_gen1dep4_mtp0_c1536 -identity: - model: - repo: nvidia/Qwen3.5-397B-A17B-NVFP4 - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 - frameworks: - tensorrt_llm: 1.3.0rc18 -model: - path: qwen3.5-fp4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 - precision: fp4 -resources: - gpu_type: gb300 - gpus_per_node: 4 - prefill_nodes: 2 - prefill_workers: 3 - gpus_per_prefill: 2 - decode_nodes: 1 - decode_workers: 1 - gpus_per_decode: 4 - het_jobs: false -backend: - type: trtllm - prefill_environment: - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TRTLLM_ENABLE_PDL: '1' - TLLM_LOG_LEVEL: INFO - TLLM_PROFILE_LOG_RANKS: all - HF_HUB_OFFLINE: '1' - PYTHONNOUSERSITE: '1' - TLLM_USE_FLASHINFER_GDN_PREFILL: '1' - decode_environment: - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TRTLLM_ENABLE_PDL: '1' - TLLM_LOG_LEVEL: INFO - TLLM_PROFILE_LOG_RANKS: all - HF_HUB_OFFLINE: '1' - PYTHONNOUSERSITE: '1' - TLLM_USE_FLASHINFER_GDN_PREFILL: '0' - trtllm_config: - prefill: - trust_remote_code: true - max_batch_size: 256 - max_num_tokens: 32768 - max_seq_len: 8395 - tensor_parallel_size: 2 - moe_expert_parallel_size: 2 - enable_attention_dp: true - pipeline_parallel_size: 1 - print_iter_log: true - enable_iter_perf_stats: true - cuda_graph_config: null - disable_overlap_scheduler: true - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - dtype: fp8 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - cache_transceiver_config: - max_tokens_in_buffer: 8192 - backend: UCX - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - context_chunking_policy: FIRST_COME_FIRST_SERVED - enable_chunked_prefill: true - decode: - trust_remote_code: true - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - enable_attention_dp: true - enable_lm_head_tp_in_adp: false - pipeline_parallel_size: 1 - max_batch_size: 320 - max_num_tokens: 16384 - max_seq_len: 9419 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 16 - - 20 - - 24 - - 32 - - 64 - - 128 - - 256 - - 384 - - 512 - print_iter_log: true - enable_iter_perf_stats: true - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.9 - dtype: fp8 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - cache_transceiver_config: - max_tokens_in_buffer: 8192 - backend: UCX - stream_interval: 100 - num_postprocess_workers: 4 - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - context_chunking_policy: FIRST_COME_FIRST_SERVED -frontend: - type: dynamo - enable_multiple_frontends: true - env: - ETCD_LEASE_TTL: '120' - DYN_TOKENIZER: "fastokens" -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "1536" - req_rate: "inf" - num_prompts_mult: 16 -dynamo: - install: true - version: 1.3.0.dev20260708 - request_plane: tcp -infra: - etcd_nats_dedicated_node: false -profiling: - type: none -health_check: - max_attempts: 720 - interval_seconds: 10 -enable_config_dump: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx4_gen1_dep16_batch16_eplb0_mtp0_282.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx4_gen1_dep16_batch16_eplb0_mtp0_282.yaml new file mode 100644 index 0000000000..6fa4cf9e14 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx4_gen1_dep16_batch16_eplb0_mtp0_282.yaml @@ -0,0 +1,132 @@ +name: disagg-gb300-4p1d-dep1-dep16-c282-stp +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + precision: fp4 + +dynamo: + install: true + wheel: 1.4.0.dev20260807 + +identity: + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + frameworks: + tensorrt_llm: 1.3.0rc24 +slurm: + time_limit: 04:00:00 +health_check: + max_attempts: 540 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 1 + prefill_workers: 4 + gpus_per_prefill: 1 + decode_nodes: 4 + decode_workers: 1 + gpus_per_decode: 16 +backend: + type: trtllm + sequential_node_start: 2 + prefill_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' + CTX_LOAD_STAGGER_S: '180' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + decode_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + trtllm_config: + prefill: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 16384 + cuda_graph_config: null + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.7 + max_batch_size: 2 + max_num_tokens: 16896 + max_seq_len: 8448 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 1 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + tensor_parallel_size: 1 + trust_remote_code: true + decode: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 16384 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.75 + max_batch_size: 16 + max_num_tokens: 16 + max_seq_len: 9472 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + moe_expert_parallel_size: 16 + num_postprocess_workers: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + stream_interval: 100 + tensor_parallel_size: 16 + trust_remote_code: true +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: '120' + DYN_TOKENIZER: "fastokens" + +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: + - 282 + req_rate: inf + num_prompts_mult: 16 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx1dep2_gen13tp4_mtp0_c78.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx4_gen3_tep8_batch64_eplb0_mtp0_210.yaml similarity index 51% rename from benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx1dep2_gen13tp4_mtp0_c78.yaml rename to benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx4_gen3_tep8_batch64_eplb0_mtp0_210.yaml index 3264cdcf68..3635f56335 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx1dep2_gen13tp4_mtp0_c78.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx4_gen3_tep8_batch64_eplb0_mtp0_210.yaml @@ -1,154 +1,138 @@ -# Source: /lustre/fsw/coreai_comparch_infbench/minma/sa_qwen_disagg/exp3_8k1k_mtp0_0609_tot/e2e_submission_1/configs-variant1/1ctxDEP2c128_13genTP4c4_E2E_C52.yaml -# 5x provenance result (server settings only; target concurrency differs): /lustre/fsw/coreai_comparch_infbench/minma/sa_qwen_disagg/exp3_8k1k_mtp0_0609_tot/e2e_submission_1/results-variant1/1ctxDEP2c128_13genTP4c4_E2E_C52__v1_sysc260_genbs5_chunk1_ctxgdn1_gengdn0_buf8192/conc260 -name: ctx1dep2_gen13tp4_mtp0_c78 -identity: - model: - repo: nvidia/Qwen3.5-397B-A17B-NVFP4 - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 - frameworks: - tensorrt_llm: 1.3.0rc18 +name: disagg-gb300-4p3d-dep1-tep8-c210-stp model: path: qwen3.5-fp4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 precision: fp4 + +dynamo: + install: true + wheel: 1.4.0.dev20260807 + +identity: + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + frameworks: + tensorrt_llm: 1.3.0rc24 +slurm: + time_limit: 04:00:00 +health_check: + max_attempts: 540 + interval_seconds: 10 resources: gpu_type: gb300 gpus_per_node: 4 prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 2 - decode_nodes: 13 - decode_workers: 13 - gpus_per_decode: 4 - het_jobs: false + prefill_workers: 4 + gpus_per_prefill: 1 + decode_nodes: 6 + decode_workers: 3 + gpus_per_decode: 8 backend: type: trtllm + sequential_node_start: 2 prefill_environment: + TLLM_LOG_LEVEL: INFO TRTLLM_SERVER_DISABLE_GC: '1' TRTLLM_WORKER_DISABLE_GC: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' TRTLLM_ENABLE_PDL: '1' - TLLM_LOG_LEVEL: INFO - TLLM_PROFILE_LOG_RANKS: all - HF_HUB_OFFLINE: '1' - PYTHONNOUSERSITE: '1' - TLLM_USE_FLASHINFER_GDN_PREFILL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' + CTX_LOAD_STAGGER_S: '180' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp decode_environment: + TLLM_LOG_LEVEL: INFO TRTLLM_SERVER_DISABLE_GC: '1' TRTLLM_WORKER_DISABLE_GC: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' TRTLLM_ENABLE_PDL: '1' - TLLM_LOG_LEVEL: INFO - TLLM_PROFILE_LOG_RANKS: all - HF_HUB_OFFLINE: '1' - PYTHONNOUSERSITE: '1' - TLLM_USE_FLASHINFER_GDN_PREFILL: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp trtllm_config: prefill: - trust_remote_code: true - max_batch_size: 64 - max_num_tokens: 32768 - max_seq_len: 8395 - tensor_parallel_size: 2 - moe_expert_parallel_size: 2 - enable_attention_dp: true - pipeline_parallel_size: 1 - print_iter_log: true - enable_iter_perf_stats: true + cache_transceiver_config: + backend: UCX + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 16384 cuda_graph_config: null disable_overlap_scheduler: true + enable_attention_dp: true kv_cache_config: + dtype: fp8 enable_block_reuse: false free_gpu_memory_fraction: 0.85 - dtype: fp8 + max_batch_size: 32 + max_num_tokens: 16896 + max_seq_len: 8448 moe_config: backend: CUTEDSL - use_low_precision_moe_combine: true + moe_expert_parallel_size: 1 nvfp4_gemm_config: allowed_backends: - cutlass - cublaslt - cutedsl - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + tensor_parallel_size: 1 + trust_remote_code: true + decode: cache_transceiver_config: - max_tokens_in_buffer: 8192 backend: UCX - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - context_chunking_policy: FIRST_COME_FIRST_SERVED - enable_chunked_prefill: true - decode: - trust_remote_code: true - tensor_parallel_size: 4 - moe_expert_parallel_size: 1 - enable_attention_dp: false - enable_lm_head_tp_in_adp: false - pipeline_parallel_size: 1 - max_batch_size: 5 - max_num_tokens: 16384 - max_seq_len: 9419 + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 16384 cuda_graph_config: - enable_padding: true batch_sizes: - 1 - 2 - 4 - 8 - 16 - - 20 - 24 - 32 + - 40 + - 48 + - 56 - 64 - - 128 - - 256 - - 384 - - 512 - print_iter_log: true - enable_iter_perf_stats: true + enable_padding: true + enable_attention_dp: false + enable_lm_head_tp_in_adp: false kv_cache_config: + dtype: fp8 enable_block_reuse: false free_gpu_memory_fraction: 0.9 - dtype: fp8 + max_batch_size: 64 + max_num_tokens: 64 + max_seq_len: 9472 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 nvfp4_gemm_config: allowed_backends: - cutlass - cublaslt - cutedsl - cuda_core - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - cache_transceiver_config: - max_tokens_in_buffer: 8192 - backend: UCX + pipeline_parallel_size: 1 + print_iter_log: true stream_interval: 100 - num_postprocess_workers: 4 - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - context_chunking_policy: FIRST_COME_FIRST_SERVED + tensor_parallel_size: 8 + trust_remote_code: true frontend: type: dynamo enable_multiple_frontends: false env: ETCD_LEASE_TTL: '120' DYN_TOKENIZER: "fastokens" + benchmark: - type: "sa-bench" + type: sa-bench isl: 8192 osl: 1024 - concurrencies: "78" - req_rate: "inf" + concurrencies: + - 210 + req_rate: inf num_prompts_mult: 16 -dynamo: - install: true - version: 1.3.0.dev20260708 - request_plane: tcp -infra: - etcd_nats_dedicated_node: false -profiling: - type: none -health_check: - max_attempts: 720 - interval_seconds: 10 -enable_config_dump: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx7_gen1_dep16_batch32_eplb0_mtp0_563.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx7_gen1_dep16_batch32_eplb0_mtp0_563.yaml new file mode 100644 index 0000000000..d477032c59 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx7_gen1_dep16_batch32_eplb0_mtp0_563.yaml @@ -0,0 +1,134 @@ +name: disagg-gb300-7p1d-dep1-dep16-c563-stp +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + precision: fp4 + +dynamo: + install: true + wheel: 1.4.0.dev20260807 + +identity: + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + frameworks: + tensorrt_llm: 1.3.0rc24 +slurm: + time_limit: 04:00:00 +health_check: + max_attempts: 540 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 2 + prefill_workers: 7 + gpus_per_prefill: 1 + decode_nodes: 4 + decode_workers: 1 + gpus_per_decode: 16 +backend: + type: trtllm + sequential_node_start: 2 + prefill_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' + CTX_LOAD_STAGGER_S: '180' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + decode_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + trtllm_config: + prefill: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 16384 + cuda_graph_config: null + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.7 + max_batch_size: 2 + max_num_tokens: 16896 + max_seq_len: 8448 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 1 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + tensor_parallel_size: 1 + trust_remote_code: true + decode: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 16384 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.75 + max_batch_size: 32 + max_num_tokens: 32 + max_seq_len: 9472 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + moe_expert_parallel_size: 16 + num_postprocess_workers: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + stream_interval: 100 + tensor_parallel_size: 16 + trust_remote_code: true +frontend: + type: dynamo + enable_multiple_frontends: true + env: + ETCD_LEASE_TTL: '120' + DYN_TOKENIZER: "fastokens" + +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: + - 563 + req_rate: inf + num_prompts_mult: 16 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx7dep2_gen12tep4_mtp0_c1152.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx7dep2_gen12tep4_mtp0_c1152.yaml deleted file mode 100644 index 47cc963396..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx7dep2_gen12tep4_mtp0_c1152.yaml +++ /dev/null @@ -1,157 +0,0 @@ -# 9th point (C768) for exp3 8k1k mtp0 srt-slurm, matching configs/01-08 (variant-1 transforms at 1.5x sysc). -# Derived from baseline point: results/e2e/7ctxDEP2c512_12genTEP4c64_E2E_C768 (conc768). -# Variant transforms vs baseline: sysc x1.5 (768->1152), CTX chunked_prefill 0->1, FlashInfer GDN prefill 0->1 (env), -# GEN max_batch_size x1.25 (64->80), KV cache_transceiver buffer 16384->8192, TLLM_PROFILE_LOG_RANKS all, adaptive num_prompts/num_warmups. -# UNLIKE 01-08: spread_workers=False -> prefill packs 2/node (pf_nodes=4), total 16 nodes fits a single NVL72 domain (segment=16). -name: ctx7dep2_gen12tep4_mtp0_c1152 -identity: - model: - repo: nvidia/Qwen3.5-397B-A17B-NVFP4 - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 - frameworks: - tensorrt_llm: 1.3.0rc18 -model: - path: qwen3.5-fp4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 - precision: fp4 -resources: - gpu_type: gb300 - gpus_per_node: 4 - prefill_nodes: 4 - prefill_workers: 7 - gpus_per_prefill: 2 - decode_nodes: 12 - decode_workers: 12 - gpus_per_decode: 4 - het_jobs: false -backend: - type: trtllm - prefill_environment: - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TRTLLM_ENABLE_PDL: '1' - TLLM_LOG_LEVEL: INFO - TLLM_PROFILE_LOG_RANKS: all - HF_HUB_OFFLINE: '1' - PYTHONNOUSERSITE: '1' - TLLM_USE_FLASHINFER_GDN_PREFILL: '1' - decode_environment: - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TRTLLM_ENABLE_PDL: '1' - TLLM_LOG_LEVEL: INFO - TLLM_PROFILE_LOG_RANKS: all - HF_HUB_OFFLINE: '1' - PYTHONNOUSERSITE: '1' - TLLM_USE_FLASHINFER_GDN_PREFILL: '0' - trtllm_config: - prefill: - trust_remote_code: true - max_batch_size: 256 - max_num_tokens: 32768 - max_seq_len: 8395 - tensor_parallel_size: 2 - moe_expert_parallel_size: 2 - enable_attention_dp: true - pipeline_parallel_size: 1 - print_iter_log: true - enable_iter_perf_stats: true - cuda_graph_config: null - disable_overlap_scheduler: true - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - dtype: fp8 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - cache_transceiver_config: - max_tokens_in_buffer: 8192 - backend: UCX - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - context_chunking_policy: FIRST_COME_FIRST_SERVED - enable_chunked_prefill: true - decode: - trust_remote_code: true - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - enable_attention_dp: false - enable_lm_head_tp_in_adp: false - pipeline_parallel_size: 1 - max_batch_size: 80 - max_num_tokens: 16384 - max_seq_len: 9419 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 16 - - 20 - - 24 - - 32 - - 64 - - 128 - - 256 - - 384 - - 512 - print_iter_log: true - enable_iter_perf_stats: true - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.9 - dtype: fp8 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - cache_transceiver_config: - max_tokens_in_buffer: 8192 - backend: UCX - stream_interval: 100 - num_postprocess_workers: 4 - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - context_chunking_policy: FIRST_COME_FIRST_SERVED -frontend: - type: dynamo - enable_multiple_frontends: true - env: - ETCD_LEASE_TTL: '120' - DYN_TOKENIZER: "fastokens" -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "1152" - req_rate: "inf" - num_prompts_mult: 16 -dynamo: - install: true - version: 1.3.0.dev20260708 - request_plane: tcp -infra: - etcd_nats_dedicated_node: false -profiling: - type: none -health_check: - max_attempts: 720 - interval_seconds: 10 -enable_config_dump: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx8dep2_gen9tep4_mtp0_c1728.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx8dep2_gen9tep4_mtp0_c1728.yaml deleted file mode 100644 index 8d19209e9e..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx8dep2_gen9tep4_mtp0_c1728.yaml +++ /dev/null @@ -1,154 +0,0 @@ -# Source: /lustre/fsw/coreai_comparch_infbench/minma/sa_qwen_disagg/exp3_8k1k_mtp0_0609_tot/e2e_submission_1/configs-variant1/8ctxDEP2c512_9genTEP4c128_E2E_C1152.yaml -# 5x provenance result (server settings only; target concurrency differs): /lustre/fsw/coreai_comparch_infbench/minma/sa_qwen_disagg/exp3_8k1k_mtp0_0609_tot/e2e_submission_1/results-variant1/8ctxDEP2c512_9genTEP4c128_E2E_C1152__v1_sysc5760_genbs160_chunk1_ctxgdn1_gengdn0_buf8192/conc5760 -name: ctx8dep2_gen9tep4_mtp0_c1728 -identity: - model: - repo: nvidia/Qwen3.5-397B-A17B-NVFP4 - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 - frameworks: - tensorrt_llm: 1.3.0rc18 -model: - path: qwen3.5-fp4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 - precision: fp4 -resources: - gpu_type: gb300 - gpus_per_node: 4 - prefill_nodes: 4 - prefill_workers: 8 - gpus_per_prefill: 2 - decode_nodes: 9 - decode_workers: 9 - gpus_per_decode: 4 - het_jobs: false -backend: - type: trtllm - prefill_environment: - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TRTLLM_ENABLE_PDL: '1' - TLLM_LOG_LEVEL: INFO - TLLM_PROFILE_LOG_RANKS: all - HF_HUB_OFFLINE: '1' - PYTHONNOUSERSITE: '1' - TLLM_USE_FLASHINFER_GDN_PREFILL: '1' - decode_environment: - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TRTLLM_ENABLE_PDL: '1' - TLLM_LOG_LEVEL: INFO - TLLM_PROFILE_LOG_RANKS: all - HF_HUB_OFFLINE: '1' - PYTHONNOUSERSITE: '1' - TLLM_USE_FLASHINFER_GDN_PREFILL: '0' - trtllm_config: - prefill: - trust_remote_code: true - max_batch_size: 256 - max_num_tokens: 32768 - max_seq_len: 8395 - tensor_parallel_size: 2 - moe_expert_parallel_size: 2 - enable_attention_dp: true - pipeline_parallel_size: 1 - print_iter_log: true - enable_iter_perf_stats: true - cuda_graph_config: null - disable_overlap_scheduler: true - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - dtype: fp8 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - cache_transceiver_config: - max_tokens_in_buffer: 8192 - backend: UCX - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - context_chunking_policy: FIRST_COME_FIRST_SERVED - enable_chunked_prefill: true - decode: - trust_remote_code: true - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - enable_attention_dp: false - enable_lm_head_tp_in_adp: false - pipeline_parallel_size: 1 - max_batch_size: 160 - max_num_tokens: 16384 - max_seq_len: 9419 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 16 - - 20 - - 24 - - 32 - - 64 - - 128 - - 256 - - 384 - - 512 - print_iter_log: true - enable_iter_perf_stats: true - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.9 - dtype: fp8 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - cache_transceiver_config: - max_tokens_in_buffer: 8192 - backend: UCX - stream_interval: 100 - num_postprocess_workers: 4 - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - context_chunking_policy: FIRST_COME_FIRST_SERVED -frontend: - type: dynamo - enable_multiple_frontends: true - env: - ETCD_LEASE_TTL: '120' - DYN_TOKENIZER: "fastokens" -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "1728" - req_rate: "inf" - num_prompts_mult: 16 -dynamo: - install: true - version: 1.3.0.dev20260708 - request_plane: tcp -infra: - etcd_nats_dedicated_node: false -profiling: - type: none -health_check: - max_attempts: 720 - interval_seconds: 10 -enable_config_dump: true diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 78fa30f657..438a3e9da0 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -6369,145 +6369,159 @@ qwen3.5-fp4-gb300-dynamo-sglang: dp-attn: true qwen3.5-fp4-gb300-dynamo-trt: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 - model: nvidia/Qwen3.5-397B-A17B-NVFP4 + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + model: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 model-prefix: qwen3.5 runner: gb300 precision: fp4 framework: dynamo-trt multinode: true disagg: true - kv-p2p-transfer: ucx + kv-p2p-transfer: nixl scenarios: fixed-seq-len: - isl: 8192 osl: 1024 search-space: - # 1P13D: 1 prefill (TP2/EP2/dp-attn), 13 decode (TP4/EP1/no-dp-attn), conc=78 - - conc-list: [78] + # 1P2D: 1 prefill (TP1/EP1/dp-attn), 2 decode (TP8/EP8/no-dp-attn), conc=42 + - conc-list: [42] prefill: num-worker: 1 - tp: 2 - ep: 2 + tp: 1 + ep: 1 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx1dep2_gen13tp4_mtp0_c78.yaml" + - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx1_gen2_tep8_batch16_eplb0_mtp0_42.yaml" decode: - num-worker: 13 - tp: 4 - ep: 1 + num-worker: 2 + tp: 8 + ep: 8 dp-attn: false - # 1P4D: 1 prefill (TP2/EP2/dp-attn), 4 decode (TP16/EP16/no-dp-attn), conc=24 - - conc-list: [24] + # 1P4D: 1 prefill (TP2/EP2/dp-attn), 4 decode (TP8/EP8/no-dp-attn), conc=8 + - conc-list: [8] prefill: num-worker: 1 tp: 2 ep: 2 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx1dep2_gen4tep16_mtp0_c24.yaml" + - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx1_gen4_tep8_batch1_eplb0_mtp0_8.yaml" decode: num-worker: 4 - tp: 16 - ep: 16 + tp: 8 + ep: 8 dp-attn: false - # 1P8D: 1 prefill (TP2/EP2/dp-attn), 8 decode (TP8/EP8/no-dp-attn), conc=48 - - conc-list: [48] + # 1P4D: 1 prefill (TP1/EP1/dp-attn), 4 decode (TP8/EP8/no-dp-attn), conc=24 + - conc-list: [24] prefill: num-worker: 1 - tp: 2 - ep: 2 + tp: 1 + ep: 1 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx1dep2_gen8tep8_mtp0_c48.yaml" + - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx1_gen4_tep8_batch4_eplb0_mtp0_24.yaml" decode: - num-worker: 8 + num-worker: 4 tp: 8 ep: 8 dp-attn: false - # 2P15D: 2 prefill (TP2/EP2/dp-attn), 15 decode (TP4/EP1/no-dp-attn), conc=180 - - conc-list: [180] + # 4P1D: 4 prefill (TP1/EP1/dp-attn), 1 decode (TP16/EP16/dp-attn), conc=282 + - conc-list: [282] prefill: - num-worker: 2 - tp: 2 - ep: 2 + num-worker: 4 + tp: 1 + ep: 1 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx2dep2_gen15tp4_mtp0_c180.yaml" + - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx4_gen1_dep16_batch16_eplb0_mtp0_282.yaml" decode: - num-worker: 15 - tp: 4 + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + # 4P3D: 4 prefill (TP1/EP1/dp-attn), 3 decode (TP8/EP8/no-dp-attn), conc=210 + - conc-list: [210] + prefill: + num-worker: 4 + tp: 1 ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx4_gen3_tep8_batch64_eplb0_mtp0_210.yaml" + decode: + num-worker: 3 + tp: 8 + ep: 8 dp-attn: false - # 2P9D: 2 prefill (TP2/EP2/dp-attn), 9 decode (TP4/EP1/no-dp-attn), conc=216 - - conc-list: [216] + # 7P1D: 7 prefill (TP1/EP1/dp-attn), 1 decode (TP16/EP16/dp-attn), conc=563 + - conc-list: [563] prefill: - num-worker: 2 - tp: 2 - ep: 2 + num-worker: 7 + tp: 1 + ep: 1 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx2dep2_gen9tp4_mtp0_c216.yaml" + - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx7_gen1_dep16_batch32_eplb0_mtp0_563.yaml" decode: - num-worker: 9 - tp: 4 - ep: 1 - dp-attn: false - # 3P1D: 3 prefill (TP2/EP2/dp-attn), 1 decode (TP4/EP4/dp-attn), conc=1536 - - conc-list: [1536] + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + # 10P1D: 10 prefill (TP1/EP1/dp-attn), 1 decode (TP8/EP8/dp-attn), conc=2150 + - conc-list: [2150] prefill: - num-worker: 3 - tp: 2 - ep: 2 + num-worker: 10 + tp: 1 + ep: 1 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx3dep2_gen1dep4_mtp0_c1536.yaml" + - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx10_gen1_dep8_batch256_eplb0_mtp0_2150.yaml" decode: num-worker: 1 - tp: 4 - ep: 4 + tp: 8 + ep: 8 dp-attn: true - # 7P12D: 7 prefill (TP2/EP2/dp-attn), 12 decode (TP4/EP4/no-dp-attn), conc=1152 - - conc-list: [1152] + # 11P1D: 11 prefill (TP1/EP1/dp-attn), 1 decode (TP16/EP16/dp-attn), conc=1076 + - conc-list: [1076] prefill: - num-worker: 7 - tp: 2 - ep: 2 + num-worker: 11 + tp: 1 + ep: 1 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx7dep2_gen12tep4_mtp0_c1152.yaml" + - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx11_gen1_dep16_batch64_eplb0_mtp0_1076.yaml" decode: - num-worker: 12 - tp: 4 - ep: 4 - dp-attn: false - # 8P9D: 8 prefill (TP2/EP2/dp-attn), 9 decode (TP4/EP4/no-dp-attn), conc=1728 - - conc-list: [1728] + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + # 16P1D: 16 prefill (TP1/EP1/dp-attn), 1 decode (TP16/EP16/dp-attn), conc=2253 + - conc-list: [2253] prefill: - num-worker: 8 - tp: 2 - ep: 2 + num-worker: 16 + tp: 1 + ep: 1 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx8dep2_gen9tep4_mtp0_c1728.yaml" + - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx16_gen1_dep16_batch128_eplb0_mtp0_2253.yaml" decode: - num-worker: 9 - tp: 4 - ep: 4 - dp-attn: false - # 10P3D: 10 prefill (TP2/EP2/dp-attn), 3 decode (TP8/EP8/dp-attn), conc=4608 - - conc-list: [4608] + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + # 24P1D: 24 prefill (TP1/EP1/dp-attn), 1 decode (TP16/EP16/dp-attn), conc=8192 + - conc-list: [8192] prefill: - num-worker: 10 - tp: 2 - ep: 2 + num-worker: 24 + tp: 1 + ep: 1 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx10dep2_gen3dep8_mtp0_c4608.yaml" + - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx24_gen1_dep16_batch512_eplb0_mtp0_8192.yaml" decode: - num-worker: 3 - tp: 8 - ep: 8 + num-worker: 1 + tp: 16 + ep: 16 dp-attn: true # GB300 SGLang AgentX, re-swept across low/mid/high concurrency after aiperf @@ -6515,8 +6529,8 @@ qwen3.5-fp4-gb300-dynamo-trt: # one TP2 worker with MTP/NEXTN + hierarchical KV cache. CONFIG_FILE carries the # full srtctl recipe; the matrix topology stays in sync for naming/GPU accounting. qwen3.5-fp4-gb300-dynamo-trt-mtp: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 - model: nvidia/Qwen3.5-397B-A17B-NVFP4 + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + model: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 model-prefix: qwen3.5 runner: gb300 precision: fp4 @@ -6529,136 +6543,151 @@ qwen3.5-fp4-gb300-dynamo-trt-mtp: - isl: 8192 osl: 1024 search-space: - # 1P1D: 1 prefill (TP2/EP2/dp-attn), 1 decode (TP16/EP16/no-dp-attn), conc=48 + # 1P2D: 1 prefill (TP1/EP1/dp-attn), 2 decode (TP8/EP8/no-dp-attn), conc=20 - spec-decoding: "mtp" - conc-list: [48] + conc-list: [20] prefill: num-worker: 1 - tp: 2 - ep: 2 + tp: 1 + ep: 1 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx1dep2_gen1tep16_mtp3_c48.yaml" + - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx1_gen2_tep8_batch8_eplb0_mtp3_20.yaml" decode: - num-worker: 1 - tp: 16 - ep: 16 + num-worker: 2 + tp: 8 + ep: 8 dp-attn: false - # 1P4D: 1 prefill (TP2/EP2/dp-attn), 4 decode (TP16/EP1/no-dp-attn), conc=4 + # 1P4D: 1 prefill (TP1/EP1/dp-attn), 4 decode (TP8/EP8/no-dp-attn), conc=12 - spec-decoding: "mtp" - conc-list: [4] + conc-list: [12] prefill: num-worker: 1 - tp: 2 - ep: 2 + tp: 1 + ep: 1 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx1dep2_gen4tp16_mtp3_c4.yaml" + - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx1_gen4_tep8_batch1_eplb0_mtp3_12.yaml" decode: num-worker: 4 - tp: 16 - ep: 1 + tp: 8 + ep: 8 dp-attn: false - # 1P7D: 1 prefill (TP2/EP2/dp-attn), 7 decode (TP8/EP8/no-dp-attn), conc=42 + # 1P4D: 1 prefill (TP1/EP1/dp-attn), 4 decode (TP8/EP8/no-dp-attn), conc=8 - spec-decoding: "mtp" - conc-list: [42] + conc-list: [8] prefill: num-worker: 1 - tp: 2 - ep: 2 + tp: 1 + ep: 1 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx1dep2_gen7tep8_mtp3_c42.yaml" + - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx1_gen4_tep8_batch2_eplb0_mtp3_8.yaml" decode: - num-worker: 7 + num-worker: 4 tp: 8 ep: 8 dp-attn: false - # 2P15D: 2 prefill (TP2/EP2/dp-attn), 15 decode (TP4/EP1/no-dp-attn), conc=90 + # 4P3D: 4 prefill (TP1/EP1/dp-attn), 3 decode (TP8/EP8/no-dp-attn), conc=126 - spec-decoding: "mtp" - conc-list: [90] + conc-list: [126] prefill: - num-worker: 2 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx2dep2_gen15tp4_mtp3_c90.yaml" - decode: - num-worker: 15 - tp: 4 + num-worker: 4 + tp: 1 ep: 1 - dp-attn: false - # 3P4D: 3 prefill (TP2/EP2/dp-attn), 4 decode (TP8/EP8/no-dp-attn), conc=192 - - spec-decoding: "mtp" - conc-list: [192] - prefill: - num-worker: 3 - tp: 2 - ep: 2 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx3dep2_gen4tep8_mtp3_c192.yaml" + - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx4_gen3_tep8_batch32_eplb0_mtp3_126.yaml" decode: - num-worker: 4 + num-worker: 3 tp: 8 ep: 8 dp-attn: false - # 3P7D: 3 prefill (TP2/EP2/dp-attn), 7 decode (TP8/EP8/no-dp-attn), conc=168 + # 5P1D: 5 prefill (TP1/EP1/dp-attn), 1 decode (TP16/EP16/dp-attn), conc=154 - spec-decoding: "mtp" - conc-list: [168] + conc-list: [154] prefill: - num-worker: 3 - tp: 2 - ep: 2 + num-worker: 5 + tp: 1 + ep: 1 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx3dep2_gen7tep8_mtp3_c168.yaml" + - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx5_gen1_dep16_batch8_eplb0_mtp3_154.yaml" decode: - num-worker: 7 - tp: 8 - ep: 8 - dp-attn: false - # 4P1D: 4 prefill (TP2/EP2/dp-attn), 1 decode (TP16/EP16/dp-attn), conc=384 + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + # 7P1D: 7 prefill (TP2/EP2/dp-attn), 1 decode (TP16/EP16/dp-attn), conc=666 - spec-decoding: "mtp" - conc-list: [384] + conc-list: [666] prefill: - num-worker: 4 + num-worker: 7 tp: 2 ep: 2 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx4dep2_gen1dep16_mtp3_c384.yaml" + - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx7_gen1_dep16_batch32_eplb0_mtp3_666.yaml" decode: num-worker: 1 tp: 16 ep: 16 dp-attn: true - # 6P5D: 6 prefill (TP2/EP2/dp-attn), 5 decode (TP8/EP8/no-dp-attn), conc=480 + # 11P1D: 11 prefill (TP1/EP1/dp-attn), 1 decode (TP8/EP8/dp-attn), conc=1229 - spec-decoding: "mtp" - conc-list: [480] + conc-list: [1229] prefill: - num-worker: 6 - tp: 2 - ep: 2 + num-worker: 11 + tp: 1 + ep: 1 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx6dep2_gen5tep8_mtp3_c480.yaml" + - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx11_gen1_dep8_batch128_eplb0_mtp3_1229.yaml" decode: - num-worker: 5 + num-worker: 1 tp: 8 ep: 8 - dp-attn: false - # 9P1D: 9 prefill (TP2/EP2/dp-attn), 1 decode (TP16/EP16/dp-attn), conc=1536 + dp-attn: true + # 17P2D: 17 prefill (TP1/EP1/dp-attn), 2 decode (TP8/EP8/dp-attn), conc=1126 - spec-decoding: "mtp" - conc-list: [1536] + conc-list: [1126] prefill: - num-worker: 9 - tp: 2 - ep: 2 + num-worker: 17 + tp: 1 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx17_gen2_dep8_batch64_eplb0_mtp3_1126.yaml" + decode: + num-worker: 2 + tp: 8 + ep: 8 + dp-attn: true + # 26P1D: 26 prefill (TP1/EP1/dp-attn), 1 decode (TP16/EP16/dp-attn), conc=4301, MTP draft-len 2 + - spec-decoding: "mtp" + conc-list: [4301] + prefill: + num-worker: 26 + tp: 1 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx26_gen1_dep16_batch256_eplb0_mtp2_4301.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + # 24P1D: 24 prefill (TP1/EP1/dp-attn), 1 decode (TP16/EP16/dp-attn), conc=8192, MTP curve tail (spec decoding disabled) + - spec-decoding: "none" + conc-list: [8192] + prefill: + num-worker: 24 + tp: 1 + ep: 1 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx9dep2_gen1dep16_mtp3_c1536.yaml" + - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx24_gen1_dep16_batch512_eplb0_mtp0_8192.yaml" decode: num-worker: 1 tp: 16 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 719491a60a..ab92e7921c 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6563,3 +6563,16 @@ - "Filter AgentX traces at the same 202,752-token context limit used by both TileRT roles so oversized Weka trajectories are excluded before replay." - "Pin SemiAnalysisAI/srt-slurm PR #10 commit d1e6c97b3baf3e87103b6d83189544c3c7d61c38, stacked on the AMD/native-router PR #7 and base runtime PR #1, including explicit native HTTP dependencies, GLM-5.1-compatible Transformers v5 router tokenization, incomplete-snapshot recovery, backend-declared conversion GPU resources, pre-container NVIDIA driver-hook activation, and lossless Slurm container-environment exports." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2750 + +- config-keys: + - qwen3.5-fp4-gb300-dynamo-trt + - qwen3.5-fp4-gb300-dynamo-trt-mtp + description: + - "Refresh Qwen3.5-397B-A17B-NVFP4 GB300 disaggregated TensorRT-LLM+Dynamo STP and MTP (3-token, with one 2-token point) 8k1k configurations with a new locally validated recipe sweep from srt-slurm." + - "Image bumped from nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc18 to 1.3.0rc24; kv-p2p-transfer switched from ucx to nixl for the STP config to match the new recipes' NIXL cache-transceiver backend (the MTP config was already nixl)." + - "Switch checkpoint from nvidia/Qwen3.5-397B-A17B-NVFP4 to nvidia/Qwen3.5-397B-A17B-NVFP4-V2 (https://huggingface.co/nvidia/Qwen3.5-397B-A17B-NVFP4-V2), matching the checkpoint already used by the dynamo-sglang and dynamo-trt-agentic-disagg GB300 configs." + - "STP: 10 configs from 1P2D (conc 42) to 24P1D (conc 8192)." + - "MTP: 10 configs from 1P2D (conc 20) to 26P1D (conc 4301, 2-token draft); the 24P1D/conc 8192 tail point shares its recipe with the STP curve, where MTP no longer helps at saturation." + - "Recipes sourced from srt-slurm (recipes/trtllm/qwen3.5-fp4/inferencex/gb300/{mtp,stp})." + - "Runner: launch_gb300-nv.sh bumped from NVIDIA/srt-slurm@v1.0.29 to v1.0.72 for the dynamo-trt+qwen3.5+fp4 path." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2730 diff --git a/runners/launch_gb300-nv.sh b/runners/launch_gb300-nv.sh index 6038c46819..97c7a14505 100644 --- a/runners/launch_gb300-nv.sh +++ b/runners/launch_gb300-nv.sh @@ -411,7 +411,7 @@ elif [[ $FRAMEWORK == "dynamo-trt" && $MODEL_PREFIX == "dsv4" ]]; then elif [[ $FRAMEWORK == "dynamo-trt" && $MODEL_PREFIX == "qwen3.5" && $PRECISION == "fp4" ]]; then git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" - git checkout v1.0.29 + git checkout v1.0.72 mkdir -p recipes/trtllm/qwen3.5/gb300-fp4/disagg cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg" \ recipes/trtllm/qwen3.5/gb300-fp4/disagg @@ -534,7 +534,7 @@ inject_synthetic_acceptance "$CONFIG_PATH" "$FRAMEWORK" || exit 1 # /scratch/models, and # - qwen3.5 fp8, whose weights are also on the compute-node /scratch/models # and which runs on srt-slurm:v1.0.25 (the release that has the preflight), -# - qwen3.5 fp4 dynamo-trt, which runs on v1.0.29 without that preflight, and +# - qwen3.5 fp4 dynamo-trt, which runs on v1.0.72 without that preflight, and # - the qwen3.5 fp4 and dsv4 sglang power lanes, which run the pinned # producer (a main-lineage fork that has the preflight) against the same # /scratch checkpoints.