diff --git a/.github/workflows/test-process-result.yml b/.github/workflows/test-process-result.yml index 3d3147fab9..2e2af1d32e 100644 --- a/.github/workflows/test-process-result.yml +++ b/.github/workflows/test-process-result.yml @@ -9,6 +9,8 @@ on: - '.github/workflows/test-process-result.yml' - 'benchmarks/benchmark_lib.sh' - 'benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/**/*.yaml' + - 'benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/**/*.yaml' + - 'benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6/b200-fp4/**/*.yaml' - 'benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-h200-tp8-mtp-kvoffload.yaml' - 'benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/agentic/disagg-h200-2p2d-pcp8-tp8-dp8-mtp.yaml' - 'benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-*/**/*.yaml' @@ -16,6 +18,9 @@ on: - 'configs/nvidia-master.yaml' - 'runners/launch_gb200-nv.sh' - 'runners/launch_gb300-nv.sh' + - 'runners/launch_b200-dgxc.sh' + - 'runners/launch_b200-nscale-slurm.sh' + - 'runners/launch_b300-nv.sh' - 'runners/launch_h200-dgxc-slurm.sh' - 'runners/inject_srt_power_concurrencies.py' - 'utils/aggregate_power.py' @@ -30,6 +35,7 @@ on: - 'utils/process_result.py' - 'utils/test_aggregate_power.py' - 'utils/test_aggregate_power_multinode.py' + - 'utils/test_b200_b300_power_official_contract.py' - 'utils/test_gb200_power_official_contract.py' - 'utils/test_gb300_power_official_contract.py' - 'utils/test_h200_power_official_contract.py' @@ -62,4 +68,4 @@ jobs: - name: Run pytest run: | cd utils - python -m pytest test_aggregate_power.py test_aggregate_power_multinode.py agentic/aggregation/test_power_adapter.py agentic/aggregation/test_power_lifecycle.py agentic/aggregation/test_process_agentic_result.py test_gb200_power_official_contract.py test_gb300_power_official_contract.py test_h200_power_official_contract.py test_inject_srt_power_concurrencies.py test_process_result.py -v + python -m pytest test_aggregate_power.py test_aggregate_power_multinode.py agentic/aggregation/test_power_adapter.py agentic/aggregation/test_power_lifecycle.py agentic/aggregation/test_process_agentic_result.py test_b200_b300_power_official_contract.py test_gb200_power_official_contract.py test_gb300_power_official_contract.py test_h200_power_official_contract.py test_inject_srt_power_concurrencies.py test_process_result.py -v diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-low-latency-1p1d-tp8-tp8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-low-latency-1p1d-tp8-tp8-mtp.yaml index 685537c6a6..2e2016a15b 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-low-latency-1p1d-tp8-tp8-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-low-latency-1p1d-tp8-tp8-mtp.yaml @@ -118,3 +118,16 @@ benchmark: req_rate: "inf" use_chat_template: true custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-low-latency-1p6d-dep8-tp8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-low-latency-1p6d-dep8-tp8-mtp.yaml index 18b377b863..bfb83c48dd 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-low-latency-1p6d-dep8-tp8-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-low-latency-1p6d-dep8-tp8-mtp.yaml @@ -125,3 +125,16 @@ benchmark: req_rate: "inf" use_chat_template: true custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-mid-curve-1p1d-dep8-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-mid-curve-1p1d-dep8-dep8-mtp.yaml index ea30d07e55..b2291ca000 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-mid-curve-1p1d-dep8-dep8-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-mid-curve-1p1d-dep8-dep8-mtp.yaml @@ -126,3 +126,16 @@ benchmark: req_rate: "inf" use_chat_template: true custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-mid-curve-1p2d-dep8-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-mid-curve-1p2d-dep8-dep8-mtp.yaml index 9a6254e989..e709c508c8 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-mid-curve-1p2d-dep8-dep8-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-mid-curve-1p2d-dep8-dep8-mtp.yaml @@ -137,3 +137,16 @@ benchmark: req_rate: "inf" use_chat_template: true custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-mid-curve-5p3d-dep8-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-mid-curve-5p3d-dep8-dep8-mtp.yaml index e5d9a708a1..b1f2e1eba7 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-mid-curve-5p3d-dep8-dep8-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-mid-curve-5p3d-dep8-dep8-mtp.yaml @@ -126,3 +126,16 @@ benchmark: req_rate: "inf" use_chat_template: true custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-1p1d-dep4-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-1p1d-dep4-dep8-mtp.yaml index a07d5bb0a7..3ea1d3e722 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-1p1d-dep4-dep8-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-1p1d-dep4-dep8-mtp.yaml @@ -132,3 +132,16 @@ benchmark: req_rate: "inf" use_chat_template: true custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-1p1d-dep4-dep8.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-1p1d-dep4-dep8.yaml index 6612aae62f..42bd30e552 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-1p1d-dep4-dep8.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-1p1d-dep4-dep8.yaml @@ -143,3 +143,16 @@ benchmark: req_rate: "inf" use_chat_template: false custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-1p1d-tp4-tp4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-1p1d-tp4-tp4-mtp.yaml index 75c1815c8a..2293fa9a81 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-1p1d-tp4-tp4-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-1p1d-tp4-tp4-mtp.yaml @@ -123,3 +123,16 @@ benchmark: req_rate: "inf" use_chat_template: true custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-1p1d-tp4-tp4.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-1p1d-tp4-tp4.yaml index 6f0c0f7f68..e55d8dc2f5 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-1p1d-tp4-tp4.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-1p1d-tp4-tp4.yaml @@ -118,3 +118,16 @@ benchmark: req_rate: "inf" use_chat_template: false custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-1p2d-dep4-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-1p2d-dep4-dep8-mtp.yaml index fef5285ceb..5737b5d734 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-1p2d-dep4-dep8-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-1p2d-dep4-dep8-mtp.yaml @@ -133,3 +133,16 @@ benchmark: req_rate: "inf" use_chat_template: true custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-1p6d-dep4-tp4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-1p6d-dep4-tp4-mtp.yaml index d47b469945..fe5819b0af 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-1p6d-dep4-tp4-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-1p6d-dep4-tp4-mtp.yaml @@ -127,3 +127,16 @@ benchmark: req_rate: "inf" use_chat_template: true custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-2p1d-dep4-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-2p1d-dep4-dep8-mtp.yaml index e796e2c680..96912af606 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-2p1d-dep4-dep8-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-2p1d-dep4-dep8-mtp.yaml @@ -133,3 +133,16 @@ benchmark: req_rate: "inf" use_chat_template: true custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-2p1d-dep4-dep8.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-2p1d-dep4-dep8.yaml index 565a74d739..4c5c5ef4db 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-2p1d-dep4-dep8.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-2p1d-dep4-dep8.yaml @@ -143,3 +143,16 @@ benchmark: req_rate: "inf" use_chat_template: false custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-4p1d-dep4-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-4p1d-dep4-dep8-mtp.yaml index 681f350d90..2b9713b708 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-4p1d-dep4-dep8-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-4p1d-dep4-dep8-mtp.yaml @@ -133,3 +133,16 @@ benchmark: req_rate: "inf" use_chat_template: true custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-4p1d-dep4-dep8.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-4p1d-dep4-dep8.yaml index 9a8123c439..8ad309af71 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-4p1d-dep4-dep8.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-4p1d-dep4-dep8.yaml @@ -143,3 +143,16 @@ benchmark: req_rate: "inf" use_chat_template: false custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-6p1d-dep4-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-6p1d-dep4-dep8-mtp.yaml index 4813ac793f..f7ff198115 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-6p1d-dep4-dep8-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-6p1d-dep4-dep8-mtp.yaml @@ -133,3 +133,16 @@ benchmark: req_rate: "inf" use_chat_template: true custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-6p1d-dep4-dep8.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-6p1d-dep4-dep8.yaml index c47335237a..6ba7519714 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-6p1d-dep4-dep8.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-6p1d-dep4-dep8.yaml @@ -143,3 +143,16 @@ benchmark: req_rate: "inf" use_chat_template: false custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-8p1d-dep4-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-8p1d-dep4-dep8-mtp.yaml index ed3b27e906..7bdc43dd73 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-8p1d-dep4-dep8-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-8p1d-dep4-dep8-mtp.yaml @@ -133,3 +133,16 @@ benchmark: req_rate: "inf" use_chat_template: true custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-1p1d-dep8-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-1p1d-dep8-dep8-mtp.yaml index d317a218a7..72e6dd1b16 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-1p1d-dep8-dep8-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-1p1d-dep8-dep8-mtp.yaml @@ -31,6 +31,7 @@ frontend: type: dynamo enable_multiple_frontends: false backend: + set_cuda_visible_devices: true # keep the aflowers/vllm-gb200-v0.20.0 provenance: bind GPUs via CVD, not --device-ids type: vllm connector: null prefill_environment: @@ -127,3 +128,16 @@ benchmark: req_rate: "inf" use_chat_template: true custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer" + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-1p1d-dep8-tp8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-1p1d-dep8-tp8-mtp.yaml index b73afe1adf..dc4e403072 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-1p1d-dep8-tp8-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-1p1d-dep8-tp8-mtp.yaml @@ -31,6 +31,7 @@ frontend: type: dynamo enable_multiple_frontends: false backend: + set_cuda_visible_devices: true # keep the aflowers/vllm-gb200-v0.20.0 provenance: bind GPUs via CVD, not --device-ids type: vllm connector: null prefill_environment: @@ -112,3 +113,16 @@ benchmark: req_rate: "inf" use_chat_template: true custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer" + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-1p2d-dep8-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-1p2d-dep8-dep8-mtp.yaml index 5c9eb254e3..1bebd09a27 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-1p2d-dep8-dep8-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-1p2d-dep8-dep8-mtp.yaml @@ -31,6 +31,7 @@ frontend: type: dynamo enable_multiple_frontends: false backend: + set_cuda_visible_devices: true # keep the aflowers/vllm-gb200-v0.20.0 provenance: bind GPUs via CVD, not --device-ids type: vllm connector: null prefill_environment: @@ -128,3 +129,16 @@ benchmark: req_rate: "inf" use_chat_template: true custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer" + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-2p1d-dep8-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-2p1d-dep8-dep8-mtp.yaml index f71c4b3a3e..2c9e9e2feb 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-2p1d-dep8-dep8-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-2p1d-dep8-dep8-mtp.yaml @@ -31,6 +31,7 @@ frontend: type: dynamo enable_multiple_frontends: false backend: + set_cuda_visible_devices: true # keep the aflowers/vllm-gb200-v0.20.0 provenance: bind GPUs via CVD, not --device-ids type: vllm connector: null prefill_environment: @@ -127,3 +128,16 @@ benchmark: req_rate: "inf" use_chat_template: true custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer" + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-3p1d-dep8-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-3p1d-dep8-dep8-mtp.yaml index 7eb6c47a34..0b7841266e 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-3p1d-dep8-dep8-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-3p1d-dep8-dep8-mtp.yaml @@ -31,6 +31,7 @@ frontend: type: dynamo enable_multiple_frontends: false backend: + set_cuda_visible_devices: true # keep the aflowers/vllm-gb200-v0.20.0 provenance: bind GPUs via CVD, not --device-ids type: vllm connector: null prefill_environment: @@ -127,3 +128,16 @@ benchmark: req_rate: "inf" use_chat_template: true custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer" + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-low-latency-c1.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-low-latency-c1.yaml index e81ef8631c..26805d4efd 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-low-latency-c1.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-low-latency-c1.yaml @@ -29,6 +29,7 @@ frontend: enable_multiple_frontends: false type: dynamo backend: + set_cuda_visible_devices: true # keep the aflowers/vllm-gb200-v0.20.0 provenance: bind GPUs via CVD, not --device-ids connector: null decode_environment: NCCL_CUMEM_ENABLE: '1' @@ -111,3 +112,16 @@ identity: frameworks: dynamo: "1.2.0.dev20260426" vllm: "0.23.0" + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-low-latency-c32-c128.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-low-latency-c32-c128.yaml index b296440158..dc2fd3bdaa 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-low-latency-c32-c128.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-low-latency-c32-c128.yaml @@ -29,6 +29,7 @@ frontend: enable_multiple_frontends: false type: dynamo backend: + set_cuda_visible_devices: true # keep the aflowers/vllm-gb200-v0.20.0 provenance: bind GPUs via CVD, not --device-ids connector: null decode_environment: NCCL_CUMEM_ENABLE: '1' @@ -111,3 +112,16 @@ identity: frameworks: dynamo: "1.2.0.dev20260426" vllm: "0.23.0" + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-low-latency-c64.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-low-latency-c64.yaml index b566c05fae..a1c93b179d 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-low-latency-c64.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-low-latency-c64.yaml @@ -29,6 +29,7 @@ frontend: enable_multiple_frontends: false type: dynamo backend: + set_cuda_visible_devices: true # keep the aflowers/vllm-gb200-v0.20.0 provenance: bind GPUs via CVD, not --device-ids connector: null decode_environment: NCCL_CUMEM_ENABLE: '1' @@ -111,3 +112,16 @@ identity: frameworks: dynamo: "1.2.0.dev20260426" vllm: "0.23.0" + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-low-middle-c256.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-low-middle-c256.yaml index 43190836a1..fe4b49077b 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-low-middle-c256.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-low-middle-c256.yaml @@ -29,6 +29,7 @@ frontend: enable_multiple_frontends: false type: dynamo backend: + set_cuda_visible_devices: true # keep the aflowers/vllm-gb200-v0.20.0 provenance: bind GPUs via CVD, not --device-ids connector: null decode_environment: NCCL_CUMEM_ENABLE: '1' @@ -115,3 +116,16 @@ identity: frameworks: dynamo: "1.2.0.dev20260426" vllm: "0.23.0" + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-low-middle-c512.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-low-middle-c512.yaml index 283bb839f3..8fe69b472f 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-low-middle-c512.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-low-middle-c512.yaml @@ -29,6 +29,7 @@ frontend: type: dynamo enable_multiple_frontends: false backend: + set_cuda_visible_devices: true # keep the aflowers/vllm-gb200-v0.20.0 provenance: bind GPUs via CVD, not --device-ids type: vllm connector: null prefill_environment: @@ -116,3 +117,16 @@ identity: frameworks: dynamo: "1.2.0.dev20260426" vllm: "0.23.0" + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b300-high-tpt-megamoe.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b300-high-tpt-megamoe.yaml index 9247a31daf..b16d38d33d 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b300-high-tpt-megamoe.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b300-high-tpt-megamoe.yaml @@ -38,6 +38,7 @@ frontend: enable_multiple_frontends: false backend: + set_cuda_visible_devices: true # keep the aflowers/vllm-gb200-v0.20.0 provenance: bind GPUs via CVD, not --device-ids type: vllm connector: null prefill_environment: @@ -128,3 +129,16 @@ identity: frameworks: dynamo: "1.2.0.dev20260426" vllm: "0.23.0" + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b300-low-latency.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b300-low-latency.yaml index 260e16fc7d..8f2a01f5c5 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b300-low-latency.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b300-low-latency.yaml @@ -38,6 +38,7 @@ frontend: enable_multiple_frontends: false backend: + set_cuda_visible_devices: true # keep the aflowers/vllm-gb200-v0.20.0 provenance: bind GPUs via CVD, not --device-ids type: vllm connector: null prefill_environment: @@ -122,3 +123,16 @@ identity: frameworks: dynamo: "1.2.0.dev20260426" vllm: "0.23.0" + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b300-mid-curve-megamoe.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b300-mid-curve-megamoe.yaml index 98b701790a..557f6b1e95 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b300-mid-curve-megamoe.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b300-mid-curve-megamoe.yaml @@ -38,6 +38,7 @@ frontend: enable_multiple_frontends: false backend: + set_cuda_visible_devices: true # keep the aflowers/vllm-gb200-v0.20.0 provenance: bind GPUs via CVD, not --device-ids type: vllm connector: null prefill_environment: @@ -128,3 +129,16 @@ identity: frameworks: dynamo: "1.2.0.dev20260426" vllm: "0.23.0" + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p1d-dep8-dep8-c2048.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p1d-dep8-dep8-c2048.yaml index e3c08dc477..b7378fe35f 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p1d-dep8-dep8-c2048.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p1d-dep8-dep8-c2048.yaml @@ -110,3 +110,16 @@ benchmark: concurrencies: '2048' req_rate: inf use_chat_template: true + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p1d-dep8-tp8-c1.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p1d-dep8-tp8-c1.yaml index 1c0abe2fa0..cf8d37ff59 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p1d-dep8-tp8-c1.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p1d-dep8-tp8-c1.yaml @@ -108,3 +108,16 @@ benchmark: concurrencies: '1' req_rate: inf use_chat_template: true + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p4d-dep4-tp4-c512.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p4d-dep4-tp4-c512.yaml index 1ca87ad9b8..f17b45287c 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p4d-dep4-tp4-c512.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p4d-dep4-tp4-c512.yaml @@ -107,3 +107,16 @@ benchmark: concurrencies: '512' req_rate: inf use_chat_template: true + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p8d-dep4-tp4-c128.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p8d-dep4-tp4-c128.yaml index 44f8ff7849..e33c6b773f 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p8d-dep4-tp4-c128.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p8d-dep4-tp4-c128.yaml @@ -107,3 +107,16 @@ benchmark: concurrencies: '128' req_rate: inf use_chat_template: true + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p8d-dep4-tp4-c32.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p8d-dep4-tp4-c32.yaml index c748107fec..07f3c13792 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p8d-dep4-tp4-c32.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p8d-dep4-tp4-c32.yaml @@ -107,3 +107,16 @@ benchmark: concurrencies: '32' req_rate: inf use_chat_template: true + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-2p1d-dep8-dep8-c8192.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-2p1d-dep8-dep8-c8192.yaml index 54e623d767..9c80baf673 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-2p1d-dep8-dep8-c8192.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-2p1d-dep8-dep8-c8192.yaml @@ -110,3 +110,16 @@ benchmark: concurrencies: '8192' req_rate: inf use_chat_template: true + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index d51ee4c22b..de9533314a 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6475,3 +6475,18 @@ - "Replace the TP2/EP2 arm with TP2/EP1 so the 2-GPU point matches qwen3.5-fp4-b200-sglang-agentic-mtp." - "Add HiCache host-DRAM KV tier arms at TP4 concurrency 40, 48, 56, and 64 and TP2 concurrency 20, 24, 28, and 32, using hicache ratio 1.5 with write_through, direct io, and page_first_direct layout." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2693 + + +- config-keys: + - kimik2.6-fp4-b200-dynamo-vllm + - dsv4-fp4-b200-dynamo-vllm + - dsv4-fp4-b300-dynamo-vllm + - dsv4-fp4-b300-dynamo-sglang + - dsv4-fp4-b200-dynamo-sglang-mtp + - dsv4-fp4-b200-dynamo-vllm-mtp + - dsv4-fp4-b300-dynamo-sglang-mtp + description: + - "Enable required dcgm-power telemetry on 37 eligible recipes: 21 B200 and 16 B300." + - "Required lanes promote exporter startup timeouts, launch failures, and endpoint resolution failures to blocking validation failures, so a recipe cannot publish a result whose power collection never started." + - "Route only enabled recipes through the immutable producer fork and preserve non-power launcher revisions." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2688 diff --git a/runners/launch_b200-dgxc.sh b/runners/launch_b200-dgxc.sh index 498ad7d1c1..f23701f1f5 100644 --- a/runners/launch_b200-dgxc.sh +++ b/runners/launch_b200-dgxc.sh @@ -3,6 +3,8 @@ # System-specific configuration for B200 DGXC Slurm cluster SLURM_PARTITION="${SLURM_PARTITION:-gpu-2}" SLURM_ACCOUNT="${SLURM_ACCOUNT:-benchmark}" +POWER_SRT_SLURM_URL="https://github.com/edwingao28/srt-slurm.git" +POWER_SRT_SLURM_PIN="e5c837f06a362dc888dfea2ee588e9f19c298270" set -x @@ -158,6 +160,32 @@ if [[ "$IS_MULTINODE" == "true" ]]; then exit 1 fi + USES_DCGM_POWER=0 + _POWER_CONFIG_FILE="${CONFIG_FILE:-}" + if [[ "${EVAL_ONLY:-false}" == "true" && -n "${EVAL_CONFIG_FILE:-}" ]]; then + _POWER_CONFIG_FILE="$EVAL_CONFIG_FILE" + fi + _RECIPE_REL="${_POWER_CONFIG_FILE%%:*}" + _RECIPE_SRC="$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/${_RECIPE_REL#recipes/}" + if [[ -n "$_POWER_CONFIG_FILE" && -f "$_RECIPE_SRC" ]] && awk ' + /^telemetry:/ { t = 1; next } + t && /^[^ ]/ { t = 0 } + t && /^ provider: dcgm-power$/ { p = 1 } + t && /^ enabled: true$/ { e = 1 } + END { exit !(p && e) } + ' "$_RECIPE_SRC"; then + USES_DCGM_POWER=1 + fi + if [[ "$USES_DCGM_POWER" == "1" && ( + "${IS_AGENTIC:-0}" == "1" || + "$MODEL_PREFIX" != "dsv4" || + "$PRECISION" != "fp4" || + "$FRAMEWORK" != "dynamo-vllm" + ) ]]; then + echo "Error: B200 DGXC dcgm-power is limited to fixed-sequence DSV4 FP4 dynamo-vllm" >&2 + exit 1 + fi + export SERVED_MODEL_NAME=$MODEL echo "Cloning srt-slurm repository..." @@ -170,7 +198,15 @@ if [[ "$IS_MULTINODE" == "true" ]]; then # Kimi K3 aggregate profiles use the srt-slurm fork that supports direct # multi-node vLLM. Pin the tested renderer so branch movement cannot change # generated rank commands between sweep points. - if [[ "$IS_AGENTIC" == "1" && $MODEL_PREFIX == "kimik3" ]]; then + if [[ "$USES_DCGM_POWER" == "1" ]]; then + git clone "$POWER_SRT_SLURM_URL" "$SRT_REPO_DIR" || exit 1 + cd "$SRT_REPO_DIR" || exit 1 + git checkout "$POWER_SRT_SLURM_PIN" || exit 1 + test "$(git rev-parse HEAD)" = "$POWER_SRT_SLURM_PIN" || { echo "Error: srt-slurm HEAD does not match POWER_SRT_SLURM_PIN=$POWER_SRT_SLURM_PIN" >&2; exit 1; } + git rev-parse HEAD > "$GITHUB_WORKSPACE/power-producer-sha.txt" + mkdir -p recipes/vllm/deepseek-v4 + cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4" recipes/vllm/deepseek-v4 + elif [[ "$IS_AGENTIC" == "1" && $MODEL_PREFIX == "kimik3" ]]; then git clone --branch klaud/direct-vllm-multinode --single-branch https://github.com/functionstackx/srt-slurm-nv.git "$SRT_REPO_DIR" || exit 1 cd "$SRT_REPO_DIR" || exit 1 git checkout df5baa93f4caf5169dea2a4236ad2cc742fe40e7 || exit 1 @@ -286,6 +322,17 @@ if [[ "$IS_MULTINODE" == "true" ]]; then import_squash "$SQUASH_FILE" "$IMAGE" || exit 1 import_squash "$NGINX_SQUASH_FILE" "$NGINX_IMAGE" || exit 1 + if [[ "$USES_DCGM_POWER" == "1" ]]; then + DCGM_EXPORTER_IMAGE="nvcr.io/nvidia/k8s/dcgm-exporter:4.6.0-4.8.3-distroless" + # enroot resolves bare paths against Docker Hub; nvcr.io pulls need the registry# form + DCGM_EXPORTER_ENROOT_REF="${DCGM_EXPORTER_IMAGE/nvcr.io\//nvcr.io#}" + DCGM_EXPORTER_SQSH="$SQUASH_DIR/$(echo "$DCGM_EXPORTER_IMAGE" | sed 's/[\/:@#]/_/g').sqsh" + import_squash "$DCGM_EXPORTER_SQSH" "$DCGM_EXPORTER_ENROOT_REF" || exit 1 + test -r "$DCGM_EXPORTER_SQSH" || { echo "Error: DCGM exporter squash not readable: $DCGM_EXPORTER_SQSH" >&2; exit 1; } + unsquashfs -l "$DCGM_EXPORTER_SQSH" > /dev/null || { echo "Error: DCGM exporter squash invalid: $DCGM_EXPORTER_SQSH" >&2; exit 1; } + sha256sum "$DCGM_EXPORTER_SQSH" > "$GITHUB_WORKSPACE/exporter-image.sha256" + fi + export ISL="$ISL" export OSL="$OSL" export EVAL_ONLY="${EVAL_ONLY:-false}" @@ -336,6 +383,11 @@ use_exclusive_sbatch_directive: true ${DEFAULT_MOUNTS_BLOCK} EOF + if [[ "$USES_DCGM_POWER" == "1" ]]; then + sed -i "/^ nginx-sqsh:/a\\ dcgm-exporter: ${DCGM_EXPORTER_SQSH}" srtslurm.yaml + grep -q "^ dcgm-exporter: " srtslurm.yaml || { echo "Error: dcgm-exporter injection failed: nginx-sqsh anchor not found in srtslurm.yaml" >&2; exit 1; } + fi + echo "Generated srtslurm.yaml:" cat srtslurm.yaml @@ -435,6 +487,12 @@ EOF echo "Found logs directory: $LOGS_DIR" + if [[ "$USES_DCGM_POWER" == "1" ]]; then + mkdir -p "$LOGS_DIR/power" + cp "$GITHUB_WORKSPACE/exporter-image.sha256" "$LOGS_DIR/power/exporter-image.sha256" + cp "$GITHUB_WORKSPACE/power-producer-sha.txt" "$LOGS_DIR/power/power-producer-sha.txt" + fi + cp -r "$LOGS_DIR" "$GITHUB_WORKSPACE/LOGS" tar czf "$GITHUB_WORKSPACE/multinode_server_logs.tar.gz" -C "$LOGS_DIR" . diff --git a/runners/launch_b200-nscale-slurm.sh b/runners/launch_b200-nscale-slurm.sh index 4bf9531d0f..e9fbd9a5e6 100755 --- a/runners/launch_b200-nscale-slurm.sh +++ b/runners/launch_b200-nscale-slurm.sh @@ -14,6 +14,8 @@ SLURM_PARTITION="batch_1" SLURM_ACCOUNT="benchmark" +POWER_SRT_SLURM_URL="https://github.com/edwingao28/srt-slurm.git" +POWER_SRT_SLURM_PIN="e5c837f06a362dc888dfea2ee588e9f19c298270" # Node-local NVMe, not a shared filesystem: much faster for the ~1.6T # DeepSeek-V4-Pro load, and already pre-staged on every nscale compute node. @@ -59,12 +61,55 @@ if [[ $FRAMEWORK != "dynamo-vllm" ]] && exit 1 fi +USES_DCGM_POWER=0 +_POWER_CONFIG_FILE="${CONFIG_FILE:-}" +if [[ "${EVAL_ONLY:-false}" == "true" && -n "${EVAL_CONFIG_FILE:-}" ]]; then + _POWER_CONFIG_FILE="$EVAL_CONFIG_FILE" +fi +_RECIPE_REL="${_POWER_CONFIG_FILE%%:*}" +_RECIPE_SRC="$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/${_RECIPE_REL#recipes/}" +if [[ -n "$_POWER_CONFIG_FILE" && -f "$_RECIPE_SRC" ]] && awk ' + /^telemetry:/ { t = 1; next } + t && /^[^ ]/ { t = 0 } + t && /^ provider: dcgm-power$/ { p = 1 } + t && /^ enabled: true$/ { e = 1 } + END { exit !(p && e) } +' "$_RECIPE_SRC"; then + USES_DCGM_POWER=1 +fi +if [[ "$USES_DCGM_POWER" == "1" && ( + "${IS_AGENTIC:-0}" == "1" || + "$PRECISION" != "fp4" || + ( "$MODEL_PREFIX" == "dsv4" && "$FRAMEWORK" != "dynamo-sglang" && "$FRAMEWORK" != "dynamo-vllm" ) || + ( "$MODEL_PREFIX" == "kimik2.6" && "$FRAMEWORK" != "dynamo-vllm" ) || + ( "$MODEL_PREFIX" != "dsv4" && "$MODEL_PREFIX" != "kimik2.6" ) +) ]]; then + echo "Error: B200 nscale dcgm-power is limited to fixed-sequence DSV4/Kimi-K2.6 FP4 lanes" >&2 + exit 1 +fi + export SERVED_MODEL_NAME=$MODEL echo "Cloning srt-slurm repository..." SRT_REPO_DIR="srt-slurm" rm -rf "$SRT_REPO_DIR" -if [[ "$IS_AGENTIC" == "1" && $MODEL_PREFIX == "kimik3" ]]; then +if [[ "$USES_DCGM_POWER" == "1" ]]; then + git clone "$POWER_SRT_SLURM_URL" "$SRT_REPO_DIR" || exit 1 + cd "$SRT_REPO_DIR" || exit 1 + git checkout "$POWER_SRT_SLURM_PIN" || exit 1 + test "$(git rev-parse HEAD)" = "$POWER_SRT_SLURM_PIN" || { echo "Error: srt-slurm HEAD does not match POWER_SRT_SLURM_PIN=$POWER_SRT_SLURM_PIN" >&2; exit 1; } + git rev-parse HEAD > "$GITHUB_WORKSPACE/power-producer-sha.txt" + if [[ "$MODEL_PREFIX" == "dsv4" && "$FRAMEWORK" == "dynamo-sglang" ]]; then + mkdir -p recipes/sglang/deepseek-v4 + cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4" recipes/sglang/deepseek-v4 + elif [[ "$MODEL_PREFIX" == "dsv4" ]]; then + mkdir -p recipes/vllm/deepseek-v4 + cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4" recipes/vllm/deepseek-v4 + else + mkdir -p recipes/vllm/kimi-k2.6 + cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6" recipes/vllm/kimi-k2.6 + fi +elif [[ "$IS_AGENTIC" == "1" && $MODEL_PREFIX == "kimik3" ]]; then # Pin the tested renderer so branch movement cannot change generated rank # commands between sweep points. git clone --branch main --single-branch https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" || exit 1 @@ -149,6 +194,17 @@ import_squash() { import_squash "$SQUASH_FILE" "$IMAGE" || exit 1 import_squash "$NGINX_SQUASH_FILE" "$NGINX_IMAGE" || exit 1 +if [[ "$USES_DCGM_POWER" == "1" ]]; then + DCGM_EXPORTER_IMAGE="nvcr.io/nvidia/k8s/dcgm-exporter:4.6.0-4.8.3-distroless" + # enroot resolves bare paths against Docker Hub; nvcr.io pulls need the registry# form + DCGM_EXPORTER_ENROOT_REF="${DCGM_EXPORTER_IMAGE/nvcr.io\//nvcr.io#}" + DCGM_EXPORTER_SQSH="$SQUASH_DIR/$(echo "$DCGM_EXPORTER_IMAGE" | sed 's/[\/:@#]/_/g').sqsh" + import_squash "$DCGM_EXPORTER_SQSH" "$DCGM_EXPORTER_ENROOT_REF" || exit 1 + test -r "$DCGM_EXPORTER_SQSH" || { echo "Error: DCGM exporter squash not readable: $DCGM_EXPORTER_SQSH" >&2; exit 1; } + unsquashfs -l "$DCGM_EXPORTER_SQSH" > /dev/null || { echo "Error: DCGM exporter squash invalid: $DCGM_EXPORTER_SQSH" >&2; exit 1; } + sha256sum "$DCGM_EXPORTER_SQSH" > "$GITHUB_WORKSPACE/exporter-image.sha256" +fi + export ISL="$ISL" export OSL="$OSL" export EVAL_ONLY="${EVAL_ONLY:-false}" @@ -193,6 +249,11 @@ use_exclusive_sbatch_directive: true ${DEFAULT_MOUNTS_BLOCK} EOF +if [[ "$USES_DCGM_POWER" == "1" ]]; then + sed -i "/^ nginx-sqsh:/a\\ dcgm-exporter: ${DCGM_EXPORTER_SQSH}" srtslurm.yaml + grep -q "^ dcgm-exporter: " srtslurm.yaml || { echo "Error: dcgm-exporter injection failed: nginx-sqsh anchor not found in srtslurm.yaml" >&2; exit 1; } +fi + echo "Generated srtslurm.yaml:" cat srtslurm.yaml @@ -235,7 +296,7 @@ SRTCTL_PREFLIGHT_ARGS=() # These weights are staged on the Slurm compute nodes, not the login node. if [[ $MODEL_PREFIX == "kimik2.6" ]] || [[ $MODEL_PREFIX == "kimik3" ]] || - [[ $MODEL_PREFIX == "dsv4" && $FRAMEWORK == "dynamo-sglang" ]]; then + [[ $MODEL_PREFIX == "dsv4" ]]; then SRTCTL_PREFLIGHT_ARGS+=(--no-preflight) fi @@ -270,6 +331,12 @@ if [ ! -d "$LOGS_DIR" ]; then exit 1 fi +if [[ "$USES_DCGM_POWER" == "1" ]]; then + mkdir -p "$LOGS_DIR/power" + cp "$GITHUB_WORKSPACE/exporter-image.sha256" "$LOGS_DIR/power/exporter-image.sha256" + cp "$GITHUB_WORKSPACE/power-producer-sha.txt" "$LOGS_DIR/power/power-producer-sha.txt" +fi + cp -r "$LOGS_DIR" "$GITHUB_WORKSPACE/LOGS" bundle_server_logs "$LOGS_DIR" "$GITHUB_WORKSPACE/multinode_server_logs.tar.gz" diff --git a/runners/launch_b300-nv.sh b/runners/launch_b300-nv.sh index f7878a8ad8..37d35c7b3d 100644 --- a/runners/launch_b300-nv.sh +++ b/runners/launch_b300-nv.sh @@ -3,6 +3,8 @@ # System-specific configuration for B300 NV Slurm cluster (sa-shared) SLURM_PARTITION="batch_1" SLURM_ACCOUNT="benchmark" +POWER_SRT_SLURM_URL="https://github.com/edwingao28/srt-slurm.git" +POWER_SRT_SLURM_PIN="e5c837f06a362dc888dfea2ee588e9f19c298270" # b300-018 repeatedly times out UCX/NIXL transfers; allow an empty override to disable this. MINIMAX_M3_SLURM_EXCLUDED_NODELIST="${MINIMAX_M3_SLURM_EXCLUDED_NODELIST-b300-018}" @@ -16,6 +18,28 @@ if [[ $FRAMEWORK != "dynamo-sglang" && $FRAMEWORK != "dynamo-trt" && $FRAMEWORK exit 1 fi +USES_DCGM_POWER=0 +_RECIPE_REL="${CONFIG_FILE%%:*}" +_RECIPE_SRC="$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/${_RECIPE_REL#recipes/}" +if [[ -n "$CONFIG_FILE" && -f "$_RECIPE_SRC" ]] && awk ' + /^telemetry:/ { t = 1; next } + t && /^[^ ]/ { t = 0 } + t && /^ provider: dcgm-power$/ { p = 1 } + t && /^ enabled: true$/ { e = 1 } + END { exit !(p && e) } +' "$_RECIPE_SRC"; then + USES_DCGM_POWER=1 +fi +if [[ "$USES_DCGM_POWER" == "1" && ( + "${IS_AGENTIC:-0}" == "1" || + "$MODEL_PREFIX" != "dsv4" || + "$PRECISION" != "fp4" || + ( "$FRAMEWORK" != "dynamo-sglang" && "$FRAMEWORK" != "dynamo-vllm" ) +) ]]; then + echo "Error: B300 dcgm-power is limited to fixed-sequence DSV4 FP4 dynamo-sglang/vllm" >&2 + exit 1 +fi + # MODEL_PATH: Override with pre-downloaded paths on B300 runner # The yaml files specify HuggingFace model IDs for portability, but we use # local paths to avoid repeated downloading on the shared B300 cluster. @@ -70,7 +94,20 @@ if [ -d "$SRT_REPO_DIR" ]; then fi # TODO(CJQ): make first class upon srt-slurm upstream refactor -if [[ "$IS_AGENTIC" == "1" ]]; then +if [[ "$USES_DCGM_POWER" == "1" ]]; then + git clone "$POWER_SRT_SLURM_URL" "$SRT_REPO_DIR" || exit 1 + cd "$SRT_REPO_DIR" || exit 1 + git checkout "$POWER_SRT_SLURM_PIN" || exit 1 + test "$(git rev-parse HEAD)" = "$POWER_SRT_SLURM_PIN" || { echo "Error: srt-slurm HEAD does not match POWER_SRT_SLURM_PIN=$POWER_SRT_SLURM_PIN" >&2; exit 1; } + git rev-parse HEAD > "$GITHUB_WORKSPACE/power-producer-sha.txt" + if [[ "$FRAMEWORK" == "dynamo-sglang" ]]; then + mkdir -p recipes/sglang/deepseek-v4 + cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4" recipes/sglang/deepseek-v4 + else + mkdir -p recipes/vllm/deepseek-v4 + cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4" recipes/vllm/deepseek-v4 + fi +elif [[ "$IS_AGENTIC" == "1" ]]; then git clone --branch cam/sa-submission-q2-2026 --single-branch https://github.com/cquil11/srt-slurm-nv.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" || exit 1 elif [[ $FRAMEWORK == "dynamo-vllm" && $MODEL_PREFIX == "dsv4" ]]; then @@ -139,6 +176,27 @@ NGINX_SQUASH_FILE="/data/squash/$(echo "$NGINX_IMAGE" | sed 's/[\/:@#]/_/g').sqs srun -N 1 -A $SLURM_ACCOUNT -p $SLURM_PARTITION bash -c "enroot import -o $SQUASH_FILE docker://$IMAGE" srun -N 1 -A $SLURM_ACCOUNT -p $SLURM_PARTITION bash -c "enroot import -o $NGINX_SQUASH_FILE docker://$NGINX_IMAGE" +if [[ "$USES_DCGM_POWER" == "1" ]]; then + DCGM_EXPORTER_IMAGE="nvcr.io/nvidia/k8s/dcgm-exporter:4.6.0-4.8.3-distroless" + # enroot resolves bare paths against Docker Hub; nvcr.io pulls need the registry# form + DCGM_EXPORTER_ENROOT_REF="${DCGM_EXPORTER_IMAGE/nvcr.io\//nvcr.io#}" + DCGM_EXPORTER_SQSH="/data/squash/$(echo "$DCGM_EXPORTER_IMAGE" | sed 's/[\/:@#]/_/g').sqsh" + DCGM_EXPORTER_LOCK="${DCGM_EXPORTER_SQSH}.lock" + srun -N 1 -A "$SLURM_ACCOUNT" -p "$SLURM_PARTITION" bash -c " + set -euo pipefail + exec 9>\"$DCGM_EXPORTER_LOCK\" + flock -w 1800 9 + if unsquashfs -l \"$DCGM_EXPORTER_SQSH\" > /dev/null 2>&1; then + exit 0 + fi + rm -f \"$DCGM_EXPORTER_SQSH\" + enroot import -o \"$DCGM_EXPORTER_SQSH\" \"docker://$DCGM_EXPORTER_ENROOT_REF\" + unsquashfs -l \"$DCGM_EXPORTER_SQSH\" > /dev/null + " || exit 1 + test -r "$DCGM_EXPORTER_SQSH" || { echo "Error: DCGM exporter squash not readable: $DCGM_EXPORTER_SQSH" >&2; exit 1; } + sha256sum "$DCGM_EXPORTER_SQSH" > "$GITHUB_WORKSPACE/exporter-image.sha256" +fi + export ISL="$ISL" export OSL="$OSL" export EVAL_ONLY="${EVAL_ONLY:-false}" @@ -173,6 +231,11 @@ default_mounts: "/opt/ucx-no-ud": "/usr/local/ucx" EOF +if [[ "$USES_DCGM_POWER" == "1" ]]; then + sed -i "/^ nginx-sqsh:/a\\ dcgm-exporter: ${DCGM_EXPORTER_SQSH}" srtslurm.yaml + grep -q "^ dcgm-exporter: " srtslurm.yaml || { echo "Error: dcgm-exporter injection failed: nginx-sqsh anchor not found in srtslurm.yaml" >&2; exit 1; } +fi + echo "Generated srtslurm.yaml:" cat srtslurm.yaml @@ -288,6 +351,12 @@ fi echo "Found logs directory: $LOGS_DIR" +if [[ "$USES_DCGM_POWER" == "1" ]]; then + mkdir -p "$LOGS_DIR/power" + cp "$GITHUB_WORKSPACE/exporter-image.sha256" "$LOGS_DIR/power/exporter-image.sha256" + cp "$GITHUB_WORKSPACE/power-producer-sha.txt" "$LOGS_DIR/power/power-producer-sha.txt" +fi + cp -r "$LOGS_DIR" "$GITHUB_WORKSPACE/LOGS" tar czf "$GITHUB_WORKSPACE/multinode_server_logs.tar.gz" -C "$LOGS_DIR" . diff --git a/utils/test_b200_b300_power_official_contract.py b/utils/test_b200_b300_power_official_contract.py new file mode 100644 index 0000000000..e083409df0 --- /dev/null +++ b/utils/test_b200_b300_power_official_contract.py @@ -0,0 +1,251 @@ +"""Contract for required B200/B300 multinode dcgm-power lanes.""" + +from __future__ import annotations + +import re +import subprocess +from collections import Counter +from pathlib import Path + +import yaml + +REPO_ROOT = Path(__file__).resolve().parents[1] +RECIPE_ROOT = REPO_ROOT / "benchmarks/multi_node/srt-slurm-recipes" +MASTER = REPO_ROOT / "configs/nvidia-master.yaml" +WORKFLOW = REPO_ROOT / ".github/workflows/test-process-result.yml" +LAUNCHERS = { + "cluster:b200-dgxc": REPO_ROOT / "runners/launch_b200-dgxc.sh", + "b200-nscale": REPO_ROOT / "runners/launch_b200-nscale-slurm.sh", + "b200-new": REPO_ROOT / "runners/launch_b200-nscale-slurm.sh", + "b300": REPO_ROOT / "runners/launch_b300-nv.sh", +} + +PRODUCER_URL = "https://github.com/edwingao28/srt-slurm.git" +PRODUCER_SHA = "e5c837f06a362dc888dfea2ee588e9f19c298270" +EXPORTER_IMAGE = "nvcr.io/nvidia/k8s/dcgm-exporter:4.6.0-4.8.3-distroless" +PROCESS_RESULT_RECIPE_GLOBS = ( + "benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/**/*.yaml", + "benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/**/*.yaml", + "benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6/b200-fp4/**/*.yaml", +) + + +def _config_files(value): + if isinstance(value, dict): + for child in value.values(): + yield from _config_files(child) + elif isinstance(value, list): + for child in value: + yield from _config_files(child) + elif isinstance(value, str) and value.startswith("CONFIG_FILE="): + yield value.removeprefix("CONFIG_FILE=").split(":", 1)[0] + + +def _power_recipes(): + master = yaml.safe_load(MASTER.read_text(encoding="utf-8")) + recipes = {} + for config in master.values(): + if not isinstance(config, dict): + continue + runner = config.get("runner") + if ( + runner not in LAUNCHERS + or config.get("model-prefix") not in {"dsv4", "kimik2.6"} + or not config.get("disagg") + ): + continue + for config_file in _config_files(config.get("scenarios", {})): + relative = config_file.removeprefix("recipes/") + path = RECIPE_ROOT / relative + if path.exists() and "/agentic/" not in path.as_posix(): + recipes[path.relative_to(REPO_ROOT).as_posix()] = { + "runner": runner, + "model": config["model-prefix"], + "framework": config["framework"], + } + return recipes + + +POWER_RECIPES = _power_recipes() + + +def test_exactly_37_supported_recipes_are_selected_from_the_master_config(): + assert len(POWER_RECIPES) == 37 + assert Counter(item["runner"] for item in POWER_RECIPES.values()) == { + "cluster:b200-dgxc": 5, + "b200-nscale": 10, + "b200-new": 6, + "b300": 16, + } + assert Counter(item["framework"] for item in POWER_RECIPES.values()) == { + "dynamo-sglang": 18, + "dynamo-vllm": 19, + } + assert all( + "kimi-k3" not in path and "/agentic/" not in path for path in POWER_RECIPES + ) + dedicated = sum( + yaml.safe_load((REPO_ROOT / path).read_text(encoding="utf-8")) + .get("infra", {}) + .get("etcd_nats_dedicated_node", False) + for path in POWER_RECIPES + ) + assert dedicated == 15 + + +def test_every_selected_recipe_uses_the_required_contract(): + for path, item in POWER_RECIPES.items(): + recipe = yaml.safe_load((REPO_ROOT / path).read_text(encoding="utf-8")) + assert recipe["benchmark"]["type"] == "sa-bench", path + assert recipe["telemetry"] == { + "enabled": True, + "provider": "dcgm-power", + "default_frequency": 1.0, + "storage_subdir": "power", + "required": True, + "startup_timeout_seconds": 120, + "request_timeout_seconds": 2, + "collector_join_timeout_seconds": 10, + "dcgm_exporter": { + "container_image": "dcgm-exporter", + "port": 19401 if item["runner"] == "b300" else 9401, + }, + }, path + + +def test_launchers_use_recipe_gating_and_the_exact_fork_commit(): + for launcher in set(LAUNCHERS.values()): + text = launcher.read_text(encoding="utf-8") + assert re.findall(r"^\s*USES_DCGM_POWER=(\S+)$", text, re.MULTILINE) == [ + "0", + "1", + ], launcher + assert re.search(r'_RECIPE_REL="\$\{(?:_POWER_)?CONFIG_FILE%%:\*\}"', text), ( + launcher + ) + assert "/^telemetry:/ { t = 1; next }" in text + assert "t && /^ provider: dcgm-power$/ { p = 1 }" in text + assert "t && /^ enabled: true$/ { e = 1 }" in text + assert f'POWER_SRT_SLURM_URL="{PRODUCER_URL}"' in text + assert f'POWER_SRT_SLURM_PIN="{PRODUCER_SHA}"' in text + assert 'git clone "$POWER_SRT_SLURM_URL" "$SRT_REPO_DIR"' in text + assert 'git checkout "$POWER_SRT_SLURM_PIN" || exit 1' in text + assert 'test "$(git rev-parse HEAD)" = "$POWER_SRT_SLURM_PIN"' in text + assert 'git rev-parse HEAD > "$GITHUB_WORKSPACE/power-producer-sha.txt"' in text + + +def test_launcher_awk_gate_accepts_all_selected_recipes_and_rejects_non_power(): + program = """ + /^telemetry:/ { t = 1; next } + t && /^[^ ]/ { t = 0 } + t && /^ provider: dcgm-power$/ { p = 1 } + t && /^ enabled: true$/ { e = 1 } + END { exit !(p && e) } + """ + for path in POWER_RECIPES: + result = subprocess.run( + ["awk", program, str(REPO_ROOT / path)], + capture_output=True, + text=True, + check=False, + ) + assert result.returncode == 0, (path, result.stderr) + + non_power = RECIPE_ROOT / "vllm/kimi-k3/agentic/agg-b200-dep8-mtp-agentic.yaml" + result = subprocess.run( + ["awk", program, str(non_power)], + capture_output=True, + text=True, + check=False, + ) + assert result.returncode != 0 + + +def test_launchers_provision_and_preserve_power_provenance(): + for launcher in set(LAUNCHERS.values()): + text = launcher.read_text(encoding="utf-8") + assert f'DCGM_EXPORTER_IMAGE="{EXPORTER_IMAGE}"' in text + assert ( + 'DCGM_EXPORTER_ENROOT_REF="${DCGM_EXPORTER_IMAGE/nvcr.io\\//nvcr.io#}"' + in text + ), launcher + assert ( + 'import_squash "$DCGM_EXPORTER_SQSH" "$DCGM_EXPORTER_ENROOT_REF"' in text + or 'enroot import -o \\"$DCGM_EXPORTER_SQSH\\" \\"docker://$DCGM_EXPORTER_ENROOT_REF\\"' + in text + ), launcher + assert 'test -r "$DCGM_EXPORTER_SQSH"' in text + assert ( + 'sha256sum "$DCGM_EXPORTER_SQSH" > "$GITHUB_WORKSPACE/exporter-image.sha256"' + in text + ) + assert 'dcgm-exporter: ${DCGM_EXPORTER_SQSH}" srtslurm.yaml' in text + assert 'grep -q "^ dcgm-exporter: " srtslurm.yaml ||' in text + assert ( + 'cp "$GITHUB_WORKSPACE/exporter-image.sha256" "$LOGS_DIR/power/exporter-image.sha256"' + in text + ) + assert ( + 'cp "$GITHUB_WORKSPACE/power-producer-sha.txt" "$LOGS_DIR/power/power-producer-sha.txt"' + in text + ) + + +def test_non_power_specialized_clone_revisions_remain_available(): + expected = { + "runners/launch_b200-dgxc.sh": { + "aflowers/vllm-gb200-v0.20.0", + "c180328b98c3793ca84a1e24a030f90545eb7d5d", + "df5baa93f4caf5169dea2a4236ad2cc742fe40e7", + }, + "runners/launch_b200-nscale-slurm.sh": { + "04e87fcc505d6d851451781a5499ca19a02ec2b4", + "aflowers/vllm-gb200-v0.20.0", + "c180328b98c3793ca84a1e24a030f90545eb7d5d", + "217f9438", + }, + "runners/launch_b300-nv.sh": { + "aflowers/vllm-gb200-v0.20.0", + "c180328b98c3793ca84a1e24a030f90545eb7d5d", + "cam/sa-submission-q2-2026", + }, + } + for relative, revisions in expected.items(): + text = (REPO_ROOT / relative).read_text(encoding="utf-8") + assert revisions <= {revision for revision in revisions if revision in text}, ( + relative + ) + + +def test_process_result_ci_covers_the_new_contract(): + workflow = WORKFLOW.read_text(encoding="utf-8") + for path in ( + "configs/nvidia-master.yaml", + "runners/launch_b200-dgxc.sh", + "runners/launch_b200-nscale-slurm.sh", + "runners/launch_b300-nv.sh", + "utils/test_b200_b300_power_official_contract.py", + ): + assert f"- '{path}'" in workflow + assert ( + "test_b200_b300_power_official_contract.py" + in workflow.split("- name: Run pytest", 1)[1] + ) + + +def test_process_result_ci_globs_cover_every_selected_recipe(): + workflow = WORKFLOW.read_text(encoding="utf-8") + covered = set() + + for recipe_glob in PROCESS_RESULT_RECIPE_GLOBS: + assert f"- '{recipe_glob}'" in workflow + result = subprocess.run( + ["git", "ls-files", f":(glob){recipe_glob}"], + cwd=REPO_ROOT, + capture_output=True, + text=True, + check=True, + ) + covered.update(result.stdout.splitlines()) + + assert set(POWER_RECIPES) <= covered