diff --git a/benchmarks/results/pr325_rocm_mi300x/cuda_cpu_comparison.json b/benchmarks/results/pr325_rocm_mi300x/cuda_cpu_comparison.json index 44baf8b3..77466099 100644 --- a/benchmarks/results/pr325_rocm_mi300x/cuda_cpu_comparison.json +++ b/benchmarks/results/pr325_rocm_mi300x/cuda_cpu_comparison.json @@ -4,7 +4,32 @@ "pull_request": "https://github.com/RL-Align/RL-Kernel/pull/321", "cuda_commit": "8576fa4bf449734ae99e9b50be8756bb282a8916", "h100_triton_replay_commit": "e64abab904880b877d26d04c0cfad020b992aa51", - "note": "User-supplied measurements; cross-hardware values are context only and are not hardware-normalized." + "note": "User-supplied measurements; cross-hardware values are context only and are not hardware-normalized.", + "official_h100_update": { + "source": "User-supplied H100 table on 2026-09-06", + "topologies": ["tp2", "tp2_sp", "tp4", "tp2_cp2", "tp2_cp2_sp", "tp8", "tp4_cp2", "tp4_cp2_sp"], + "directions": ["forward", "train_fwd_bwd"], + "fields": ["official_h100_ms", "official_h100_summary_ms"], + "git_commit": "98dcd38fb635e1e0eab9035aa4c4b40483cc36b8", + "gpu": "NVIDIA H100 80GB", + "torch": "2.13.0+cu130", + "cuda": "13.0", + "units": "ms", + "shape": {"tokens": 32, "hidden": 4096, "intermediate": 12288}, + "dtype": "bfloat16", + "warmup": 5, + "samples": 20, + "training_samples": 10, + "timing": "synchronized wall clock; slowest rank per sample", + "baseline": "F.linear + NCCL at the same TP/CP/SP topology; TP=1 is used only for untimed RL-Kernel bitwise checks", + "validation": {"reference": "RL-Kernel CUDA TP=1", "output_mismatch": 0, "hidden_gradient_mismatch": 0, "weight_gradient_mismatch": 0, "repeat_mismatch": 0, "train_infer_mismatch": 0}, + "environment_variables": {"NCCL_NVLS_ENABLE": "0", "NCCL_IB_DISABLE": "1", "RL_KERNEL_DET_GEMM_BACKEND": "sm90"}, + "commands": [ + "env PYTHONPATH=/tmp/rlk-head-98dcd38-build-GYkK6iE5/repo CUDA_VISIBLE_DEVICES=0,1 NCCL_NVLS_ENABLE=0 NCCL_IB_DISABLE=1 RL_KERNEL_DET_GEMM_BACKEND=sm90 FFN_WORLD_SIZE=2 FFN_WARMUP=5 FFN_SAMPLES=20 FFN_TRAINING_SAMPLES=10 /home/ellm/miniconda3/envs/hsr_env_rl_kernel/bin/python /tmp/rlk-head-98dcd38-build-GYkK6iE5/repo/run_h100_distributed_ffn_data.py", + "env PYTHONPATH=/tmp/rlk-head-98dcd38-build-GYkK6iE5/repo CUDA_VISIBLE_DEVICES=0,1,2,3 NCCL_NVLS_ENABLE=0 NCCL_IB_DISABLE=1 RL_KERNEL_DET_GEMM_BACKEND=sm90 FFN_WORLD_SIZE=4 FFN_WARMUP=5 FFN_SAMPLES=20 FFN_TRAINING_SAMPLES=10 /home/ellm/miniconda3/envs/hsr_env_rl_kernel/bin/python /tmp/rlk-head-98dcd38-build-GYkK6iE5/repo/run_h100_distributed_ffn_data.py" + ], + "note": "All 16 official distributed rows are updated from user-supplied H100 measurements. CUDA and other historical rows remain unchanged; top-level environment and methodology describe those historical data. All H100 world sizes used 5/20/10 warmup/forward/training settings. ROCm uses 10/50/20 CLI settings (its distributed training halves warmup to 5); shapes, topology and timing aggregation match, but sample counts differ. The user discarded H100 10/50/20 reruns contaminated by competing GPU 0-3 workloads. The supplied H100 runner and individual samples are not available in this repository." + } }, "environment": { "gpu": "NVIDIA H100 80GB HBM3", @@ -39,22 +64,22 @@ {"tokens": 32, "direction": "train_fwd_bwd", "official_h100_ms": 0.7842, "cuda_h100_ms": 9.1635, "triton_h100_ms": 5.4261, "official_cpu_ms": 65.9689} ], "distributed": [ - {"name": "tp2", "direction": "forward", "official_h100_ms": 0.1552, "cuda_h100_ms": 2.7896}, - {"name": "tp2", "direction": "train_fwd_bwd", "official_h100_ms": 0.6301, "cuda_h100_ms": 7.8125}, - {"name": "tp2_sp", "direction": "forward", "official_h100_ms": 0.1552, "cuda_h100_ms": 3.3310}, - {"name": "tp2_sp", "direction": "train_fwd_bwd", "official_h100_ms": 0.6301, "cuda_h100_ms": 11.7695}, - {"name": "tp4", "direction": "forward", "official_h100_ms": 0.1530, "cuda_h100_ms": 2.1687}, - {"name": "tp4", "direction": "train_fwd_bwd", "official_h100_ms": 0.5714, "cuda_h100_ms": 9.3954}, - {"name": "tp2_cp2", "direction": "forward", "official_h100_ms": 0.1530, "cuda_h100_ms": 2.8261}, - {"name": "tp2_cp2", "direction": "train_fwd_bwd", "official_h100_ms": 0.5714, "cuda_h100_ms": 16.1722}, - {"name": "tp2_cp2_sp", "direction": "forward", "official_h100_ms": 0.1530, "cuda_h100_ms": 3.5814}, - {"name": "tp2_cp2_sp", "direction": "train_fwd_bwd", "official_h100_ms": 0.5714, "cuda_h100_ms": 16.7894}, - {"name": "tp8", "direction": "forward", "official_h100_ms": 0.1537, "cuda_h100_ms": 2.3206}, - {"name": "tp8", "direction": "train_fwd_bwd", "official_h100_ms": 0.5006, "cuda_h100_ms": 10.1375}, - {"name": "tp4_cp2", "direction": "forward", "official_h100_ms": 0.1537, "cuda_h100_ms": 2.2141}, - {"name": "tp4_cp2", "direction": "train_fwd_bwd", "official_h100_ms": 0.5006, "cuda_h100_ms": 16.5617}, - {"name": "tp4_cp2_sp", "direction": "forward", "official_h100_ms": 0.1537, "cuda_h100_ms": 3.2972}, - {"name": "tp4_cp2_sp", "direction": "train_fwd_bwd", "official_h100_ms": 0.5006, "cuda_h100_ms": 17.1457} + {"name": "tp2", "direction": "forward", "official_h100_ms": 0.1653, "cuda_h100_ms": 2.7896, "official_h100_summary_ms": {"median_ms": 0.1653, "p95_ms": 0.2055, "min_ms": 0.1420, "max_ms": 0.3013}}, + {"name": "tp2", "direction": "train_fwd_bwd", "official_h100_ms": 0.6347, "cuda_h100_ms": 7.8125, "official_h100_summary_ms": {"median_ms": 0.6347, "p95_ms": 0.6955, "min_ms": 0.5826, "max_ms": 0.6997}}, + {"name": "tp2_sp", "direction": "forward", "official_h100_ms": 0.1974, "cuda_h100_ms": 3.3310, "official_h100_summary_ms": {"median_ms": 0.1974, "p95_ms": 0.2161, "min_ms": 0.1807, "max_ms": 0.2273}}, + {"name": "tp2_sp", "direction": "train_fwd_bwd", "official_h100_ms": 0.7162, "cuda_h100_ms": 11.7695, "official_h100_summary_ms": {"median_ms": 0.7162, "p95_ms": 1.0543, "min_ms": 0.6713, "max_ms": 1.2747}}, + {"name": "tp4", "direction": "forward", "official_h100_ms": 0.1742, "cuda_h100_ms": 2.1687, "official_h100_summary_ms": {"median_ms": 0.1742, "p95_ms": 0.8594, "min_ms": 0.1545, "max_ms": 1.4339}}, + {"name": "tp4", "direction": "train_fwd_bwd", "official_h100_ms": 0.6460, "cuda_h100_ms": 9.3954, "official_h100_summary_ms": {"median_ms": 0.6460, "p95_ms": 0.7029, "min_ms": 0.5959, "max_ms": 0.7232}}, + {"name": "tp2_cp2", "direction": "forward", "official_h100_ms": 0.1600, "cuda_h100_ms": 2.8261, "official_h100_summary_ms": {"median_ms": 0.1600, "p95_ms": 0.8807, "min_ms": 0.1411, "max_ms": 1.4457}}, + {"name": "tp2_cp2", "direction": "train_fwd_bwd", "official_h100_ms": 1.2175, "cuda_h100_ms": 16.1722, "official_h100_summary_ms": {"median_ms": 1.2175, "p95_ms": 2.2282, "min_ms": 1.1868, "max_ms": 2.2589}}, + {"name": "tp2_cp2_sp", "direction": "forward", "official_h100_ms": 0.2425, "cuda_h100_ms": 3.5814, "official_h100_summary_ms": {"median_ms": 0.2425, "p95_ms": 0.2821, "min_ms": 0.2271, "max_ms": 0.3538}}, + {"name": "tp2_cp2_sp", "direction": "train_fwd_bwd", "official_h100_ms": 1.5878, "cuda_h100_ms": 16.7894, "official_h100_summary_ms": {"median_ms": 1.5878, "p95_ms": 1.6838, "min_ms": 1.4295, "max_ms": 1.7189}}, + {"name": "tp8", "direction": "forward", "official_h100_ms": 0.3962, "cuda_h100_ms": 2.3206, "official_h100_summary_ms": {"median_ms": 0.3962, "p95_ms": 1.1780, "min_ms": 0.3521, "max_ms": 1.2054}}, + {"name": "tp8", "direction": "train_fwd_bwd", "official_h100_ms": 1.2062, "cuda_h100_ms": 10.1375, "official_h100_summary_ms": {"median_ms": 1.2062, "p95_ms": 1.4400, "min_ms": 1.1445, "max_ms": 1.4588}}, + {"name": "tp4_cp2", "direction": "forward", "official_h100_ms": 0.3806, "cuda_h100_ms": 2.2141, "official_h100_summary_ms": {"median_ms": 0.3806, "p95_ms": 1.4873, "min_ms": 0.3540, "max_ms": 2.5012}}, + {"name": "tp4_cp2", "direction": "train_fwd_bwd", "official_h100_ms": 1.5640, "cuda_h100_ms": 16.5617, "official_h100_summary_ms": {"median_ms": 1.5640, "p95_ms": 2.7782, "min_ms": 1.4680, "max_ms": 2.9183}}, + {"name": "tp4_cp2_sp", "direction": "forward", "official_h100_ms": 0.5009, "cuda_h100_ms": 3.2972, "official_h100_summary_ms": {"median_ms": 0.5009, "p95_ms": 0.5523, "min_ms": 0.4745, "max_ms": 0.5670}}, + {"name": "tp4_cp2_sp", "direction": "train_fwd_bwd", "official_h100_ms": 1.7319, "cuda_h100_ms": 17.1457, "official_h100_summary_ms": {"median_ms": 1.7319, "p95_ms": 1.9220, "min_ms": 1.7104, "max_ms": 1.9283}} ], "dtype_accuracy": { "tokens": 8, diff --git a/benchmarks/results/pr325_rocm_mi300x/distributed_ffn_overhead.png b/benchmarks/results/pr325_rocm_mi300x/distributed_ffn_overhead.png index 28584265..38800f54 100644 Binary files a/benchmarks/results/pr325_rocm_mi300x/distributed_ffn_overhead.png and b/benchmarks/results/pr325_rocm_mi300x/distributed_ffn_overhead.png differ diff --git a/benchmarks/results/pr325_rocm_mi300x/report.md b/benchmarks/results/pr325_rocm_mi300x/report.md index 2c371bb4..9adde7d1 100644 --- a/benchmarks/results/pr325_rocm_mi300x/report.md +++ b/benchmarks/results/pr325_rocm_mi300x/report.md @@ -10,6 +10,11 @@ This is an operator-only MI300X report. It does not load or benchmark a model ch ## Environment +This table describes the MI300X distributed rerun. The preserved single-GPU +and dtype observations come from `caef501101a3906c733076f31f3b5a9870169d16` +with Transformers 5.10.4; their values and figures are unchanged. Per-series +provenance is recorded in `results.json:measurement_sources`. + | Field | Value | |---|---| | NCCL_IB_DISABLE | 1 | @@ -17,14 +22,14 @@ This is an operator-only MI300X report. It does not load or benchmark a model ch | deterministic_compute | ROCm-native Triton | | deterministic_transport | fixed-tree HIP IPC with RCCL fallback on ROCm | | distributed_speed_comparison | four same-topology H100/MI300X paths | -| git_commit | caef501101a3906c733076f31f3b5a9870169d16 | +| git_commit | 98dcd38fb635e1e0eab9035aa4c4b40483cc36b8 | | gpu | AMD Instinct MI300X | | gpu_count | 8 | | hip | 7.14.60850 | | python | 3.12.3 | | single_gpu_speed_context | Hugging Face Transformers Qwen3MLP, TP=1 | | torch | 2.12.0+rocm7.14.0a20260608 | -| transformers | 5.10.4 | +| transformers | 5.13.1 | ## Methodology @@ -39,17 +44,42 @@ This is an operator-only MI300X report. It does not load or benchmark a model ch - Implementation note: the current ROCm deterministic communication operator is adopted from PR #357. This changes the implementation under test, not the benchmark comparison contract. - Single-GPU timing: GPU events, median and p95; distributed timing: synchronized wall clock, slowest rank/sample. - Distributed workers: one NUMA-local CPU per GPU rank to reduce host-scheduler noise in synchronized wall-clock samples. -- 10 warmups, 50 measured forward samples, and 20 measured forward+backward samples. +- MI300X distributed: 10 forward warmups and 50 forward samples; the script halves the training warmup to 5, followed by 20 forward+backward samples. +- H100 official distributed: user-supplied measurements at `98dcd38fb635e1e0eab9035aa4c4b40483cc36b8`, NVIDIA H100 80GB, PyTorch 2.13.0+cu130 / CUDA 13.0. All world sizes use 5 warmups, 20 forward samples, and 10 forward+backward samples. Shape, BF16 dtype, topology, and synchronized slowest-rank wall-clock timing match the ROCm run. Sample counts differ; p95, particularly with 10 training samples, should be interpreted cautiously. +- H100 official uses same-topology `F.linear` + NCCL. The submitted settings include `NCCL_NVLS_ENABLE=0`, `NCCL_IB_DISABLE=1`, and `RL_KERNEL_DET_GEMM_BACKEND=sm90`. Commands are retained in `cuda_cpu_comparison.json:source.official_h100_update`. The user discarded H100 10/50/20 reruns affected by competing workloads on GPUs 0–3. +- H100 CUDA values remain the historical measurements at `8576fa4bf449734ae99e9b50be8756bb282a8916`, as requested. The four plotted series do not all come from one commit or sampling configuration. Ratios below are recomputed from the displayed columns; they are not the ratios in the newly supplied CUDA tables and do not isolate the cost of determinism in a controlled same-commit experiment. +- The MI300X server was shared: another process held about 180 GiB on GPU 7. All layouts completed successfully, but the results are not exclusive-node measurements. - `NCCL_IB_DISABLE=1` keeps the distributed run on intra-node XGMI. Median, p95, min, and max values are available in `results.json`. -Reproduce from the repository root: +Reproduce the ROCm measurement from commit `98dcd38` into a separate output directory: ```bash -python benchmarks/benchmark_rocm_ffn.py \ +NCCL_IB_DISABLE=1 OMP_NUM_THREADS=1 python benchmarks/benchmark_rocm_ffn.py \ --warmup 10 \ --samples 50 \ --training-samples 20 \ - --output-dir benchmarks/results/pr325_rocm_mi300x + --output-dir benchmarks/results/pr325_rocm_mi300x_reproduction +``` + +This combined artifact preserves historical single-GPU and H100 CUDA data; +a new benchmark run alone does not recreate those historical sections. +Regenerate the figures directly from the synchronized JSON, without rerunning +measurements: + +```bash +python - <<'PY' +import json +import runpy +from pathlib import Path + +directory = Path("benchmarks/results/pr325_rocm_mi300x") +benchmark = runpy.run_path("benchmarks/benchmark_rocm_ffn.py") +benchmark["_write_figures"]( + json.loads((directory / "results.json").read_text()), + directory, + json.loads((directory / "cuda_cpu_comparison.json").read_text()), +) +PY ``` ## Results summary @@ -57,8 +87,8 @@ python benchmarks/benchmark_rocm_ffn.py \ - TP=1 exactness baseline: **0 mismatched elements** across topology forward outputs, training outputs, dHidden, and dWeights. - Repeat mismatch: **0**; training/inference forward mismatch: **0**. - Single-GPU deterministic Triton packed-cache latency is **3.92-7.64x** the official Qwen3MLP TP=1 latency across M=1/8/32 and forward/training. -- MI300X deterministic Triton latency is **0.14-0.38x** the H100 deterministic CUDA latency for the same distributed layouts. Both official distributed paths are reported alongside them. -- Versus the previous deterministic MI300X benchmark, PR #357 improves **16/16** rows, with a mean latency reduction of **22.1%**. +- MI300X deterministic Triton latency is **0.12-0.32x** the H100 deterministic CUDA latency for the same distributed layouts. Both official distributed paths are reported alongside them. +- Versus the historical MI300X timings embedded in the script, this rerun is faster in **16/16** rows, with a mean latency reduction of **36.1%**. This is historical context, not a controlled attribution to PR #357. - The separate official-Qwen3MLP FP16 versus FP32 observation has relative-L2 error **6.544e-04** for (M,H,I)=(8,4096,12288). ## Single-GPU FFN speed @@ -80,45 +110,73 @@ Every row compares the same distributed topology and direction. No TP=1 latency | Parallel layout | Direction | H100 official distributed (ms) | H100 deterministic CUDA (ms) | MI300X official distributed (ms) | MI300X deterministic Triton (ms) | H100 det / official | MI300X det / official | |---|---|---:|---:|---:|---:|---:|---:| -| tp2 | forward | 0.1552 | 2.7896 | 0.2323 | 0.8759 | 17.97x | 3.77x | -| tp2 | train_fwd_bwd | 0.6301 | 7.8125 | 0.6984 | 2.0802 | 12.40x | 2.98x | -| tp2_sp | forward | 0.1552 | 3.3310 | 0.3066 | 0.8885 | 21.46x | 2.90x | -| tp2_sp | train_fwd_bwd | 0.6301 | 11.7695 | 1.4217 | 2.5187 | 18.68x | 1.77x | -| tp4 | forward | 0.1530 | 2.1687 | 0.2380 | 0.8324 | 14.17x | 3.50x | -| tp4 | train_fwd_bwd | 0.5714 | 9.3954 | 0.8280 | 2.2969 | 16.44x | 2.77x | -| tp2_cp2 | forward | 0.1530 | 2.8261 | 0.2317 | 0.7562 | 18.47x | 3.26x | -| tp2_cp2 | train_fwd_bwd | 0.5714 | 16.1722 | 4.4830 | 2.5281 | 28.30x | 0.56x | -| tp2_cp2_sp | forward | 0.1530 | 3.5814 | 0.3262 | 0.8067 | 23.41x | 2.47x | -| tp2_cp2_sp | train_fwd_bwd | 0.5714 | 16.7894 | 4.5870 | 2.5462 | 29.38x | 0.56x | -| tp8 | forward | 0.1537 | 2.3206 | 0.2075 | 0.7065 | 15.10x | 3.41x | -| tp8 | train_fwd_bwd | 0.5006 | 10.1375 | 0.8685 | 2.0507 | 20.25x | 2.36x | -| tp4_cp2 | forward | 0.1537 | 2.2141 | 0.2424 | 0.8306 | 14.41x | 3.43x | -| tp4_cp2 | train_fwd_bwd | 0.5006 | 16.5617 | 3.0071 | 2.4620 | 33.08x | 0.82x | -| tp4_cp2_sp | forward | 0.1537 | 3.2972 | 0.3153 | 0.7618 | 21.45x | 2.42x | -| tp4_cp2_sp | train_fwd_bwd | 0.5006 | 17.1457 | 2.9346 | 2.4232 | 34.25x | 0.83x | - -### PR #357 latency change versus the previous benchmark - -This comparison changes only the deterministic ROCm communication implementation. It is not included as another series in the main four-path figure. +| tp2 | forward | 0.1653 | 2.7896 | 0.2164 | 0.6669 | 16.88x | 3.08x | +| tp2 | train_fwd_bwd | 0.6347 | 7.8125 | 0.6532 | 1.6757 | 12.31x | 2.57x | +| tp2_sp | forward | 0.1974 | 3.3310 | 0.2825 | 0.7038 | 16.87x | 2.49x | +| tp2_sp | train_fwd_bwd | 0.7162 | 11.7695 | 0.8634 | 1.7942 | 16.43x | 2.08x | +| tp4 | forward | 0.1742 | 2.1687 | 0.2240 | 0.6677 | 12.45x | 2.98x | +| tp4 | train_fwd_bwd | 0.6460 | 9.3954 | 0.6778 | 1.6612 | 14.54x | 2.45x | +| tp2_cp2 | forward | 0.1600 | 2.8261 | 0.2261 | 0.6825 | 17.66x | 3.02x | +| tp2_cp2 | train_fwd_bwd | 1.2175 | 16.1722 | 4.0314 | 1.9632 | 13.28x | 0.49x | +| tp2_cp2_sp | forward | 0.2425 | 3.5814 | 0.2989 | 0.7069 | 14.77x | 2.36x | +| tp2_cp2_sp | train_fwd_bwd | 1.5878 | 16.7894 | 4.2115 | 1.9622 | 10.57x | 0.47x | +| tp8 | forward | 0.3962 | 2.3206 | 0.2006 | 0.6680 | 5.86x | 3.33x | +| tp8 | train_fwd_bwd | 1.2062 | 10.1375 | 0.6640 | 1.7631 | 8.40x | 2.66x | +| tp4_cp2 | forward | 0.3806 | 2.2141 | 0.2377 | 0.6979 | 5.82x | 2.94x | +| tp4_cp2 | train_fwd_bwd | 1.5640 | 16.5617 | 2.4378 | 2.0162 | 10.59x | 0.83x | +| tp4_cp2_sp | forward | 0.5009 | 3.2972 | 0.3100 | 0.7566 | 6.58x | 2.44x | +| tp4_cp2_sp | train_fwd_bwd | 1.7319 | 17.1457 | 2.8061 | 1.9804 | 9.90x | 0.71x | + +### H100 official timing distributions + +User-supplied values in milliseconds, retained at their supplied precision. The chart uses the median column. + +| Parallel layout | Direction | Median | p95 | Min | Max | +|---|---|---:|---:|---:|---:| +| tp2 | forward | 0.1653 | 0.2055 | 0.1420 | 0.3013 | +| tp2 | train_fwd_bwd | 0.6347 | 0.6955 | 0.5826 | 0.6997 | +| tp2_sp | forward | 0.1974 | 0.2161 | 0.1807 | 0.2273 | +| tp2_sp | train_fwd_bwd | 0.7162 | 1.0543 | 0.6713 | 1.2747 | +| tp4 | forward | 0.1742 | 0.8594 | 0.1545 | 1.4339 | +| tp4 | train_fwd_bwd | 0.6460 | 0.7029 | 0.5959 | 0.7232 | +| tp2_cp2 | forward | 0.1600 | 0.8807 | 0.1411 | 1.4457 | +| tp2_cp2 | train_fwd_bwd | 1.2175 | 2.2282 | 1.1868 | 2.2589 | +| tp2_cp2_sp | forward | 0.2425 | 0.2821 | 0.2271 | 0.3538 | +| tp2_cp2_sp | train_fwd_bwd | 1.5878 | 1.6838 | 1.4295 | 1.7189 | +| tp8 | forward | 0.3962 | 1.1780 | 0.3521 | 1.2054 | +| tp8 | train_fwd_bwd | 1.2062 | 1.4400 | 1.1445 | 1.4588 | +| tp4_cp2 | forward | 0.3806 | 1.4873 | 0.3540 | 2.5012 | +| tp4_cp2 | train_fwd_bwd | 1.5640 | 2.7782 | 1.4680 | 2.9183 | +| tp4_cp2_sp | forward | 0.5009 | 0.5523 | 0.4745 | 0.5670 | +| tp4_cp2_sp | train_fwd_bwd | 1.7319 | 1.9220 | 1.7104 | 1.9283 | + +The same statistics are embedded in `results.json:distributed_platform_comparison.rows[].h100_official_distributed_summary_ms`. MI300X median/p95/min/max are in `results.json:distributed_ffn[]` under `official_distributed` and `triton`. + +### Historical MI300X latency context + +The previous values are constants from an older benchmark; they were not rerun +in this session. Implementation and run conditions differ, so these changes +cannot be attributed solely to PR #357. They are not included as another series +in the main figure. | Parallel layout | Direction | Previous (ms) | Current (ms) | Latency reduction | |---|---|---:|---:|---:| -| tp2 | forward | 0.9040 | 0.8759 | 3.1% | -| tp2 | train_fwd_bwd | 2.6997 | 2.0802 | 22.9% | -| tp2_sp | forward | 1.0561 | 0.8885 | 15.9% | -| tp2_sp | train_fwd_bwd | 2.9646 | 2.5187 | 15.0% | -| tp4 | forward | 0.8359 | 0.8324 | 0.4% | -| tp4 | train_fwd_bwd | 2.6999 | 2.2969 | 14.9% | -| tp2_cp2 | forward | 0.9016 | 0.7562 | 16.1% | -| tp2_cp2 | train_fwd_bwd | 3.5606 | 2.5281 | 29.0% | -| tp2_cp2_sp | forward | 1.1296 | 0.8067 | 28.6% | -| tp2_cp2_sp | train_fwd_bwd | 3.9929 | 2.5462 | 36.2% | -| tp8 | forward | 1.0860 | 0.7065 | 34.9% | -| tp8 | train_fwd_bwd | 2.5620 | 2.0507 | 20.0% | -| tp4_cp2 | forward | 1.0536 | 0.8306 | 21.2% | -| tp4_cp2 | train_fwd_bwd | 3.2599 | 2.4620 | 24.5% | -| tp4_cp2_sp | forward | 1.1928 | 0.7618 | 36.1% | -| tp4_cp2_sp | train_fwd_bwd | 3.7498 | 2.4232 | 35.4% | +| tp2 | forward | 0.9040 | 0.6669 | 26.2% | +| tp2 | train_fwd_bwd | 2.6997 | 1.6757 | 37.9% | +| tp2_sp | forward | 1.0561 | 0.7038 | 33.4% | +| tp2_sp | train_fwd_bwd | 2.9646 | 1.7942 | 39.5% | +| tp4 | forward | 0.8359 | 0.6677 | 20.1% | +| tp4 | train_fwd_bwd | 2.6999 | 1.6612 | 38.5% | +| tp2_cp2 | forward | 0.9016 | 0.6825 | 24.3% | +| tp2_cp2 | train_fwd_bwd | 3.5606 | 1.9632 | 44.9% | +| tp2_cp2_sp | forward | 1.1296 | 0.7069 | 37.4% | +| tp2_cp2_sp | train_fwd_bwd | 3.9929 | 1.9622 | 50.9% | +| tp8 | forward | 1.0860 | 0.6680 | 38.5% | +| tp8 | train_fwd_bwd | 2.5620 | 1.7631 | 31.2% | +| tp4_cp2 | forward | 1.0536 | 0.6979 | 33.8% | +| tp4_cp2 | train_fwd_bwd | 3.2599 | 2.0162 | 38.2% | +| tp4_cp2_sp | forward | 1.1928 | 0.7566 | 36.6% | +| tp4_cp2_sp | train_fwd_bwd | 3.7498 | 1.9804 | 47.2% | ## CUDA GPU and CPU performance context @@ -144,7 +202,10 @@ The same-H100 CUDA/Triton ratio is the hardware-matched comparison. CPU and MI30 | M=32, forward | 8.5392 | 0.1311 | 2.2529 | 4.0277 | 1.79x | 0.1146 | 0.8761 | | M=32, forward+backward | 65.9689 | 0.7842 | 5.4261 | 9.1635 | 1.69x | 0.4182 | 2.3188 | -Both H100 columns used in the main distributed table are the user-supplied distributed timings. They are joined directly with MI300X rows of the same topology and direction; TP=1 values are excluded from all four columns. +The main distributed table combines the updated user-supplied H100 official +measurements with preserved historical H100 CUDA measurements, joined to the +MI300X rerun by topology and direction. TP=1 latency is excluded from all four +columns. The single-GPU/CPU table above retains its original measurements. ## Topology exactness versus Triton TP=1 diff --git a/benchmarks/results/pr325_rocm_mi300x/results.json b/benchmarks/results/pr325_rocm_mi300x/results.json index 575e685c..4af88d65 100644 --- a/benchmarks/results/pr325_rocm_mi300x/results.json +++ b/benchmarks/results/pr325_rocm_mi300x/results.json @@ -18,13 +18,13 @@ "direction": "forward", "hidden": 4096, "intermediate": 12288, - "latency_ratio_triton_vs_official_distributed": 3.770714554916725, + "latency_ratio_triton_vs_official_distributed": 3.0819895795813808, "name": "tp2", "official_distributed": { - "max_ms": 1.3578161597251892, - "median_ms": 0.23228488862514496, - "min_ms": 0.2129673957824707, - "p95_ms": 0.8294882718473673 + "max_ms": 0.4664650186896324, + "median_ms": 0.2163875033147633, + "min_ms": 0.20921201212331653, + "p95_ms": 0.3178337909048423 }, "repeat_mismatch_count": 0, "sequence_parallel": false, @@ -35,10 +35,10 @@ "tp_size": 2, "train_infer_mismatch_count": 0, "triton": { - "max_ms": 1.9319094717502594, - "median_ms": 0.8758800104260445, - "min_ms": 0.8196169510483742, - "p95_ms": 1.4833177905529737 + "max_ms": 0.8963949512690306, + "median_ms": 0.666904030367732, + "min_ms": 0.651060021482408, + "p95_ms": 0.8558208704926074 }, "weight_layout": "packed_forward_cache", "world_size": 2 @@ -48,13 +48,13 @@ "direction": "train_fwd_bwd", "hidden": 4096, "intermediate": 12288, - "latency_ratio_triton_vs_official_distributed": 2.978428538282444, + "latency_ratio_triton_vs_official_distributed": 2.565423034622706, "name": "tp2", "official_distributed": { - "max_ms": 1.2473920360207558, - "median_ms": 0.6984230130910873, - "min_ms": 0.6383182480931282, - "p95_ms": 1.2350523378700018 + "max_ms": 0.911829003598541, + "median_ms": 0.6532029947265983, + "min_ms": 0.6297890213318169, + "p95_ms": 0.880479981424287 }, "repeat_mismatch_count": 0, "sequence_parallel": false, @@ -67,10 +67,10 @@ "tp_size": 2, "train_infer_mismatch_count": 0, "triton": { - "max_ms": 3.5044336691498756, - "median_ms": 2.0802030339837074, - "min_ms": 1.9138418138027191, - "p95_ms": 3.1109470874071126 + "max_ms": 1.919640984851867, + "median_ms": 1.6757420089561492, + "min_ms": 1.653273997362703, + "p95_ms": 1.91476940526627 }, "weight_layout": "packed_forward_cache", "world_size": 2 @@ -80,13 +80,13 @@ "direction": "forward", "hidden": 4096, "intermediate": 12288, - "latency_ratio_triton_vs_official_distributed": 2.8983204548458, + "latency_ratio_triton_vs_official_distributed": 2.491084021205794, "name": "tp2_sp", "official_distributed": { - "max_ms": 2.685968764126301, - "median_ms": 0.3065601922571659, - "min_ms": 0.28007570654153824, - "p95_ms": 0.8993070106953382 + "max_ms": 0.6120420293882489, + "median_ms": 0.28252098127268255, + "min_ms": 0.27053296798840165, + "p95_ms": 0.45364995894487914 }, "repeat_mismatch_count": 0, "sequence_parallel": true, @@ -97,10 +97,10 @@ "tp_size": 2, "train_infer_mismatch_count": 0, "triton": { - "max_ms": 2.250421792268753, - "median_ms": 0.888509675860405, - "min_ms": 0.8365819230675697, - "p95_ms": 2.004028530791401 + "max_ms": 1.0910549899563193, + "median_ms": 0.7037835021037608, + "min_ms": 0.6829579942859709, + "p95_ms": 0.9322065598098561 }, "weight_layout": "packed_forward_cache", "world_size": 2 @@ -110,13 +110,13 @@ "direction": "train_fwd_bwd", "hidden": 4096, "intermediate": 12288, - "latency_ratio_triton_vs_official_distributed": 1.771568250466244, + "latency_ratio_triton_vs_official_distributed": 2.078108717055042, "name": "tp2_sp", "official_distributed": { - "max_ms": 2.6939501985907555, - "median_ms": 1.4217207208275795, - "min_ms": 0.8287103846669197, - "p95_ms": 2.582016121596098 + "max_ms": 1.0751819936558604, + "median_ms": 0.8633609977550805, + "min_ms": 0.8363159722648561, + "p95_ms": 1.061795518035069 }, "repeat_mismatch_count": 0, "sequence_parallel": true, @@ -129,10 +129,10 @@ "tp_size": 2, "train_infer_mismatch_count": 0, "triton": { - "max_ms": 5.430372431874275, - "median_ms": 2.5186752900481224, - "min_ms": 1.9955337047576904, - "p95_ms": 3.6994490772485746 + "max_ms": 2.282280009239912, + "median_ms": 1.7941580154001713, + "min_ms": 1.7498169909231365, + "p95_ms": 2.036252798279748 }, "weight_layout": "packed_forward_cache", "world_size": 2 @@ -142,13 +142,13 @@ "direction": "forward", "hidden": 4096, "intermediate": 12288, - "latency_ratio_triton_vs_official_distributed": 3.4979071801625334, + "latency_ratio_triton_vs_official_distributed": 2.980595497031791, "name": "tp4", "official_distributed": { - "max_ms": 0.8178036659955978, - "median_ms": 0.23796828463673592, - "min_ms": 0.21947640925645828, - "p95_ms": 0.7959454320371151 + "max_ms": 0.47576898941770196, + "median_ms": 0.2240189933218062, + "min_ms": 0.21787500008940697, + "p95_ms": 0.4700541991041973 }, "repeat_mismatch_count": 0, "sequence_parallel": false, @@ -159,10 +159,10 @@ "tp_size": 4, "train_infer_mismatch_count": 0, "triton": { - "max_ms": 2.6544714346528053, - "median_ms": 0.8323909714818001, - "min_ms": 0.6374167278409004, - "p95_ms": 1.891128625720739 + "max_ms": 0.9099859744310379, + "median_ms": 0.6677100027445704, + "min_ms": 0.6433079834096134, + "p95_ms": 0.8967982023023069 }, "weight_layout": "packed_forward_cache", "world_size": 4 @@ -172,13 +172,13 @@ "direction": "train_fwd_bwd", "hidden": 4096, "intermediate": 12288, - "latency_ratio_triton_vs_official_distributed": 2.7738258924780523, + "latency_ratio_triton_vs_official_distributed": 2.4507841825531296, "name": "tp4", "official_distributed": { - "max_ms": 2.372283488512039, - "median_ms": 0.82804961130023, - "min_ms": 0.6606811657547951, - "p95_ms": 1.4657761901617057 + "max_ms": 0.9008519700728357, + "median_ms": 0.6778055103495717, + "min_ms": 0.6429579807445407, + "p95_ms": 0.8946675225161016 }, "repeat_mismatch_count": 0, "sequence_parallel": false, @@ -191,10 +191,10 @@ "tp_size": 4, "train_infer_mismatch_count": 0, "triton": { - "max_ms": 3.301742486655712, - "median_ms": 2.296865452080965, - "min_ms": 1.6987714916467667, - "p95_ms": 2.88137081079185 + "max_ms": 2.011897973716259, + "median_ms": 1.661155023612082, + "min_ms": 1.6400930471718311, + "p95_ms": 1.8894306500442326 }, "weight_layout": "packed_forward_cache", "world_size": 4 @@ -204,13 +204,13 @@ "direction": "forward", "hidden": 4096, "intermediate": 12288, - "latency_ratio_triton_vs_official_distributed": 3.264261952381431, + "latency_ratio_triton_vs_official_distributed": 3.0189364393278644, "name": "tp2_cp2", "official_distributed": { - "max_ms": 0.8327467367053032, - "median_ms": 0.23166416212916374, - "min_ms": 0.21467916667461395, - "p95_ms": 0.652501266449689 + "max_ms": 0.47343497863039374, + "median_ms": 0.22607200662605464, + "min_ms": 0.21609198302030563, + "p95_ms": 0.435622059740126 }, "repeat_mismatch_count": 0, "sequence_parallel": false, @@ -221,10 +221,10 @@ "tp_size": 2, "train_infer_mismatch_count": 0, "triton": { - "max_ms": 2.562844194471836, - "median_ms": 0.7562125101685524, - "min_ms": 0.6980272009968758, - "p95_ms": 2.0815798547118893 + "max_ms": 0.9265610133297741, + "median_ms": 0.6824970187153667, + "min_ms": 0.6685259868390858, + "p95_ms": 0.918726235977374 }, "weight_layout": "packed_forward_cache", "world_size": 4 @@ -234,13 +234,13 @@ "direction": "train_fwd_bwd", "hidden": 4096, "intermediate": 12288, - "latency_ratio_triton_vs_official_distributed": 0.5639282606270454, + "latency_ratio_triton_vs_official_distributed": 0.486978322994483, "name": "tp2_cp2", "official_distributed": { - "max_ms": 5.045952275395393, - "median_ms": 4.483005963265896, - "min_ms": 4.005086608231068, - "p95_ms": 5.044202227145433 + "max_ms": 4.347208014223725, + "median_ms": 4.031431977637112, + "min_ms": 3.9591389941051602, + "p95_ms": 4.243721647071652 }, "repeat_mismatch_count": 0, "sequence_parallel": false, @@ -253,10 +253,10 @@ "tp_size": 2, "train_infer_mismatch_count": 0, "triton": { - "max_ms": 5.543430335819721, - "median_ms": 2.5280937552452087, - "min_ms": 1.921333372592926, - "p95_ms": 4.515600809827447 + "max_ms": 2.2346400073729455, + "median_ms": 1.963219983736053, + "min_ms": 1.8437679973430932, + "p95_ms": 2.1102602826431394 }, "weight_layout": "packed_forward_cache", "world_size": 4 @@ -266,13 +266,13 @@ "direction": "forward", "hidden": 4096, "intermediate": 12288, - "latency_ratio_triton_vs_official_distributed": 2.473328555193229, + "latency_ratio_triton_vs_official_distributed": 2.3647336647848283, "name": "tp2_cp2_sp", "official_distributed": { - "max_ms": 0.9094811975955963, - "median_ms": 0.32615475356578827, - "min_ms": 0.2819793298840523, - "p95_ms": 0.8830895647406578 + "max_ms": 0.8284339564852417, + "median_ms": 0.298925006063655, + "min_ms": 0.28363202000036836, + "p95_ms": 0.6086287612561136 }, "repeat_mismatch_count": 0, "sequence_parallel": true, @@ -283,10 +283,10 @@ "tp_size": 2, "train_infer_mismatch_count": 0, "triton": { - "max_ms": 2.9444824904203415, - "median_ms": 0.8066878654062748, - "min_ms": 0.7346402853727341, - "p95_ms": 1.7819570843130346 + "max_ms": 1.149271964095533, + "median_ms": 0.706878025084734, + "min_ms": 0.6899579893797636, + "p95_ms": 0.9988334029912947 }, "weight_layout": "packed_forward_cache", "world_size": 4 @@ -296,13 +296,13 @@ "direction": "train_fwd_bwd", "hidden": 4096, "intermediate": 12288, - "latency_ratio_triton_vs_official_distributed": 0.5550966504336481, + "latency_ratio_triton_vs_official_distributed": 0.4659099701126574, "name": "tp2_cp2_sp", "official_distributed": { - "max_ms": 6.967155262827873, - "median_ms": 4.586996044963598, - "min_ms": 4.187238402664661, - "p95_ms": 6.129474937915803 + "max_ms": 4.578732012305409, + "median_ms": 4.211539984680712, + "min_ms": 4.1677289991639555, + "p95_ms": 4.504749839543365 }, "repeat_mismatch_count": 0, "sequence_parallel": true, @@ -315,10 +315,10 @@ "tp_size": 2, "train_infer_mismatch_count": 0, "triton": { - "max_ms": 4.86080814152956, - "median_ms": 2.546226140111685, - "min_ms": 1.9829655066132545, - "p95_ms": 3.2532437238842262 + "max_ms": 2.256051986478269, + "median_ms": 1.9621984683908522, + "min_ms": 1.894902961794287, + "p95_ms": 2.1671605063602333 }, "weight_layout": "packed_forward_cache", "world_size": 4 @@ -328,13 +328,13 @@ "direction": "forward", "hidden": 4096, "intermediate": 12288, - "latency_ratio_triton_vs_official_distributed": 3.4050468145461728, + "latency_ratio_triton_vs_official_distributed": 3.3305188727841, "name": "tp8", "official_distributed": { - "max_ms": 1.0286271572113037, - "median_ms": 0.20749308168888092, - "min_ms": 0.1928461715579033, - "p95_ms": 0.5358625203371044 + "max_ms": 0.5265750223770738, + "median_ms": 0.20057399524375796, + "min_ms": 0.18795003416016698, + "p95_ms": 0.4599942360073328 }, "repeat_mismatch_count": 0, "sequence_parallel": false, @@ -345,10 +345,10 @@ "tp_size": 8, "train_infer_mismatch_count": 0, "triton": { - "max_ms": 1.8369676545262337, - "median_ms": 0.7065236568450928, - "min_ms": 0.6407918408513069, - "p95_ms": 1.6900700516998768 + "max_ms": 0.9437259868718684, + "median_ms": 0.6680154765490443, + "min_ms": 0.646052008960396, + "p95_ms": 0.8954497781815007 }, "weight_layout": "packed_forward_cache", "world_size": 8 @@ -358,13 +358,13 @@ "direction": "train_fwd_bwd", "hidden": 4096, "intermediate": 12288, - "latency_ratio_triton_vs_official_distributed": 2.3611604677472107, + "latency_ratio_triton_vs_official_distributed": 2.6553104389775055, "name": "tp8", "official_distributed": { - "max_ms": 2.177082933485508, - "median_ms": 0.8685095235705376, - "min_ms": 0.6101867184042931, - "p95_ms": 1.8530789297074082 + "max_ms": 0.8651090320199728, + "median_ms": 0.6639944913331419, + "min_ms": 0.6107499939389527, + "p95_ms": 0.8394304692046718 }, "repeat_mismatch_count": 0, "sequence_parallel": false, @@ -377,10 +377,10 @@ "tp_size": 8, "train_infer_mismatch_count": 0, "triton": { - "max_ms": 5.000975914299488, - "median_ms": 2.0506903529167175, - "min_ms": 1.6332557424902916, - "p95_ms": 4.608689062297344 + "max_ms": 1.9813220133073628, + "median_ms": 1.7631115042604506, + "min_ms": 1.6018060268834233, + "p95_ms": 1.856191793922335 }, "weight_layout": "packed_forward_cache", "world_size": 8 @@ -390,13 +390,13 @@ "direction": "forward", "hidden": 4096, "intermediate": 12288, - "latency_ratio_triton_vs_official_distributed": 3.4271073503642904, + "latency_ratio_triton_vs_official_distributed": 2.9361465256107735, "name": "tp4_cp2", "official_distributed": { - "max_ms": 1.757740043103695, - "median_ms": 0.2423599362373352, - "min_ms": 0.2230815589427948, - "p95_ms": 0.8416332770138979 + "max_ms": 0.557270017452538, + "median_ms": 0.2377095224801451, + "min_ms": 0.22454396821558475, + "p95_ms": 0.4702101548900828 }, "repeat_mismatch_count": 0, "sequence_parallel": false, @@ -407,10 +407,10 @@ "tp_size": 4, "train_infer_mismatch_count": 0, "triton": { - "max_ms": 2.3825177922844887, - "median_ms": 0.8305935189127922, - "min_ms": 0.6722584366798401, - "p95_ms": 1.6696299426257601 + "max_ms": 0.9553729905746877, + "median_ms": 0.697949988534674, + "min_ms": 0.6752759800292552, + "p95_ms": 0.9218062448780984 }, "weight_layout": "packed_forward_cache", "world_size": 8 @@ -420,13 +420,13 @@ "direction": "train_fwd_bwd", "hidden": 4096, "intermediate": 12288, - "latency_ratio_triton_vs_official_distributed": 0.8187486604980407, + "latency_ratio_triton_vs_official_distributed": 0.8270425369545604, "name": "tp4_cp2", "official_distributed": { - "max_ms": 5.41381910443306, - "median_ms": 3.0070655047893524, - "min_ms": 2.4463413283228874, - "p95_ms": 4.813990509137511 + "max_ms": 2.754604967776686, + "median_ms": 2.437841991195455, + "min_ms": 2.3977020173333585, + "p95_ms": 2.7159105316968635 }, "repeat_mismatch_count": 0, "sequence_parallel": false, @@ -439,10 +439,10 @@ "tp_size": 4, "train_infer_mismatch_count": 0, "triton": { - "max_ms": 4.88690659403801, - "median_ms": 2.462030854076147, - "min_ms": 1.8442394211888313, - "p95_ms": 4.497065208852291 + "max_ms": 2.177905000280589, + "median_ms": 2.0161990250926465, + "min_ms": 1.8386300071142614, + "p95_ms": 2.136603739927523 }, "weight_layout": "packed_forward_cache", "world_size": 8 @@ -452,13 +452,13 @@ "direction": "forward", "hidden": 4096, "intermediate": 12288, - "latency_ratio_triton_vs_official_distributed": 2.416452985140764, + "latency_ratio_triton_vs_official_distributed": 2.440941702768503, "name": "tp4_cp2_sp", "official_distributed": { - "max_ms": 0.9128358215093613, - "median_ms": 0.31526200473308563, - "min_ms": 0.28138794004917145, - "p95_ms": 0.8974010124802589 + "max_ms": 1.010014966595918, + "median_ms": 0.3099615278188139, + "min_ms": 0.28650701278820634, + "p95_ms": 0.8037662366405118 }, "repeat_mismatch_count": 0, "sequence_parallel": true, @@ -469,10 +469,10 @@ "tp_size": 4, "train_infer_mismatch_count": 0, "triton": { - "max_ms": 2.155771479010582, - "median_ms": 0.7618158124387264, - "min_ms": 0.6957026198506355, - "p95_ms": 1.7150717787444583 + "max_ms": 1.0109160211868584, + "median_ms": 0.7565980195067823, + "min_ms": 0.7171489996835589, + "p95_ms": 0.9613319649361074 }, "weight_layout": "packed_forward_cache", "world_size": 8 @@ -482,13 +482,13 @@ "direction": "train_fwd_bwd", "hidden": 4096, "intermediate": 12288, - "latency_ratio_triton_vs_official_distributed": 0.8257421849400354, + "latency_ratio_triton_vs_official_distributed": 0.7057388970769427, "name": "tp4_cp2_sp", "official_distributed": { - "max_ms": 4.409045912325382, - "median_ms": 2.9345820657908916, - "min_ms": 2.5761546567082405, - "p95_ms": 4.245032416656614 + "max_ms": 3.3931249636225402, + "median_ms": 2.8060810000170022, + "min_ms": 2.6044599944725633, + "p95_ms": 3.112885437440127 }, "repeat_mismatch_count": 0, "sequence_parallel": true, @@ -501,10 +501,10 @@ "tp_size": 4, "train_infer_mismatch_count": 0, "triton": { - "max_ms": 4.953155294060707, - "median_ms": 2.4232082068920135, - "min_ms": 1.9790586084127426, - "p95_ms": 3.922184882685543 + "max_ms": 2.211904968135059, + "median_ms": 1.9803605100605637, + "min_ms": 1.918218971695751, + "p95_ms": 2.177845605183393 }, "weight_layout": "packed_forward_cache", "world_size": 8 @@ -512,227 +512,324 @@ ], "distributed_platform_comparison": { "contract": "same M=32 workload, direction, and TP/CP/SP topology", + "note": "H100 CUDA values are preserved historical measurements; H100 official and MI300X distributed values are updated. Ratios are recomputed from these displayed columns, not taken from the user-supplied newer CUDA runs.", "rows": [ { "cp_size": 1, - "deterministic_mi300x_over_h100_ratio": 0.3139805027337412, + "deterministic_mi300x_over_h100_ratio": 0.23906797761963436, "direction": "forward", "h100_deterministic_cuda_ms": 2.7896, - "h100_deterministic_over_official_ratio": 17.97422680412371, - "h100_official_distributed_ms": 0.1552, - "mi300x_deterministic_over_official_ratio": 3.770714554916725, - "mi300x_deterministic_triton_ms": 0.8758800104260445, - "mi300x_official_distributed_ms": 0.23228488862514496, + "h100_deterministic_over_official_ratio": 16.875983061101028, + "h100_official_distributed_ms": 0.1653, + "h100_official_distributed_summary_ms": { + "max_ms": 0.3013, + "median_ms": 0.1653, + "min_ms": 0.142, + "p95_ms": 0.2055 + }, + "mi300x_deterministic_over_official_ratio": 3.0819895795813808, + "mi300x_deterministic_triton_ms": 0.666904030367732, + "mi300x_official_distributed_ms": 0.2163875033147633, "name": "tp2", "sequence_parallel": false, "tp_size": 2 }, { "cp_size": 1, - "deterministic_mi300x_over_h100_ratio": 0.26626598834991455, + "deterministic_mi300x_over_h100_ratio": 0.2144949771463871, "direction": "train_fwd_bwd", "h100_deterministic_cuda_ms": 7.8125, - "h100_deterministic_over_official_ratio": 12.398825583240756, - "h100_official_distributed_ms": 0.6301, - "mi300x_deterministic_over_official_ratio": 2.978428538282444, - "mi300x_deterministic_triton_ms": 2.0802030339837074, - "mi300x_official_distributed_ms": 0.6984230130910873, + "h100_deterministic_over_official_ratio": 12.308964865290688, + "h100_official_distributed_ms": 0.6347, + "h100_official_distributed_summary_ms": { + "max_ms": 0.6997, + "median_ms": 0.6347, + "min_ms": 0.5826, + "p95_ms": 0.6955 + }, + "mi300x_deterministic_over_official_ratio": 2.565423034622706, + "mi300x_deterministic_triton_ms": 1.6757420089561492, + "mi300x_official_distributed_ms": 0.6532029947265983, "name": "tp2", "sequence_parallel": false, "tp_size": 2 }, { "cp_size": 1, - "deterministic_mi300x_over_h100_ratio": 0.26673962049246625, + "deterministic_mi300x_over_h100_ratio": 0.2112829486952149, "direction": "forward", "h100_deterministic_cuda_ms": 3.331, - "h100_deterministic_over_official_ratio": 21.46262886597938, - "h100_official_distributed_ms": 0.1552, - "mi300x_deterministic_over_official_ratio": 2.8983204548458, - "mi300x_deterministic_triton_ms": 0.888509675860405, - "mi300x_official_distributed_ms": 0.3065601922571659, + "h100_deterministic_over_official_ratio": 16.87436676798379, + "h100_official_distributed_ms": 0.1974, + "h100_official_distributed_summary_ms": { + "max_ms": 0.2273, + "median_ms": 0.1974, + "min_ms": 0.1807, + "p95_ms": 0.2161 + }, + "mi300x_deterministic_over_official_ratio": 2.491084021205794, + "mi300x_deterministic_triton_ms": 0.7037835021037608, + "mi300x_official_distributed_ms": 0.28252098127268255, "name": "tp2_sp", "sequence_parallel": true, "tp_size": 2 }, { "cp_size": 1, - "deterministic_mi300x_over_h100_ratio": 0.21400019457480116, + "deterministic_mi300x_over_h100_ratio": 0.15244131147458864, "direction": "train_fwd_bwd", "h100_deterministic_cuda_ms": 11.7695, - "h100_deterministic_over_official_ratio": 18.678781145849868, - "h100_official_distributed_ms": 0.6301, - "mi300x_deterministic_over_official_ratio": 1.771568250466244, - "mi300x_deterministic_triton_ms": 2.5186752900481224, - "mi300x_official_distributed_ms": 1.4217207208275795, + "h100_deterministic_over_official_ratio": 16.433258866238482, + "h100_official_distributed_ms": 0.7162, + "h100_official_distributed_summary_ms": { + "max_ms": 1.2747, + "median_ms": 0.7162, + "min_ms": 0.6713, + "p95_ms": 1.0543 + }, + "mi300x_deterministic_over_official_ratio": 2.078108717055042, + "mi300x_deterministic_triton_ms": 1.7941580154001713, + "mi300x_official_distributed_ms": 0.8633609977550805, "name": "tp2_sp", "sequence_parallel": true, "tp_size": 2 }, { "cp_size": 1, - "deterministic_mi300x_over_h100_ratio": 0.38382024783593865, + "deterministic_mi300x_over_h100_ratio": 0.3078849092749437, "direction": "forward", "h100_deterministic_cuda_ms": 2.1687, - "h100_deterministic_over_official_ratio": 14.174509803921568, - "h100_official_distributed_ms": 0.153, - "mi300x_deterministic_over_official_ratio": 3.4979071801625334, - "mi300x_deterministic_triton_ms": 0.8323909714818001, - "mi300x_official_distributed_ms": 0.23796828463673592, + "h100_deterministic_over_official_ratio": 12.449483352468427, + "h100_official_distributed_ms": 0.1742, + "h100_official_distributed_summary_ms": { + "max_ms": 1.4339, + "median_ms": 0.1742, + "min_ms": 0.1545, + "p95_ms": 0.8594 + }, + "mi300x_deterministic_over_official_ratio": 2.980595497031791, + "mi300x_deterministic_triton_ms": 0.6677100027445704, + "mi300x_official_distributed_ms": 0.2240189933218062, "name": "tp4", "sequence_parallel": false, "tp_size": 4 }, { "cp_size": 1, - "deterministic_mi300x_over_h100_ratio": 0.2444670213169173, + "deterministic_mi300x_over_h100_ratio": 0.1768051411980418, "direction": "train_fwd_bwd", "h100_deterministic_cuda_ms": 9.3954, - "h100_deterministic_over_official_ratio": 16.44277213860693, - "h100_official_distributed_ms": 0.5714, - "mi300x_deterministic_over_official_ratio": 2.7738258924780523, - "mi300x_deterministic_triton_ms": 2.296865452080965, - "mi300x_official_distributed_ms": 0.82804961130023, + "h100_deterministic_over_official_ratio": 14.543962848297214, + "h100_official_distributed_ms": 0.646, + "h100_official_distributed_summary_ms": { + "max_ms": 0.7232, + "median_ms": 0.646, + "min_ms": 0.5959, + "p95_ms": 0.7029 + }, + "mi300x_deterministic_over_official_ratio": 2.4507841825531296, + "mi300x_deterministic_triton_ms": 1.661155023612082, + "mi300x_official_distributed_ms": 0.6778055103495717, "name": "tp4", "sequence_parallel": false, "tp_size": 4 }, { "cp_size": 2, - "deterministic_mi300x_over_h100_ratio": 0.26758165322124217, + "deterministic_mi300x_over_h100_ratio": 0.24149783047852758, "direction": "forward", "h100_deterministic_cuda_ms": 2.8261, - "h100_deterministic_over_official_ratio": 18.47124183006536, - "h100_official_distributed_ms": 0.153, - "mi300x_deterministic_over_official_ratio": 3.264261952381431, - "mi300x_deterministic_triton_ms": 0.7562125101685524, - "mi300x_official_distributed_ms": 0.23166416212916374, + "h100_deterministic_over_official_ratio": 17.663124999999997, + "h100_official_distributed_ms": 0.16, + "h100_official_distributed_summary_ms": { + "max_ms": 1.4457, + "median_ms": 0.16, + "min_ms": 0.1411, + "p95_ms": 0.8807 + }, + "mi300x_deterministic_over_official_ratio": 3.0189364393278644, + "mi300x_deterministic_triton_ms": 0.6824970187153667, + "mi300x_official_distributed_ms": 0.22607200662605464, "name": "tp2_cp2", "sequence_parallel": false, "tp_size": 2 }, { "cp_size": 2, - "deterministic_mi300x_over_h100_ratio": 0.15632342880036165, + "deterministic_mi300x_over_h100_ratio": 0.12139473811454553, "direction": "train_fwd_bwd", "h100_deterministic_cuda_ms": 16.1722, - "h100_deterministic_over_official_ratio": 28.302765138256913, - "h100_official_distributed_ms": 0.5714, - "mi300x_deterministic_over_official_ratio": 0.5639282606270454, - "mi300x_deterministic_triton_ms": 2.5280937552452087, - "mi300x_official_distributed_ms": 4.483005963265896, + "h100_deterministic_over_official_ratio": 13.28312114989733, + "h100_official_distributed_ms": 1.2175, + "h100_official_distributed_summary_ms": { + "max_ms": 2.2589, + "median_ms": 1.2175, + "min_ms": 1.1868, + "p95_ms": 2.2282 + }, + "mi300x_deterministic_over_official_ratio": 0.486978322994483, + "mi300x_deterministic_triton_ms": 1.963219983736053, + "mi300x_official_distributed_ms": 4.031431977637112, "name": "tp2_cp2", "sequence_parallel": false, "tp_size": 2 }, { "cp_size": 2, - "deterministic_mi300x_over_h100_ratio": 0.22524372184237304, + "deterministic_mi300x_over_h100_ratio": 0.19737477664732617, "direction": "forward", "h100_deterministic_cuda_ms": 3.5814, - "h100_deterministic_over_official_ratio": 23.4078431372549, - "h100_official_distributed_ms": 0.153, - "mi300x_deterministic_over_official_ratio": 2.473328555193229, - "mi300x_deterministic_triton_ms": 0.8066878654062748, - "mi300x_official_distributed_ms": 0.32615475356578827, + "h100_deterministic_over_official_ratio": 14.768659793814432, + "h100_official_distributed_ms": 0.2425, + "h100_official_distributed_summary_ms": { + "max_ms": 0.3538, + "median_ms": 0.2425, + "min_ms": 0.2271, + "p95_ms": 0.2821 + }, + "mi300x_deterministic_over_official_ratio": 2.3647336647848283, + "mi300x_deterministic_triton_ms": 0.706878025084734, + "mi300x_official_distributed_ms": 0.298925006063655, "name": "tp2_cp2_sp", "sequence_parallel": true, "tp_size": 2 }, { "cp_size": 2, - "deterministic_mi300x_over_h100_ratio": 0.15165676796738922, + "deterministic_mi300x_over_h100_ratio": 0.1168712680852712, "direction": "train_fwd_bwd", "h100_deterministic_cuda_ms": 16.7894, - "h100_deterministic_over_official_ratio": 29.3829191459573, - "h100_official_distributed_ms": 0.5714, - "mi300x_deterministic_over_official_ratio": 0.5550966504336481, - "mi300x_deterministic_triton_ms": 2.546226140111685, - "mi300x_official_distributed_ms": 4.586996044963598, + "h100_deterministic_over_official_ratio": 10.574001763446278, + "h100_official_distributed_ms": 1.5878, + "h100_official_distributed_summary_ms": { + "max_ms": 1.7189, + "median_ms": 1.5878, + "min_ms": 1.4295, + "p95_ms": 1.6838 + }, + "mi300x_deterministic_over_official_ratio": 0.4659099701126574, + "mi300x_deterministic_triton_ms": 1.9621984683908522, + "mi300x_official_distributed_ms": 4.211539984680712, "name": "tp2_cp2_sp", "sequence_parallel": true, "tp_size": 2 }, { "cp_size": 1, - "deterministic_mi300x_over_h100_ratio": 0.3044573200228789, + "deterministic_mi300x_over_h100_ratio": 0.2878632580147566, "direction": "forward", "h100_deterministic_cuda_ms": 2.3206, - "h100_deterministic_over_official_ratio": 15.098243331164607, - "h100_official_distributed_ms": 0.1537, - "mi300x_deterministic_over_official_ratio": 3.4050468145461728, - "mi300x_deterministic_triton_ms": 0.7065236568450928, - "mi300x_official_distributed_ms": 0.20749308168888092, + "h100_deterministic_over_official_ratio": 5.857142857142858, + "h100_official_distributed_ms": 0.3962, + "h100_official_distributed_summary_ms": { + "max_ms": 1.2054, + "median_ms": 0.3962, + "min_ms": 0.3521, + "p95_ms": 1.178 + }, + "mi300x_deterministic_over_official_ratio": 3.3305188727841, + "mi300x_deterministic_triton_ms": 0.6680154765490443, + "mi300x_official_distributed_ms": 0.20057399524375796, "name": "tp8", "sequence_parallel": false, "tp_size": 8 }, { "cp_size": 1, - "deterministic_mi300x_over_h100_ratio": 0.20228758105220396, + "deterministic_mi300x_over_h100_ratio": 0.17391975381114186, "direction": "train_fwd_bwd", "h100_deterministic_cuda_ms": 10.1375, - "h100_deterministic_over_official_ratio": 20.25069916100679, - "h100_official_distributed_ms": 0.5006, - "mi300x_deterministic_over_official_ratio": 2.3611604677472107, - "mi300x_deterministic_triton_ms": 2.0506903529167175, - "mi300x_official_distributed_ms": 0.8685095235705376, + "h100_deterministic_over_official_ratio": 8.40449345050572, + "h100_official_distributed_ms": 1.2062, + "h100_official_distributed_summary_ms": { + "max_ms": 1.4588, + "median_ms": 1.2062, + "min_ms": 1.1445, + "p95_ms": 1.44 + }, + "mi300x_deterministic_over_official_ratio": 2.6553104389775055, + "mi300x_deterministic_triton_ms": 1.7631115042604506, + "mi300x_official_distributed_ms": 0.6639944913331419, "name": "tp8", "sequence_parallel": false, "tp_size": 8 }, { "cp_size": 2, - "deterministic_mi300x_over_h100_ratio": 0.3751382136817633, + "deterministic_mi300x_over_h100_ratio": 0.3152296592451443, "direction": "forward", "h100_deterministic_cuda_ms": 2.2141, - "h100_deterministic_over_official_ratio": 14.4053350683149, - "h100_official_distributed_ms": 0.1537, - "mi300x_deterministic_over_official_ratio": 3.4271073503642904, - "mi300x_deterministic_triton_ms": 0.8305935189127922, - "mi300x_official_distributed_ms": 0.2423599362373352, + "h100_deterministic_over_official_ratio": 5.81739358906989, + "h100_official_distributed_ms": 0.3806, + "h100_official_distributed_summary_ms": { + "max_ms": 2.5012, + "median_ms": 0.3806, + "min_ms": 0.354, + "p95_ms": 1.4873 + }, + "mi300x_deterministic_over_official_ratio": 2.9361465256107735, + "mi300x_deterministic_triton_ms": 0.697949988534674, + "mi300x_official_distributed_ms": 0.2377095224801451, "name": "tp4_cp2", "sequence_parallel": false, "tp_size": 4 }, { "cp_size": 2, - "deterministic_mi300x_over_h100_ratio": 0.14865809995810497, + "deterministic_mi300x_over_h100_ratio": 0.12173865153291309, "direction": "train_fwd_bwd", "h100_deterministic_cuda_ms": 16.5617, - "h100_deterministic_over_official_ratio": 33.083699560527364, - "h100_official_distributed_ms": 0.5006, - "mi300x_deterministic_over_official_ratio": 0.8187486604980407, - "mi300x_deterministic_triton_ms": 2.462030854076147, - "mi300x_official_distributed_ms": 3.0070655047893524, + "h100_deterministic_over_official_ratio": 10.589322250639384, + "h100_official_distributed_ms": 1.564, + "h100_official_distributed_summary_ms": { + "max_ms": 2.9183, + "median_ms": 1.564, + "min_ms": 1.468, + "p95_ms": 2.7782 + }, + "mi300x_deterministic_over_official_ratio": 0.8270425369545604, + "mi300x_deterministic_triton_ms": 2.0161990250926465, + "mi300x_official_distributed_ms": 2.437841991195455, "name": "tp4_cp2", "sequence_parallel": false, "tp_size": 4 }, { "cp_size": 2, - "deterministic_mi300x_over_h100_ratio": 0.23104931834245007, + "deterministic_mi300x_over_h100_ratio": 0.22946682624856918, "direction": "forward", "h100_deterministic_cuda_ms": 3.2972, - "h100_deterministic_over_official_ratio": 21.45217957059206, - "h100_official_distributed_ms": 0.1537, - "mi300x_deterministic_over_official_ratio": 2.416452985140764, - "mi300x_deterministic_triton_ms": 0.7618158124387264, - "mi300x_official_distributed_ms": 0.31526200473308563, + "h100_deterministic_over_official_ratio": 6.5825514074665605, + "h100_official_distributed_ms": 0.5009, + "h100_official_distributed_summary_ms": { + "max_ms": 0.567, + "median_ms": 0.5009, + "min_ms": 0.4745, + "p95_ms": 0.5523 + }, + "mi300x_deterministic_over_official_ratio": 2.440941702768503, + "mi300x_deterministic_triton_ms": 0.7565980195067823, + "mi300x_official_distributed_ms": 0.3099615278188139, "name": "tp4_cp2_sp", "sequence_parallel": true, "tp_size": 4 }, { "cp_size": 2, - "deterministic_mi300x_over_h100_ratio": 0.1413303747815495, + "deterministic_mi300x_over_h100_ratio": 0.11550187569248054, "direction": "train_fwd_bwd", "h100_deterministic_cuda_ms": 17.1457, - "h100_deterministic_over_official_ratio": 34.250299640431486, - "h100_official_distributed_ms": 0.5006, - "mi300x_deterministic_over_official_ratio": 0.8257421849400354, - "mi300x_deterministic_triton_ms": 2.4232082068920135, - "mi300x_official_distributed_ms": 2.9345820657908916, + "h100_deterministic_over_official_ratio": 9.899936485940298, + "h100_official_distributed_ms": 1.7319, + "h100_official_distributed_summary_ms": { + "max_ms": 1.9283, + "median_ms": 1.7319, + "min_ms": 1.7104, + "p95_ms": 1.922 + }, + "mi300x_deterministic_over_official_ratio": 0.7057388970769427, + "mi300x_deterministic_triton_ms": 1.9803605100605637, + "mi300x_official_distributed_ms": 2.8060810000170022, "name": "tp4_cp2_sp", "sequence_parallel": true, "tp_size": 4 @@ -746,14 +843,176 @@ "deterministic_compute": "ROCm-native Triton", "deterministic_transport": "fixed-tree HIP IPC with RCCL fallback on ROCm", "distributed_speed_comparison": "four same-topology H100/MI300X paths", - "git_commit": "caef501101a3906c733076f31f3b5a9870169d16", + "git_commit": "98dcd38fb635e1e0eab9035aa4c4b40483cc36b8", "gpu": "AMD Instinct MI300X", "gpu_count": 8, "hip": "7.14.60850", "python": "3.12.3", "single_gpu_speed_context": "Hugging Face Transformers Qwen3MLP, TP=1", "torch": "2.12.0+rocm7.14.0a20260608", - "transformers": "5.10.4" + "transformers": "5.13.1" + }, + "measurement_sources": { + "h100_cuda_distributed": { + "environment": { + "architecture": "sm_90", + "cpu": "Intel(R) Xeon(R) Platinum 8468", + "cpu_threads": 96, + "cuda": "13.0", + "deterministic_compute": "native CUDA kernels", + "deterministic_transport": "fixed-order CUDA IPC", + "gpu": "NVIDIA H100 80GB HBM3", + "gpu_count": 8, + "python": "3.11.15", + "torch": "2.13.0+cu130", + "transformers": "5.13.1" + }, + "git_commit": "8576fa4bf449734ae99e9b50be8756bb282a8916", + "methodology": { + "cpu_timing": "wall clock with 96 PyTorch intra-op threads", + "dtype": "bfloat16", + "gpu_timing": "GPU events for single GPU; synchronized slowest-rank wall clock for distributed", + "hidden": 4096, + "intermediate": 12288, + "samples": 10, + "tokens": [ + 1, + 8, + 32 + ], + "training_samples": 5, + "warmup": 3 + }, + "note": "Historical CUDA values preserved at the user's request; newer supplied CUDA values and ratios are not substituted.", + "source": "cuda_cpu_comparison.json:distributed[].cuda_h100_ms" + }, + "h100_official_distributed": { + "baseline": "F.linear + NCCL at the same TP/CP/SP topology; TP=1 is used only for untimed RL-Kernel bitwise checks", + "commands": [ + "env PYTHONPATH=/tmp/rlk-head-98dcd38-build-GYkK6iE5/repo CUDA_VISIBLE_DEVICES=0,1 NCCL_NVLS_ENABLE=0 NCCL_IB_DISABLE=1 RL_KERNEL_DET_GEMM_BACKEND=sm90 FFN_WORLD_SIZE=2 FFN_WARMUP=5 FFN_SAMPLES=20 FFN_TRAINING_SAMPLES=10 /home/ellm/miniconda3/envs/hsr_env_rl_kernel/bin/python /tmp/rlk-head-98dcd38-build-GYkK6iE5/repo/run_h100_distributed_ffn_data.py", + "env PYTHONPATH=/tmp/rlk-head-98dcd38-build-GYkK6iE5/repo CUDA_VISIBLE_DEVICES=0,1,2,3 NCCL_NVLS_ENABLE=0 NCCL_IB_DISABLE=1 RL_KERNEL_DET_GEMM_BACKEND=sm90 FFN_WORLD_SIZE=4 FFN_WARMUP=5 FFN_SAMPLES=20 FFN_TRAINING_SAMPLES=10 /home/ellm/miniconda3/envs/hsr_env_rl_kernel/bin/python /tmp/rlk-head-98dcd38-build-GYkK6iE5/repo/run_h100_distributed_ffn_data.py" + ], + "cuda": "13.0", + "directions": [ + "forward", + "train_fwd_bwd" + ], + "dtype": "bfloat16", + "environment_variables": { + "NCCL_IB_DISABLE": "1", + "NCCL_NVLS_ENABLE": "0", + "RL_KERNEL_DET_GEMM_BACKEND": "sm90" + }, + "fields": [ + "official_h100_ms", + "official_h100_summary_ms" + ], + "git_commit": "98dcd38fb635e1e0eab9035aa4c4b40483cc36b8", + "gpu": "NVIDIA H100 80GB", + "note": "All 16 official distributed rows are updated from user-supplied H100 measurements. CUDA and other historical rows remain unchanged; top-level environment and methodology describe those historical data. All H100 world sizes used 5/20/10 warmup/forward/training settings. ROCm uses 10/50/20 CLI settings (its distributed training halves warmup to 5); shapes, topology and timing aggregation match, but sample counts differ. The user discarded H100 10/50/20 reruns contaminated by competing GPU 0-3 workloads. The supplied H100 runner and individual samples are not available in this repository.", + "samples": 20, + "shape": { + "hidden": 4096, + "intermediate": 12288, + "tokens": 32 + }, + "source": "User-supplied H100 table on 2026-09-06", + "timing": "synchronized wall clock; slowest rank per sample", + "topologies": [ + "tp2", + "tp2_sp", + "tp4", + "tp2_cp2", + "tp2_cp2_sp", + "tp8", + "tp4_cp2", + "tp4_cp2_sp" + ], + "torch": "2.13.0+cu130", + "training_samples": 10, + "units": "ms", + "validation": { + "hidden_gradient_mismatch": 0, + "output_mismatch": 0, + "reference": "RL-Kernel CUDA TP=1", + "repeat_mismatch": 0, + "train_infer_mismatch": 0, + "weight_gradient_mismatch": 0 + }, + "warmup": 5 + }, + "mi300x_distributed": { + "effective_forward_warmup": 10, + "effective_training_warmup": 5, + "environment": { + "NCCL_IB_DISABLE": "1", + "architecture": "gfx942:sramecc+:xnack-", + "deterministic_compute": "ROCm-native Triton", + "deterministic_transport": "fixed-tree HIP IPC with RCCL fallback on ROCm", + "distributed_speed_comparison": "four same-topology H100/MI300X paths", + "git_commit": "98dcd38fb635e1e0eab9035aa4c4b40483cc36b8", + "gpu": "AMD Instinct MI300X", + "gpu_count": 8, + "hip": "7.14.60850", + "python": "3.12.3", + "single_gpu_speed_context": "Hugging Face Transformers Qwen3MLP, TP=1", + "torch": "2.12.0+rocm7.14.0a20260608", + "transformers": "5.13.1" + }, + "methodology": { + "distributed_timing": "synchronized wall clock, slowest rank/sample", + "distributed_worker_cpu_affinity": "one NUMA-local CPU per GPU rank", + "operator_only": true, + "samples": 50, + "single_gpu_timing": "GPU events, median and p95", + "tp1_forward_cache_bytes": 301989888, + "training_samples": 20, + "triton_weight_layout": "packed_forward_cache_outside_timed_region", + "warmup": 10 + }, + "native_baseline_validation": { + "scope": "All eight TP/CP/SP layouts; output, dHidden, dGate, dUp and dDown checked against unsharded Hugging Face Qwen3MLP", + "fp64_shape": [32, 64, 512], + "fp64_max_relative_l2": 9.345613086441527e-16, + "bf16_shape": [32, 4096, 12288], + "bf16_vs_fp32_max_relative_l2": 0.005519561003893614, + "timing_repeats": 3, + "max_median_latency_change_percent_rounded": 6.7, + "note": "Independent local validation passed; numerical errors are tolerance checks of the native baseline, not bitwise equality claims." + }, + "shared_server_note": "An unrelated process held about 180 GiB on GPU 7. All layouts completed; these are not exclusive-node measurements.", + "source": "MI300X rerun on 2026-09-06; measured summaries are embedded in this file under distributed_ffn" + }, + "note": "Top-level environment and methodology describe the MI300X distributed rerun. Single-GPU and dtype observations are preserved historical data with separate provenance below.", + "single_gpu_and_dtype_observations": { + "environment": { + "NCCL_IB_DISABLE": "1", + "architecture": "gfx942:sramecc+:xnack-", + "deterministic_compute": "ROCm-native Triton", + "deterministic_transport": "fixed-tree HIP IPC with RCCL fallback on ROCm", + "distributed_speed_comparison": "four same-topology H100/MI300X paths", + "git_commit": "caef501101a3906c733076f31f3b5a9870169d16", + "gpu": "AMD Instinct MI300X", + "gpu_count": 8, + "hip": "7.14.60850", + "python": "3.12.3", + "single_gpu_speed_context": "Hugging Face Transformers Qwen3MLP, TP=1", + "torch": "2.12.0+rocm7.14.0a20260608", + "transformers": "5.10.4" + }, + "methodology": { + "distributed_timing": "synchronized wall clock, slowest rank/sample", + "distributed_worker_cpu_affinity": "one NUMA-local CPU per GPU rank", + "operator_only": true, + "samples": 50, + "single_gpu_timing": "GPU events, median and p95", + "tp1_forward_cache_bytes": 301989888, + "training_samples": 20, + "triton_weight_layout": "packed_forward_cache_outside_timed_region", + "warmup": 10 + }, + "source": "Preserved original MI300X artifact" + } }, "methodology": { "distributed_timing": "synchronized wall clock, slowest rank/sample", @@ -767,116 +1026,117 @@ "warmup": 10 }, "previous_deterministic_comparison": { + "note": "Historical constants versus the current rerun; not a controlled attribution to PR #357.", "rows": [ { - "current_ms": 0.8758800104260445, + "current_ms": 0.666904030367732, "direction": "forward", - "latency_reduction_ratio": 0.031099087163262373, + "latency_reduction_ratio": 0.2622688997280703, "name": "tp2", "previous_ms": 0.9039933793246746 }, { - "current_ms": 2.0802030339837074, + "current_ms": 1.6757420089561492, "direction": "train_fwd_bwd", - "latency_reduction_ratio": 0.22945943339972408, + "latency_reduction_ratio": 0.3792782839162665, "name": "tp2", "previous_ms": 2.6996671222150326 }, { - "current_ms": 0.888509675860405, + "current_ms": 0.7037835021037608, "direction": "forward", - "latency_reduction_ratio": 0.158719653904269, + "latency_reduction_ratio": 0.33362658357888275, "name": "tp2_sp", "previous_ms": 1.0561398230493069 }, { - "current_ms": 2.5186752900481224, + "current_ms": 1.7941580154001713, "direction": "train_fwd_bwd", - "latency_reduction_ratio": 0.1504226199863158, + "latency_reduction_ratio": 0.3948104100290333, "name": "tp2_sp", "previous_ms": 2.9646214097738266 }, { - "current_ms": 0.8323909714818001, + "current_ms": 0.6677100027445704, "direction": "forward", - "latency_reduction_ratio": 0.00417650162140959, + "latency_reduction_ratio": 0.20119110656402106, "name": "tp4", "previous_ms": 0.8358820341527462 }, { - "current_ms": 2.296865452080965, + "current_ms": 1.661155023612082, "direction": "train_fwd_bwd", - "latency_reduction_ratio": 0.1492708192299168, + "latency_reduction_ratio": 0.38472971889177976, "name": "tp4", "previous_ms": 2.6998785324394703 }, { - "current_ms": 0.7562125101685524, + "current_ms": 0.6824970187153667, "direction": "forward", - "latency_reduction_ratio": 0.16124577124706252, + "latency_reduction_ratio": 0.24300741807194692, "name": "tp2_cp2", "previous_ms": 0.9015901014208794 }, { - "current_ms": 2.5280937552452087, + "current_ms": 1.963219983736053, "direction": "train_fwd_bwd", - "latency_reduction_ratio": 0.28997936652606293, + "latency_reduction_ratio": 0.4486253946836861, "name": "tp2_cp2", "previous_ms": 3.5605919547379017 }, { - "current_ms": 0.8066878654062748, + "current_ms": 0.706878025084734, "direction": "forward", - "latency_reduction_ratio": 0.28586663566666093, + "latency_reduction_ratio": 0.3742248967971883, "name": "tp2_cp2_sp", "previous_ms": 1.1296039447188377 }, { - "current_ms": 2.546226140111685, + "current_ms": 1.9621984683908522, "direction": "train_fwd_bwd", - "latency_reduction_ratio": 0.36230506379306326, + "latency_reduction_ratio": 0.5085730967042195, "name": "tp2_cp2_sp", "previous_ms": 3.992859274148941 }, { - "current_ms": 0.7065236568450928, + "current_ms": 0.6680154765490443, "direction": "forward", - "latency_reduction_ratio": 0.34939832397754955, + "latency_reduction_ratio": 0.38485854728140545, "name": "tp8", "previous_ms": 1.0859542526304722 }, { - "current_ms": 2.0506903529167175, + "current_ms": 1.7631115042604506, "direction": "train_fwd_bwd", - "latency_reduction_ratio": 0.19957512534856403, + "latency_reduction_ratio": 0.3118228196729178, "name": "tp8", "previous_ms": 2.5620022788643837 }, { - "current_ms": 0.8305935189127922, + "current_ms": 0.697949988534674, "direction": "forward", - "latency_reduction_ratio": 0.21168450338208877, + "latency_reduction_ratio": 0.33757634836066475, "name": "tp4_cp2", "previous_ms": 1.0536308400332928 }, { - "current_ms": 2.462030854076147, + "current_ms": 2.0161990250926465, "direction": "train_fwd_bwd", - "latency_reduction_ratio": 0.24475513187788767, + "latency_reduction_ratio": 0.381517106378663, "name": "tp4_cp2", "previous_ms": 3.259910736232996 }, { - "current_ms": 0.7618158124387264, + "current_ms": 0.7565980195067823, "direction": "forward", - "latency_reduction_ratio": 0.3613095756305654, + "latency_reduction_ratio": 0.3656840639616845, "name": "tp4_cp2_sp", "previous_ms": 1.1927778832614422 }, { - "current_ms": 2.4232082068920135, + "current_ms": 1.9803605100605637, "direction": "train_fwd_bwd", - "latency_reduction_ratio": 0.3537747388734832, + "latency_reduction_ratio": 0.4718741113129781, "name": "tp4_cp2_sp", "previous_ms": 3.7497887387871742 }