diff --git a/docs/nvbench_compare_robust.md b/docs/nvbench_compare_robust.md index 2ae41b30..7f47fe71 100644 --- a/docs/nvbench_compare_robust.md +++ b/docs/nvbench_compare_robust.md @@ -64,7 +64,8 @@ nvbench-compare-robust --display explain reference.json compare.json nvbench-compare-legacy reference.json compare.json ``` -Plot the comparison summary, or plot timings along a positive numeric axis. Add +Plot the comparison summary, or plot timings along a positive numeric axis. By +default, plotting uses Matplotlib's interactive `plt.show()` behavior. Add `--dark` to the summary plot when it should use a dark theme: ```bash @@ -72,6 +73,20 @@ nvbench-compare-robust --plot --dark reference.json compare.json nvbench-compare-robust --plot-along "Elements{io}" reference.json compare.json ``` +Save plots to files when running in CI, remote shells, or scripted workflows: + +```bash +nvbench-compare-robust --plot --plot-output compare.png reference.json compare.json +nvbench-compare-robust \ + --plot-along "Elements{io}" \ + --plot-along-output "plots/{benchmark}-device{device}-{axis}.png" \ + reference.json compare.json +``` + +When `--plot` and `--plot-along` are used together, choose one output mode for +both plots: either omit both output options to show both plots interactively, or +provide both `--plot-output` and `--plot-along-output` to save both plots. + Generate Python code with bulk sample/frequency filenames for every displayed row: @@ -246,6 +261,11 @@ file. The generated script contains a `bulk_rows` list. Each entry corresponds to one row that `nvbench-compare-robust` prints in its display tables after all benchmark, axis, device, and threshold filters are applied. +This output is also useful when the built-in `--plot` or `--plot-along` views +are too generic. The generated `bulk_rows` data and `load_bulk_data(row)` helper +let users build custom Matplotlib, Seaborn, or notebook visualizations from the +same paired comparison rows used by the tool. + Use `stdout` instead of a file path to print the generated Python code: ```bash @@ -298,6 +318,11 @@ Each `bulk_rows` entry includes: The generated script also defines `load_bulk_data(row)`, which reads the float32 sample and frequency files for a selected row. +When directory inputs are used, `bulk_rows` contains rows from all matching JSON +file pairs. Each row records its `reference_json` and `compare_json` path, so +custom plotting code can group rows by source file, benchmark, device, axis, or +reason code. + Select the first displayed row: ```python @@ -317,6 +342,17 @@ If `-b` and `-a` narrow the report to one comparison of interest, the desired entry is usually available positionally as `bulk_rows[0]`. If duplicate states remain after filtering, use `occurrence` to distinguish them. +Plot the selected row with regular Python plotting tools: + +```python +import matplotlib.pyplot as plt + +plt.hist(arrays["reference_samples"], alpha=0.5, label="reference") +plt.hist(arrays["compare_samples"], alpha=0.5, label="compare") +plt.legend() +plt.show() +``` + ## Time Estimates And Intervals `nvbench-compare-robust` first tries to build a robust timing input for both @@ -646,3 +682,59 @@ fraction: use `--threshold-diff 5` for a 5% threshold. This option affects table output. It does not change summary counters or the data used by `--plot-along`. + +### `--plot-output PATH` + +Save the summary plot generated by `--plot` to `PATH` and do not call +`plt.show()`. When this option is omitted, `--plot` keeps the default +interactive behavior. + +Interactive and file-based plot output cannot be mixed in one invocation. If +`--plot` and `--plot-along` are both requested, provide output paths for both +plots or omit output paths for both plots. + +Directory comparisons can generate one summary plot per matched JSON file pair. +When multiple summary plots resolve to the same output path, later plots are +written to the first available sibling path using `-copy-N` before the file +extension, such as `compare-copy-1.png`. The same disambiguation is used if the +requested file already exists. The actual saved path and any disambiguation +warning are written to stderr. + +### `--plot-along-output PATH_OR_TEMPLATE` + +Save plots generated by `--plot-along` to files and do not call `plt.show()`. +Because `--plot-along` may produce one plot per benchmark/device pair, this +option accepts filename templates with these fields: + +- `{benchmark}`: benchmark name +- `{device}`: compare-device id +- `{axis}`: selected plot axis name +- `{pair}`: zero-based positional device-pair index + +Field values are sanitized before substitution so benchmark or axis names from +input JSON cannot introduce path separators. Directory structure should be +written literally in the template, as in `plots/{benchmark}.png`. + +For example: + +```bash +nvbench-compare-robust \ + --plot-along "Elements{io}" \ + --plot-along-output "plots/{benchmark}-pair{pair}-{axis}.png" \ + reference.json compare.json +``` + +A plain path without template fields is valid. If multiple plot-along figures +resolve to the same path, later plots are written to the first available sibling +path using `-copy-N` before the file extension, such as +`plot-along-copy-1.png`. The same disambiguation is used if the requested file +already exists. The actual saved path and any disambiguation warning are +written to stderr. + +To avoid copy-suffixed filenames, narrow the comparison with `--benchmark`, +`--axis`, `--reference-devices`, or `--compare-devices`, pass the JSON files of +interest directly instead of a directory, or add more template fields such as +`{benchmark}`, `{device}`, `{pair}`, and `{axis}`. For duplicate-heavy +benchmarks, crowded legends, presentation-quality plots, or a different +grouping scheme, use `--bulk-debug-python` to export the paired rows and build a +custom visualization. diff --git a/python/scripts/_nvbench_compare_plotting.py b/python/scripts/_nvbench_compare_plotting.py new file mode 100644 index 00000000..464f0d44 --- /dev/null +++ b/python/scripts/_nvbench_compare_plotting.py @@ -0,0 +1,619 @@ +#!/usr/bin/env python +# +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +from __future__ import annotations + +import math +import os +import re +import sys +from collections.abc import Mapping, Sequence +from dataclasses import dataclass, field +from pathlib import Path +from string import Formatter +from typing import Any + +if __package__: + from .nvbench_tooling_deps import ToolingDependency, require_tooling_dependency +else: + from nvbench_tooling_deps import ( # type: ignore[no-redef] + ToolingDependency, + require_tooling_dependency, + ) + + +PlotAlongData = dict[str, dict[str, dict[float, float | None]]] +AxisValue = Mapping[str, Any] +ComparisonPlotEntry = tuple[str, float, str, str] + +PLOT_ALONG_OUTPUT_TEMPLATE_FIELDS = frozenset({"benchmark", "device", "axis", "pair"}) +PLOT_OUTPUT_FIELD_SAFE_CHARS = re.compile(r"[^A-Za-z0-9_-]+") + + +def parse_plot_axis_value(axis_name: str, axis_value: Any) -> float: + try: + value = float(axis_value) + except (TypeError, ValueError) as exc: + raise ValueError( + f"--plot-along requires numeric axis values; " + f"axis {axis_name!r} has value {axis_value!r}" + ) from exc + if not is_positive_finite(value): + raise ValueError( + f"--plot-along requires positive finite axis values; " + f"axis {axis_name!r} has value {axis_value!r}" + ) + return value + + +def extract_plot_axis_value( + axis_values: Sequence[AxisValue], + plot_along: str, + benchmark_name: str, + state_name: str, +) -> tuple[float, list[str]]: + axis_name_parts: list[str] = [] + for axis_value in axis_values: + if axis_value["name"] != plot_along: + axis_name_parts.append(f"""{axis_value["name"]} = {axis_value["value"]}""") + else: + return ( + parse_plot_axis_value(axis_value["name"], axis_value["value"]), + axis_name_parts, + ) + raise ValueError( + f"--plot-along axis {plot_along!r} is not present in " + f"benchmark {benchmark_name!r} state {state_name!r}" + ) + + +def format_plot_series_key( + state_key: str, + occurrence: int, + occurrence_count: int, + axis_name_parts: Sequence[str], +) -> str: + parts: list[str] = [] + if state_key: + parts.append(state_key) + if occurrence_count > 1: + parts.append(f"occurrence={occurrence + 1}/{occurrence_count}") + parts.extend(axis_name_parts) + return ", ".join(parts) + + +def ensure_plot_output_parent(output: str) -> None: + output_path = Path(output) + parent = output_path.parent + if parent != Path("."): + parent.mkdir(parents=True, exist_ok=True) + + +def add_copy_suffix(path: Path, counter: int) -> Path: + return path.with_name(f"{path.stem}-copy-{counter}{path.suffix}") + + +def resolve_plot_output_path( + output: str | None, + output_paths: set[str], + *, + description: str, +) -> str | None: + if output is None: + return None + + normalized_output_path = os.path.abspath(output) + if normalized_output_path not in output_paths and not os.path.exists(output): + output_paths.add(normalized_output_path) + return output + + output_path = Path(output) + counter = 1 + while True: + candidate_path = add_copy_suffix(output_path, counter) + normalized_candidate_path = os.path.abspath(candidate_path) + if ( + normalized_candidate_path not in output_paths + and not candidate_path.exists() + ): + output_paths.add(normalized_candidate_path) + resolved_output = str(candidate_path) + print( + f"Warning: {description} output {output!r} is unavailable; " + f"writing to {resolved_output!r} instead.", + file=sys.stderr, + ) + return resolved_output + counter += 1 + + +def validate_plot_along_output_template(output_template: str) -> None: + try: + parsed_fields = [ + (field_name, format_spec, conversion) + for _, field_name, format_spec, conversion in Formatter().parse( + output_template + ) + ] + except ValueError as exc: + raise ValueError(f"--plot-along-output template is invalid: {exc}") from exc + + valid_fields = ", ".join( + f"{{{field}}}" for field in sorted(PLOT_ALONG_OUTPUT_TEMPLATE_FIELDS) + ) + for field_name, format_spec, conversion in parsed_fields: + if field_name is None: + continue + if ( + field_name not in PLOT_ALONG_OUTPUT_TEMPLATE_FIELDS + or format_spec + or conversion + ): + raise ValueError( + f"--plot-along-output supports template fields {valid_fields}; " + f"got {{{field_name}}}" + ) + + +def validate_plot_along_axis_name(plot_along: str | None) -> None: + if plot_along is not None and not plot_along: + raise ValueError("--plot-along requires a non-empty axis name") + + +def sanitize_plot_output_component(value: object) -> str: + sanitized = PLOT_OUTPUT_FIELD_SAFE_CHARS.sub("_", str(value)) + sanitized = sanitized.strip("._-") + return sanitized or "value" + + +def format_plot_along_output_path( + output_template: str | None, + *, + benchmark_name: str, + device_id: int, + axis_name: str, + device_pair_index: int = 0, +) -> str | None: + if output_template is None: + return None + + # Keep this helper self-validating; main() validates CLI input earlier so + # bad templates fail before any comparison work starts. + validate_plot_along_output_template(output_template) + try: + return output_template.format( + benchmark=sanitize_plot_output_component(benchmark_name), + device=sanitize_plot_output_component(device_id), + axis=sanitize_plot_output_component(axis_name), + pair=sanitize_plot_output_component(device_pair_index), + ) + except (IndexError, KeyError, ValueError) as exc: + raise ValueError(f"--plot-along-output template is invalid: {exc}") from exc + + +def save_or_show_plot(fig: Any, plt: Any, output: str | None, description: str) -> None: + if output is None: + plt.show() + return + + try: + ensure_plot_output_parent(output) + fig.savefig(output, dpi=150) + except (OSError, ValueError) as exc: + raise ValueError(f"failed to write {description} to {output!r}: {exc}") from exc + print(f"Saved {description} to {output}", file=sys.stderr) + + +def use_noninteractive_matplotlib_backend(matplotlib: Any) -> None: + matplotlib.use("Agg") + + +def validate_plot_output_modes( + *, + plot: bool, + plot_output: str | None, + plot_along: str | None, + plot_along_output: str | None, +) -> bool: + requested_modes = [] + if plot: + requested_modes.append(("--plot", "--plot-output", plot_output is not None)) + if plot_along is not None: + requested_modes.append( + ("--plot-along", "--plot-along-output", plot_along_output is not None) + ) + + interactive_modes = [ + mode for mode, _, has_output in requested_modes if not has_output + ] + file_modes = [mode for mode, _, has_output in requested_modes if has_output] + if interactive_modes and file_modes: + missing_output_options = [ + output_option + for _, output_option, has_output in requested_modes + if not has_output + ] + provided_output_options = [ + output_option + for _, output_option, has_output in requested_modes + if has_output + ] + raise ValueError( + "Cannot mix interactive and file-based plot output in one invocation. " + f"{', '.join(interactive_modes)} would call plt.show(), while " + f"{', '.join(file_modes)} would save to files. " + "Add " + f"{', '.join(missing_output_options)} to save every requested plot, " + "or omit " + f"{', '.join(provided_output_options)} to show every requested plot interactively." + ) + + return bool(file_modes) + + +def plot_comparison_entries( + entries: Sequence[ComparisonPlotEntry], + title: str | None = None, + dark: bool = False, + output: str | None = None, + *, + tool_name: str = "nvbench-compare-robust", + force_noninteractive_backend: bool | None = None, +) -> int: + if not entries: + print("No comparison data to plot.", file=sys.stderr) + return 1 + + matplotlib = require_tooling_dependency( + ToolingDependency("matplotlib", "matplotlib", "plot rendering", extra="plot"), + tool_name=tool_name, + ) + if force_noninteractive_backend is None: + force_noninteractive_backend = output is not None + if force_noninteractive_backend: + use_noninteractive_matplotlib_backend(matplotlib) + + plt = require_tooling_dependency( + ToolingDependency( + "matplotlib.pyplot", "matplotlib", "plot rendering", extra="plot" + ), + tool_name=tool_name, + ) + ticker = require_tooling_dependency( + ToolingDependency( + "matplotlib.ticker", "matplotlib", "plot axis formatting", extra="plot" + ), + tool_name=tool_name, + ) + PercentFormatter = ticker.PercentFormatter + + labels, values, statuses, bench_names = map(list, zip(*entries, strict=True)) + + status_colors = { + "SLOW": "red", + "FAST": "green", + "SAME": "blue", + } + colors = [status_colors.get(status, "gray") for status in statuses] + + fig_height = max(4.0, 0.3 * len(entries) + 1.5) + fig, ax = plt.subplots(figsize=(10, fig_height)) + try: + if dark: + fig.patch.set_facecolor("black") + ax.set_facecolor("black") + ax.tick_params(colors="white") + ax.xaxis.label.set_color("white") + ax.yaxis.label.set_color("white") + ax.title.set_color("white") + for spine in ax.spines.values(): + spine.set_color("white") + + y_pos = range(len(labels)) + ax.barh(y_pos, values, color=colors) + ax.set_yticks(y_pos) + ax.set_yticklabels(labels) + ax.invert_yaxis() + ax.set_ylim(len(labels) - 0.5, -0.5) + + separator_color = "white" if dark else "gray" + ax.axvline(0, color=separator_color, linewidth=1, alpha=0.6) + for index in range(1, len(bench_names)): + if bench_names[index] != bench_names[index - 1]: + ax.axhline(index - 0.5, color=separator_color, linewidth=0.6, alpha=0.4) + ax.xaxis.set_major_formatter(PercentFormatter(1.0)) + + if title: + ax.set_title(title) + + min_val = min(values) + max_val = max(values) + if min_val == max_val: + pad = 0.05 if min_val == 0 else abs(min_val) * 0.1 + ax.set_xlim(min_val - pad, max_val + pad) + else: + pad = (max_val - min_val) * 0.1 + ax.set_xlim(min_val - pad, max_val + pad) + + fig.tight_layout() + + save_or_show_plot(fig, plt, output, "comparison plot") + finally: + plt.close(fig) + return 0 + + +def make_plot_along_data() -> PlotAlongData: + return { + "cmp": {}, + "ref": {}, + "cmp_noise": {}, + "ref_noise": {}, + } + + +def has_plot_along_data(plot_data: PlotAlongData) -> bool: + return any(axis_times for axis_times in plot_data["cmp"].values()) + + +def is_positive_finite(value: float | None) -> bool: + return value is not None and math.isfinite(value) and value > 0.0 + + +def is_usable_noise(value: float | None) -> bool: + return value is not None and math.isfinite(value) and value >= 0.0 + + +@dataclass +class PlotCollector: + plot_along: str | None + plot_summary: bool + dark: bool + plot_output: str | None + plot_along_output: str | None + output_paths: set[str] + tool_name: str + comparison_entries: list[ComparisonPlotEntry] = field(default_factory=list) + comparison_device_names: set[str] = field(default_factory=set) + plt: Any = None + force_noninteractive_backend: bool = field(init=False) + noninteractive_backend_selected: bool = field(init=False, default=False) + + def __post_init__(self) -> None: + validate_plot_along_axis_name(self.plot_along) + self.force_noninteractive_backend = validate_plot_output_modes( + plot=self.plot_summary, + plot_output=self.plot_output, + plot_along=self.plot_along, + plot_along_output=self.plot_along_output, + ) + if self.plot_along: + if self.force_noninteractive_backend: + matplotlib = require_tooling_dependency( + ToolingDependency( + "matplotlib", + "matplotlib", + "per-axis plot rendering", + extra="plot", + ), + tool_name=self.tool_name, + ) + use_noninteractive_matplotlib_backend(matplotlib) + self.noninteractive_backend_selected = True + self.plt = require_tooling_dependency( + ToolingDependency( + "matplotlib.pyplot", + "matplotlib", + "per-axis plot rendering", + extra="plot", + ), + tool_name=self.tool_name, + ) + sns = require_tooling_dependency( + ToolingDependency( + "seaborn", "seaborn", "per-axis plot styling", extra="plot" + ), + tool_name=self.tool_name, + ) + + sns.set_theme() + + def make_plot_along_data(self) -> PlotAlongData: + return make_plot_along_data() + + def record_plot_along( + self, + plot_data: PlotAlongData, + *, + ref_time: float | None, + cmp_time: float | None, + ref_noise: float | None, + cmp_noise: float | None, + axis_values: Sequence[AxisValue], + benchmark_name: str, + state_name: str, + occurrence: int, + occurrence_count: int, + ) -> None: + if ( + self.plot_along is None + or not is_positive_finite(ref_time) + or not is_positive_finite(cmp_time) + ): + return + + axis_value, axis_name_parts = extract_plot_axis_value( + axis_values, self.plot_along, benchmark_name, state_name + ) + axis_name = format_plot_series_key( + state_name, + occurrence, + occurrence_count, + axis_name_parts, + ) + + if axis_name not in plot_data["cmp"]: + plot_data["cmp"][axis_name] = {} + plot_data["ref"][axis_name] = {} + plot_data["cmp_noise"][axis_name] = {} + plot_data["ref_noise"][axis_name] = {} + + plot_data["cmp"][axis_name][axis_value] = cmp_time + plot_data["ref"][axis_name][axis_value] = ref_time + plot_data["cmp_noise"][axis_name][axis_value] = cmp_noise + plot_data["ref_noise"][axis_name][axis_value] = ref_noise + + def has_plot_along_data(self, plot_data: PlotAlongData) -> bool: + return bool(self.plot_along) and has_plot_along_data(plot_data) + + def record_summary_entry( + self, + *, + benchmark_name: str, + axis_label: str, + cmp_device_name: str | None, + frac_diff: float | None, + status: str, + ) -> None: + if not self.plot_summary or frac_diff is None or not math.isfinite(frac_diff): + return + + if axis_label: + label = f"{benchmark_name} | {axis_label}" + else: + label = benchmark_name + if cmp_device_name: + self.comparison_device_names.add(cmp_device_name) + self.comparison_entries.append((label, frac_diff, status, benchmark_name)) + + def render_plot_along( + self, + plot_data: PlotAlongData, + *, + benchmark_name: str, + cmp_device_id: int, + cmp_device_index: int, + cmp_device_name: str, + ) -> None: + if self.plot_along is None or self.plt is None: + return + + plot_along_output_path = format_plot_along_output_path( + self.plot_along_output, + benchmark_name=benchmark_name, + device_id=cmp_device_id, + axis_name=self.plot_along, + device_pair_index=cmp_device_index, + ) + plot_along_output_path = resolve_plot_output_path( + plot_along_output_path, + self.output_paths, + description="plot-along", + ) + + fig = self.plt.figure() + try: + self.plt.xscale("log") + self.plt.yscale("log") + self.plt.xlabel(self.plot_along) + self.plt.ylabel("time [s]") + self.plt.title(cmp_device_name) + + for axis in plot_data["cmp"].keys(): + self._plot_line(plot_data, "cmp", "-", axis, axis) + self._plot_line(plot_data, "ref", "--", axis + " ref", axis) + + self.plt.legend() + save_or_show_plot( + fig, self.plt, plot_along_output_path, "plot-along output" + ) + finally: + self.plt.close(fig) + + def _plot_line( + self, plot_data: PlotAlongData, key: str, shape: str, label: str, data_axis: str + ) -> None: + axis_times = plot_data[key][data_axis] + if not axis_times: + return + axis_noise = plot_data[key + "_noise"][data_axis] + series = sorted( + ( + ( + float(axis_value), + axis_times[axis_value], + axis_noise[axis_value], + ) + for axis_value in axis_times + ), + key=lambda item: item[0], + ) + x, y, noise = map(list, zip(*series, strict=True)) + + p = self.plt.plot(x, y, shape, marker="o", label=label) + + def plot_confidence_band(first: int, last: int) -> None: + if last - first < 2: + return + + band_x = x[first:last] + band_y = y[first:last] + band_noise = noise[first:last] + top = [band_y[i] + band_y[i] * band_noise[i] for i in range(len(band_x))] + bottom = [ + max( + band_y[i] - band_y[i] * band_noise[i], + band_y[i] * 0.001, + ) + for i in range(len(band_x)) + ] + self.plt.fill_between( + band_x, bottom, top, color=p[0].get_color(), alpha=0.1 + ) + + start = None + for i, noise_value in enumerate(noise): + if is_usable_noise(noise_value) and start is None: + start = i + if not is_usable_noise(noise_value) and start is not None: + plot_confidence_band(start, i) + start = None + + if start is not None: + plot_confidence_band(start, len(x)) + + def render_summary(self, global_axis_filters: Sequence[Mapping[str, Any]]) -> None: + if not self.plot_summary: + return + + title = "GPU timing change" + if len(self.comparison_device_names) == 1: + title = f"{title} - {next(iter(self.comparison_device_names))}" + if global_axis_filters: + axis_label = ", ".join( + axis_filter["display"] + for axis_filter in global_axis_filters + if len(axis_filter["values"]) == 1 + ) + if axis_label: + title = f"{title} ({axis_label})" + plot_output = self.plot_output + if self.comparison_entries: + plot_output = resolve_plot_output_path( + self.plot_output, + self.output_paths, + description="comparison plot", + ) + plot_comparison_entries( + self.comparison_entries, + title=title, + dark=self.dark, + output=plot_output, + tool_name=self.tool_name, + force_noninteractive_backend=( + self.force_noninteractive_backend + and not self.noninteractive_backend_selected + ), + ) diff --git a/python/scripts/nvbench_compare_robust.py b/python/scripts/nvbench_compare_robust.py index ebc60fc8..8b28a524 100644 --- a/python/scripts/nvbench_compare_robust.py +++ b/python/scripts/nvbench_compare_robust.py @@ -30,6 +30,7 @@ NumpyArray: TypeAlias = Any if __package__: + from . import _nvbench_compare_plotting as plotting from .nvbench_json import reader from .nvbench_tooling_deps import ( MissingToolingDependencyError, @@ -37,6 +38,7 @@ require_tooling_dependency, ) else: + import _nvbench_compare_plotting as plotting # type: ignore[no-redef] from nvbench_json import reader # type: ignore[no-redef] from nvbench_tooling_deps import ( # type: ignore[no-redef] MissingToolingDependencyError, @@ -1322,38 +1324,6 @@ def derive_absolute_dispersion(relative_dispersion, center): return None -def parse_plot_axis_value(axis_name, axis_value): - try: - value = float(axis_value) - except (TypeError, ValueError) as exc: - raise ValueError( - f"--plot-along requires numeric axis values; " - f"axis {axis_name!r} has value {axis_value!r}" - ) from exc - if not is_positive_finite(value): - raise ValueError( - f"--plot-along requires positive finite axis values; " - f"axis {axis_name!r} has value {axis_value!r}" - ) - return value - - -def extract_plot_axis_value(axis_values, plot_along, benchmark_name, state_name): - axis_name_parts = [] - for axis_value in axis_values: - if axis_value["name"] != plot_along: - axis_name_parts.append(f"""{axis_value["name"]} = {axis_value["value"]}""") - else: - return ( - parse_plot_axis_value(axis_value["name"], axis_value["value"]), - axis_name_parts, - ) - raise ValueError( - f"--plot-along axis {plot_along!r} is not present in " - f"benchmark {benchmark_name!r} state {state_name!r}" - ) - - def make_timing_interval(lower, upper, center): if ( not is_positive_finite(lower) @@ -2967,100 +2937,6 @@ def format_axis_values(axis_values, axes, axis_filters=None): return " ".join(parts) -def format_plot_series_key(state_key, occurrence, occurrence_count, axis_name_parts): - parts = [] - if state_key: - parts.append(state_key) - if occurrence_count > 1: - parts.append(f"occurrence={occurrence + 1}/{occurrence_count}") - parts.extend(axis_name_parts) - return ", ".join(parts) - - -def plot_comparison_entries(entries, title=None, dark=False): - if not entries: - print("No comparison data to plot.") - return 1 - - matplotlib = require_tooling_dependency( - ToolingDependency("matplotlib", "matplotlib", "plot rendering", extra="plot"), - tool_name=current_tool_name(), - ) - if not os.environ.get("DISPLAY"): - matplotlib.use("Agg") - - plt = require_tooling_dependency( - ToolingDependency( - "matplotlib.pyplot", "matplotlib", "plot rendering", extra="plot" - ), - tool_name=current_tool_name(), - ) - ticker = require_tooling_dependency( - ToolingDependency( - "matplotlib.ticker", "matplotlib", "plot axis formatting", extra="plot" - ), - tool_name=current_tool_name(), - ) - PercentFormatter = ticker.PercentFormatter - - labels, values, statuses, bench_names = map(list, zip(*entries)) - - status_colors = { - "SLOW": "red", - "FAST": "green", - "SAME": "blue", - } - colors = [status_colors.get(status, "gray") for status in statuses] - - fig_height = max(4.0, 0.3 * len(entries) + 1.5) - fig, ax = plt.subplots(figsize=(10, fig_height)) - if dark: - fig.patch.set_facecolor("black") - ax.set_facecolor("black") - ax.tick_params(colors="white") - ax.xaxis.label.set_color("white") - ax.yaxis.label.set_color("white") - ax.title.set_color("white") - for spine in ax.spines.values(): - spine.set_color("white") - - y_pos = range(len(labels)) - ax.barh(y_pos, values, color=colors) - ax.set_yticks(y_pos) - ax.set_yticklabels(labels) - ax.invert_yaxis() - ax.set_ylim(len(labels) - 0.5, -0.5) - - separator_color = "white" if dark else "gray" - ax.axvline(0, color=separator_color, linewidth=1, alpha=0.6) - for index in range(1, len(bench_names)): - if bench_names[index] != bench_names[index - 1]: - ax.axhline(index - 0.5, color=separator_color, linewidth=0.6, alpha=0.4) - ax.xaxis.set_major_formatter(PercentFormatter(1.0)) - - if title: - ax.set_title(title) - - min_val = min(values) - max_val = max(values) - if min_val == max_val: - pad = 0.05 if min_val == 0 else abs(min_val) * 0.1 - ax.set_xlim(min_val - pad, max_val + pad) - else: - pad = (max_val - min_val) * 0.1 - ax.set_xlim(min_val - pad, max_val + pad) - - fig.tight_layout() - - if not os.environ.get("DISPLAY"): - output = "nvbench_compare_robust.png" - fig.savefig(output, dpi=150) - print(f"Saved comparison plot to {output}") - else: - plt.show() - return 0 - - def compare_benches( run_data: ComparisonRunData, ref_benches, @@ -3080,31 +2956,24 @@ def compare_benches( comparison_thresholds=None, display="intervals", bulk_debug_rows=None, + plot_output=None, + plot_along_output=None, + plot_output_paths=None, ): if comparison_thresholds is None: comparison_thresholds = get_default_thresholds() - - if plot_along: - plt = require_tooling_dependency( - ToolingDependency( - "matplotlib.pyplot", - "matplotlib", - "per-axis plot rendering", - extra="plot", - ), - tool_name=current_tool_name(), - ) - sns = require_tooling_dependency( - ToolingDependency( - "seaborn", "seaborn", "per-axis plot styling", extra="plot" - ), - tool_name=current_tool_name(), - ) - - sns.set_theme() - - comparison_entries = [] - comparison_device_names = set() + if plot_output_paths is None: + plot_output_paths = set() + + plot_collector = plotting.PlotCollector( + plot_along=plot_along, + plot_summary=plot, + dark=dark, + plot_output=plot_output, + plot_along_output=plot_along_output, + output_paths=plot_output_paths, + tool_name=current_tool_name(), + ) for cmp_bench in cmp_benches: ref_bench = find_matching_bench(cmp_bench, ref_benches) if not ref_bench: @@ -3171,12 +3040,7 @@ def compare_benches( rows = [] row_comparisons = [] - plot_data: dict[str, dict[str, dict[float, float | None]]] = { - "cmp": {}, - "ref": {}, - "cmp_noise": {}, - "ref_noise": {}, - } + plot_data = plot_collector.make_plot_along_data() counters: dict[Any, int] = {} for cmp_state in cmp_device_states: @@ -3238,31 +3102,18 @@ def compare_benches( if comparison is None: continue - if ( - plot_along - and is_positive_finite(comparison.ref_time) - and is_positive_finite(comparison.cmp_time) - ): - axis_value, axis_name_parts = extract_plot_axis_value( - axis_values, plot_along, cmp_bench["name"], cmp_state_name - ) - axis_name = format_plot_series_key( - cmp_state_name, - occurrence, - cmp_state_counts[cmp_state_key], - axis_name_parts, - ) - - if axis_name not in plot_data["cmp"]: - plot_data["cmp"][axis_name] = {} - plot_data["ref"][axis_name] = {} - plot_data["cmp_noise"][axis_name] = {} - plot_data["ref_noise"][axis_name] = {} - - plot_data["cmp"][axis_name][axis_value] = comparison.cmp_time - plot_data["ref"][axis_name][axis_value] = comparison.ref_time - plot_data["cmp_noise"][axis_name][axis_value] = comparison.cmp_noise - plot_data["ref_noise"][axis_name][axis_value] = comparison.ref_noise + plot_collector.record_plot_along( + plot_data, + ref_time=comparison.ref_time, + cmp_time=comparison.cmp_time, + ref_noise=comparison.ref_noise, + cmp_noise=comparison.cmp_noise, + axis_values=axis_values, + benchmark_name=cmp_bench["name"], + state_name=cmp_state_name, + occurrence=occurrence, + occurrence_count=cmp_state_counts[cmp_state_key], + ) run_data.stats.record(comparison.status, comparison.reason) if comparison.status == ComparisonStatus.UNKNOWN or ( @@ -3294,34 +3145,21 @@ def compare_benches( comparison=comparison, ) ) - if ( - plot - and comparison.frac_diff is not None - and math.isfinite(comparison.frac_diff) - ): + if plot: axis_label = format_axis_values(axis_values, axes, axis_filters) - if axis_label: - label = f"""{cmp_bench["name"]} | {axis_label}""" - else: - label = cmp_bench["name"] cmp_device = find_device_by_id( cmp_state["device"], run_data.cmp_devices ) - if cmp_device: - comparison_device_names.add(cmp_device["name"]) - comparison_entries.append( - ( - label, - comparison.frac_diff, - comparison.status.value, - cmp_bench["name"], - ) + plot_collector.record_summary_entry( + benchmark_name=cmp_bench["name"], + axis_label=axis_label, + cmp_device_name=cmp_device["name"] if cmp_device else None, + frac_diff=comparison.frac_diff, + status=comparison.status.value, ) has_rows = len(rows) > 0 - has_plot_along_data = bool(plot_along) and any( - axis_times for axis_times in plot_data["cmp"].values() - ) + has_plot_along_data = plot_collector.has_plot_along_data(plot_data) cmp_device = find_device_by_id(cmp_device_id, run_data.cmp_devices) ref_device = find_device_by_id(ref_device_id, run_data.ref_devices) @@ -3361,89 +3199,15 @@ def compare_benches( print("") if has_plot_along_data: - fig = plt.figure() - try: - plt.xscale("log") - plt.yscale("log") - plt.xlabel(plot_along) - plt.ylabel("time [s]") - plt.title(cmp_device["name"]) - - def plot_line(key, shape, label, data_axis, data=plot_data): - axis_times = data[key][data_axis] - if not axis_times: - return - axis_noise = data[key + "_noise"][data_axis] - series = sorted( - ( - ( - float(axis_value), - axis_times[axis_value], - axis_noise[axis_value], - ) - for axis_value in axis_times - ), - key=lambda item: item[0], - ) - x, y, noise = map(list, zip(*series, strict=True)) - - p = plt.plot(x, y, shape, marker="o", label=label) - - def plot_confidence_band(first, last): - if last - first < 2: - return - - band_x = x[first:last] - band_y = y[first:last] - band_noise = noise[first:last] - top = [ - band_y[i] + band_y[i] * band_noise[i] - for i in range(len(band_x)) - ] - bottom = [ - max( - band_y[i] - band_y[i] * band_noise[i], - band_y[i] * 0.001, - ) - for i in range(len(band_x)) - ] - plt.fill_between( - band_x, bottom, top, color=p[0].get_color(), alpha=0.1 - ) + plot_collector.render_plot_along( + plot_data, + benchmark_name=cmp_bench["name"], + cmp_device_id=cmp_device_id, + cmp_device_index=cmp_device_index, + cmp_device_name=cmp_device["name"], + ) - start = None - for i, noise_value in enumerate(noise): - if is_usable_noise(noise_value) and start is None: - start = i - if not is_usable_noise(noise_value) and start is not None: - plot_confidence_band(start, i) - start = None - - if start is not None: - plot_confidence_band(start, len(x)) - - for axis in plot_data["cmp"].keys(): - plot_line("cmp", "-", axis, axis) - plot_line("ref", "--", axis + " ref", axis) - - plt.legend() - plt.show() - finally: - plt.close(fig) - - if plot: - title = "GPU timing change" - if len(comparison_device_names) == 1: - title = f"{title} - {next(iter(comparison_device_names))}" - if filter_plan.global_axis_filters: - axis_label = ", ".join( - axis_filter["display"] - for axis_filter in filter_plan.global_axis_filters - if len(axis_filter["values"]) == 1 - ) - if axis_label: - title = f"{title} ({axis_label})" - plot_comparison_entries(comparison_entries, title=title, dark=dark) + plot_collector.render_summary(filter_plan.global_axis_filters) def main() -> int: @@ -3510,6 +3274,19 @@ def main() -> int: help="plot comparison summary", action="store_true", ) + parser.add_argument( + "--plot-output", + default=None, + help="save --plot output to this path instead of showing it interactively", + ) + parser.add_argument( + "--plot-along-output", + default=None, + help=( + "save --plot-along output to this path or filename template instead " + "of showing it interactively" + ), + ) parser.add_argument( "--dark", action="store_true", @@ -3570,6 +3347,35 @@ def main() -> int: if args.dump_config: print(dump_comparison_config(comparison_preset, comparison_thresholds), end="") return 0 + if args.plot_output is not None and not args.plot: + print("--plot-output requires --plot") + return 1 + if args.plot_along_output is not None and args.plot_along is None: + print("--plot-along-output requires --plot-along") + return 1 + try: + plotting.validate_plot_along_axis_name(args.plot_along) + except ValueError as exc: + print(str(exc)) + return 1 + if args.plot_along_output is not None: + try: + plotting.validate_plot_along_output_template(args.plot_along_output) + except ValueError as exc: + print(str(exc)) + return 1 + # Validate here for early CLI errors; PlotCollector repeats this for direct + # compare_benches() callers. + try: + plotting.validate_plot_output_modes( + plot=args.plot, + plot_output=args.plot_output, + plot_along=args.plot_along, + plot_along_output=args.plot_along_output, + ) + except ValueError as exc: + print(str(exc)) + return 1 try: filter_plan = build_benchmark_filter_plan(args.filter_actions) @@ -3587,25 +3393,11 @@ def main() -> int: parser.print_help() return 1 - try: - load_nvbench_compare_tooling(load_color=not args.no_color) - except MissingToolingDependencyError as exc: - print(str(exc), file=sys.stderr) - return 1 - - bulk_debug_output = ( - None - if args.bulk_debug_python is None - else BulkDebugOutput(args.bulk_debug_python) - ) - bulk_debug_rows: list[dict[str, Any]] | None = ( - [] if bulk_debug_output is not None else None - ) - - # if provided two directories, find all the exactly named files - # in both and treat them as the reference and compare + input_dirs = tuple(os.path.isdir(path) for path in files_or_dirs) to_compare = [] - if os.path.isdir(files_or_dirs[0]) and os.path.isdir(files_or_dirs[1]): + # If provided two directories, find all the exactly named files + # in both and treat them as the reference and compare. + if all(input_dirs): for f in os.listdir(files_or_dirs[1]): if os.path.splitext(f)[1] != ".json": continue @@ -3627,7 +3419,23 @@ def main() -> int: ) return 1 + try: + load_nvbench_compare_tooling(load_color=not args.no_color) + except MissingToolingDependencyError as exc: + print(str(exc), file=sys.stderr) + return 1 + + bulk_debug_output = ( + None + if args.bulk_debug_python is None + else BulkDebugOutput(args.bulk_debug_python) + ) + bulk_debug_rows: list[dict[str, Any]] | None = ( + [] if bulk_debug_output is not None else None + ) + stats = ComparisonStats() + plot_output_paths: set[str] = set() for ref, comp in to_compare: try: @@ -3704,6 +3512,9 @@ def main() -> int: comparison_thresholds=comparison_thresholds, display=args.display, bulk_debug_rows=bulk_debug_rows, + plot_output=args.plot_output, + plot_along_output=args.plot_along_output, + plot_output_paths=plot_output_paths, ) except MissingToolingDependencyError as exc: print(str(exc), file=sys.stderr) diff --git a/python/test/test_nvbench_compare_robust.py b/python/test/test_nvbench_compare_robust.py index b2c2ce69..b6d39a4c 100644 --- a/python/test/test_nvbench_compare_robust.py +++ b/python/test/test_nvbench_compare_robust.py @@ -20,8 +20,86 @@ class DummyLine: def get_color(self): return "black" + class DummyText: + def set_color(self, *args, **kwargs): + pass + + class DummyAxis: + def __init__(self): + self.label = DummyText() + + def set_major_formatter(self, *args, **kwargs): + pass + + class DummySpine: + def set_color(self, *args, **kwargs): + pass + + class DummyPatch: + def set_facecolor(self, *args, **kwargs): + pass + + class DummyAxes: + def __init__(self): + self.xaxis = DummyAxis() + self.yaxis = DummyAxis() + self.title = DummyText() + self.spines = { + "left": DummySpine(), + "right": DummySpine(), + "top": DummySpine(), + "bottom": DummySpine(), + } + + def set_facecolor(self, *args, **kwargs): + pass + + def tick_params(self, *args, **kwargs): + pass + + def barh(self, *args, **kwargs): + pass + + def set_yticks(self, *args, **kwargs): + pass + + def set_yticklabels(self, *args, **kwargs): + pass + + def invert_yaxis(self, *args, **kwargs): + pass + + def set_ylim(self, *args, **kwargs): + pass + + def axvline(self, *args, **kwargs): + pass + + def axhline(self, *args, **kwargs): + pass + + def set_title(self, *args, **kwargs): + pass + + def set_xlim(self, *args, **kwargs): + pass + + class DummyFigure: + def __init__(self): + self.patch = DummyPatch() + + def tight_layout(self, *args, **kwargs): + pass + + def savefig(self, *args, **kwargs): + pyplot.savefig_calls.append({"args": args, "kwargs": kwargs}) + pyplot = types.ModuleType("matplotlib.pyplot") - pyplot.figure = lambda *args, **kwargs: None + pyplot.close_calls = [] + pyplot.savefig_calls = [] + pyplot.show_calls = [] + pyplot.figure = lambda *args, **kwargs: DummyFigure() + pyplot.subplots = lambda *args, **kwargs: (DummyFigure(), DummyAxes()) pyplot.xscale = lambda *args, **kwargs: None pyplot.yscale = lambda *args, **kwargs: None pyplot.xlabel = lambda *args, **kwargs: None @@ -30,13 +108,20 @@ def get_color(self): pyplot.plot = lambda *args, **kwargs: [DummyLine()] pyplot.fill_between = lambda *args, **kwargs: None pyplot.legend = lambda *args, **kwargs: None - pyplot.show = lambda *args, **kwargs: None - pyplot.close = lambda *args, **kwargs: None + pyplot.show = lambda *args, **kwargs: pyplot.show_calls.append((args, kwargs)) + pyplot.close = lambda *args, **kwargs: pyplot.close_calls.append((args, kwargs)) matplotlib = types.ModuleType("matplotlib") + matplotlib.use_calls = [] + matplotlib.use = lambda *args, **kwargs: matplotlib.use_calls.append((args, kwargs)) matplotlib.pyplot = pyplot monkeypatch.setitem(sys.modules, "matplotlib", matplotlib) monkeypatch.setitem(sys.modules, "matplotlib.pyplot", pyplot) + monkeypatch.setitem( + sys.modules, + "matplotlib.ticker", + types.SimpleNamespace(PercentFormatter=lambda *args, **kwargs: object()), + ) monkeypatch.setitem( sys.modules, "seaborn", @@ -90,6 +175,7 @@ def test_nvbench_compare_imports_from_packaged_script_path(tmp_path, monkeypatch for package in [tmp_path / "cuda", tmp_path / "cuda" / "bench", package_dir]: (package / "__init__.py").write_text("", encoding="utf-8") for filename in [ + "_nvbench_compare_plotting.py", "nvbench_compare_robust.py", "nvbench_tooling_deps.py", ]: @@ -101,6 +187,7 @@ def test_nvbench_compare_imports_from_packaged_script_path(tmp_path, monkeypatch "cuda", "cuda.bench", "cuda.bench.scripts", + "cuda.bench.scripts._nvbench_compare_plotting", "cuda.bench.scripts.nvbench_compare_robust", "cuda.bench.scripts.nvbench_json", "cuda.bench.scripts.nvbench_tooling_deps", @@ -2219,7 +2306,9 @@ def test_compare_benches_marks_unavailable_noise_undecided( def test_plot_along_rejects_states_without_selected_axis(monkeypatch, nvbench_compare): run_data = make_comparison_run_data(nvbench_compare) monkeypatch.setattr( - nvbench_compare, "plot_comparison_entries", lambda *args, **kwargs: None + nvbench_compare.plotting, + "plot_comparison_entries", + lambda *args, **kwargs: None, ) ref_benches = [ @@ -2321,6 +2410,205 @@ def fake_plot(x, y, shape, *args, **kwargs): assert [call["shape"] for call in plot_calls] == ["-", "--"] +def test_plot_along_output_saves_without_showing(tmp_path, nvbench_compare): + run_data = make_comparison_run_data(nvbench_compare) + pyplot = sys.modules["matplotlib.pyplot"] + output = tmp_path / "plots" / "bench-device0-A.png" + + nvbench_compare.compare_benches( + run_data, + [make_benchmark([make_state(nvbench_compare, "state", axis_value=1)])], + [make_benchmark([make_state(nvbench_compare, "state", axis_value=1)])], + threshold=0.0, + plot_along="A", + plot=False, + dark=False, + filter_plan=make_filter_plan(nvbench_compare), + no_color=True, + plot_along_output=str(output), + ) + + assert [call["args"][0] for call in pyplot.savefig_calls] == [str(output)] + assert output.parent.is_dir() + assert pyplot.show_calls == [] + + +def test_plot_outputs_force_agg_once_when_both_plot_modes_save( + tmp_path, nvbench_compare +): + run_data = make_comparison_run_data(nvbench_compare) + matplotlib = sys.modules["matplotlib"] + pyplot = sys.modules["matplotlib.pyplot"] + plot_output = tmp_path / "compare.png" + plot_along_output = tmp_path / "plot-along.png" + + nvbench_compare.compare_benches( + run_data, + [make_benchmark([make_state(nvbench_compare, "state", axis_value=1)])], + [make_benchmark([make_state(nvbench_compare, "state", axis_value=1)])], + threshold=0.0, + plot_along="A", + plot=True, + dark=False, + filter_plan=make_filter_plan(nvbench_compare), + no_color=True, + plot_output=str(plot_output), + plot_along_output=str(plot_along_output), + ) + + assert matplotlib.use_calls == [(("Agg",), {})] + assert [call["args"][0] for call in pyplot.savefig_calls] == [ + str(plot_along_output), + str(plot_output), + ] + assert pyplot.show_calls == [] + + +def test_compare_benches_rejects_empty_plot_along_axis(nvbench_compare): + run_data = make_comparison_run_data(nvbench_compare) + + with pytest.raises(ValueError, match="--plot-along requires a non-empty axis name"): + nvbench_compare.compare_benches( + run_data, + [make_benchmark([make_state(nvbench_compare, "state", axis_value=1)])], + [make_benchmark([make_state(nvbench_compare, "state", axis_value=1)])], + threshold=0.0, + plot_along="", + plot=False, + dark=False, + filter_plan=make_filter_plan(nvbench_compare), + no_color=True, + ) + + +def test_plot_along_output_template_expands_per_plot(tmp_path, nvbench_compare): + run_data = make_comparison_run_data(nvbench_compare) + pyplot = sys.modules["matplotlib.pyplot"] + output_template = str(tmp_path / "{benchmark}-device{device}-{axis}.png") + + nvbench_compare.compare_benches( + run_data, + [ + make_benchmark( + [make_state(nvbench_compare, "state", axis_value=1)], name="bench1" + ), + make_benchmark( + [make_state(nvbench_compare, "state", axis_value=1)], name="bench2" + ), + ], + [ + make_benchmark( + [make_state(nvbench_compare, "state", axis_value=1)], name="bench1" + ), + make_benchmark( + [make_state(nvbench_compare, "state", axis_value=1)], name="bench2" + ), + ], + threshold=0.0, + plot_along="A", + plot=False, + dark=False, + filter_plan=make_filter_plan(nvbench_compare), + no_color=True, + plot_along_output=output_template, + ) + + assert [call["args"][0] for call in pyplot.savefig_calls] == [ + str(tmp_path / "bench1-device0-A.png"), + str(tmp_path / "bench2-device0-A.png"), + ] + assert pyplot.show_calls == [] + + +def test_plot_along_output_template_pair_disambiguates_repeated_compare_devices( + tmp_path, nvbench_compare +): + ref_devices = [{"id": 0, "name": "GPU 0"}, {"id": 1, "name": "GPU 1"}] + cmp_devices = [{"id": 0, "name": "GPU 0"}] + run_data = make_comparison_run_data( + nvbench_compare, ref_devices=ref_devices, cmp_devices=cmp_devices + ) + pyplot = sys.modules["matplotlib.pyplot"] + output_template = str(tmp_path / "{benchmark}-pair{pair}-device{device}-{axis}.png") + + ref_bench = make_benchmark( + [ + make_state(nvbench_compare, "state", axis_value=1, device=0), + make_state(nvbench_compare, "state", axis_value=1, device=1), + ] + ) + cmp_bench = make_benchmark( + [ + make_state(nvbench_compare, "state", axis_value=1, device=0), + ] + ) + cmp_bench["devices"] = [0, 0] + + nvbench_compare.compare_benches( + run_data, + [ref_bench], + [cmp_bench], + threshold=0.0, + plot_along="A", + plot=False, + dark=False, + filter_plan=make_filter_plan(nvbench_compare), + no_color=True, + reference_device_filter=[0, 1], + compare_device_filter=[0, 0], + plot_along_output=output_template, + ) + + assert [call["args"][0] for call in pyplot.savefig_calls] == [ + str(tmp_path / "bench-pair0-device0-A.png"), + str(tmp_path / "bench-pair1-device0-A.png"), + ] + + +def test_plot_along_output_disambiguates_duplicate_paths( + tmp_path, capsys, nvbench_compare +): + run_data = make_comparison_run_data(nvbench_compare) + pyplot = sys.modules["matplotlib.pyplot"] + + nvbench_compare.compare_benches( + run_data, + [ + make_benchmark( + [make_state(nvbench_compare, "state", axis_value=1)], + name="bench1", + ), + make_benchmark( + [make_state(nvbench_compare, "state", axis_value=1)], + name="bench2", + ), + ], + [ + make_benchmark( + [make_state(nvbench_compare, "state", axis_value=1)], + name="bench1", + ), + make_benchmark( + [make_state(nvbench_compare, "state", axis_value=1)], + name="bench2", + ), + ], + threshold=0.0, + plot_along="A", + plot=False, + dark=False, + filter_plan=make_filter_plan(nvbench_compare), + no_color=True, + plot_along_output=str(tmp_path / "plot.png"), + ) + + assert [call["args"][0] for call in pyplot.savefig_calls] == [ + str(tmp_path / "plot.png"), + str(tmp_path / "plot-copy-1.png"), + ] + assert "Warning: plot-along output" in capsys.readouterr().err + + def test_compare_benches_validates_device_metadata_when_threshold_hides_rows( nvbench_compare, ): @@ -3437,7 +3725,9 @@ def fake_plot_comparison_entries(entries, *args, **kwargs): return 0 monkeypatch.setattr( - nvbench_compare, "plot_comparison_entries", fake_plot_comparison_entries + nvbench_compare.plotting, + "plot_comparison_entries", + fake_plot_comparison_entries, ) run_data = make_comparison_run_data(nvbench_compare) @@ -3557,7 +3847,7 @@ def test_compare_benches_summary_plot_title_describes_timing( plot_calls = [] monkeypatch.setattr( - nvbench_compare, + nvbench_compare.plotting, "plot_comparison_entries", lambda *args, **kwargs: plot_calls.append({"args": args, "kwargs": kwargs}), ) @@ -3577,6 +3867,130 @@ def test_compare_benches_summary_plot_title_describes_timing( assert plot_calls[0]["kwargs"]["title"] == "GPU timing change - Test GPU" +def test_plot_comparison_entries_shows_without_output(nvbench_compare): + pyplot = sys.modules["matplotlib.pyplot"] + + assert ( + nvbench_compare.plotting.plot_comparison_entries( + [("bench", 0.01, "FAST", "bench")], title="GPU timing change" + ) + == 0 + ) + + assert len(pyplot.show_calls) == 1 + assert pyplot.savefig_calls == [] + assert len(pyplot.close_calls) == 1 + + +def test_plot_comparison_entries_saves_without_showing(tmp_path, nvbench_compare): + pyplot = sys.modules["matplotlib.pyplot"] + output = tmp_path / "plots" / "compare.png" + + assert ( + nvbench_compare.plotting.plot_comparison_entries( + [("bench", 0.01, "FAST", "bench")], + title="GPU timing change", + output=str(output), + ) + == 0 + ) + + assert [call["args"][0] for call in pyplot.savefig_calls] == [str(output)] + assert pyplot.savefig_calls[0]["kwargs"]["dpi"] == 150 + assert output.parent.is_dir() + assert pyplot.show_calls == [] + assert len(pyplot.close_calls) == 1 + + +def test_save_or_show_plot_creates_parent_directories(tmp_path, nvbench_compare): + class DummyFigure: + def savefig(self, *args, **kwargs): + pass + + pyplot = sys.modules["matplotlib.pyplot"] + output = tmp_path / "plots" / "nested" / "compare.png" + + nvbench_compare.plotting.save_or_show_plot( + DummyFigure(), pyplot, str(output), "comparison plot" + ) + + assert output.parent.is_dir() + + +def test_resolve_plot_output_path_disambiguates_existing_files( + tmp_path, capsys, nvbench_compare +): + output = tmp_path / "compare.png" + output.write_text("", encoding="utf-8") + (tmp_path / "compare-copy-1.png").write_text("", encoding="utf-8") + + resolved = nvbench_compare.plotting.resolve_plot_output_path( + str(output), set(), description="comparison plot" + ) + + assert resolved == str(tmp_path / "compare-copy-2.png") + warning = capsys.readouterr().err + assert "Warning: comparison plot output" in warning + assert str(output) in warning + assert str(tmp_path / "compare-copy-2.png") in warning + + +def test_save_or_show_plot_reports_parent_directory_errors(tmp_path, nvbench_compare): + class DummyFigure: + def savefig(self, *args, **kwargs): + raise AssertionError("savefig should not be called") + + pyplot = sys.modules["matplotlib.pyplot"] + output_parent = tmp_path / "not-a-directory" + output_parent.write_text("", encoding="utf-8") + output = output_parent / "compare.png" + + output_text = str(output) + with pytest.raises(ValueError) as exc_info: + nvbench_compare.plotting.save_or_show_plot( + DummyFigure(), pyplot, output_text, "comparison plot" + ) + assert "failed to write comparison plot" in str(exc_info.value) + assert output_text in str(exc_info.value) + + +def test_save_or_show_plot_reports_save_errors(tmp_path, nvbench_compare): + class DummyFigure: + def savefig(self, *args, **kwargs): + raise OSError("disk full") + + pyplot = sys.modules["matplotlib.pyplot"] + output = tmp_path / "compare.png" + + output_text = str(output) + with pytest.raises(ValueError) as exc_info: + nvbench_compare.plotting.save_or_show_plot( + DummyFigure(), pyplot, output_text, "comparison plot" + ) + assert "failed to write comparison plot" in str(exc_info.value) + assert output_text in str(exc_info.value) + + +def test_save_or_show_plot_reports_unsupported_output_formats( + tmp_path, nvbench_compare +): + class DummyFigure: + def savefig(self, *args, **kwargs): + raise ValueError("unsupported format") + + pyplot = sys.modules["matplotlib.pyplot"] + output = tmp_path / "compare.unsupported" + + output_text = str(output) + with pytest.raises(ValueError) as exc_info: + nvbench_compare.plotting.save_or_show_plot( + DummyFigure(), pyplot, output_text, "comparison plot" + ) + assert "failed to write comparison plot" in str(exc_info.value) + assert output_text in str(exc_info.value) + assert "unsupported format" in str(exc_info.value) + + def test_compare_benches_explain_display_uses_explicit_intervals( monkeypatch, nvbench_compare ): @@ -3663,6 +4077,390 @@ def fake_compare_benches(*args, **kwargs): assert captured["display"] == "explain" +def test_main_passes_plot_output_options_to_compare_benches( + monkeypatch, nvbench_compare +): + devices = [{"id": 0, "name": "Test GPU"}] + root = { + "devices": devices, + "benchmarks": [], + } + captured = {} + + monkeypatch.setattr(nvbench_compare.reader, "read_file", lambda _: root) + + def fake_compare_benches(*args, **kwargs): + del args + captured["plot_output"] = kwargs["plot_output"] + captured["plot_along_output"] = kwargs["plot_along_output"] + + monkeypatch.setattr(nvbench_compare, "compare_benches", fake_compare_benches) + monkeypatch.setattr( + sys, + "argv", + [ + "nvbench_compare", + "--plot", + "--plot-output", + "compare.png", + "--plot-along", + "A", + "--plot-along-output", + "plots/{benchmark}-{axis}.png", + "ref.json", + "cmp.json", + ], + ) + + assert nvbench_compare.main() == 0 + assert captured["plot_output"] == "compare.png" + assert captured["plot_along_output"] == "plots/{benchmark}-{axis}.png" + + +def test_main_rejects_invalid_plot_along_output_template_before_comparing( + monkeypatch, capsys, nvbench_compare +): + def fail_compare_benches(*args, **kwargs): + del args, kwargs + raise AssertionError("compare_benches should not be called") + + monkeypatch.setattr(nvbench_compare, "compare_benches", fail_compare_benches) + monkeypatch.setattr( + sys, + "argv", + [ + "nvbench_compare", + "--plot-along", + "A", + "--plot-along-output", + "plots/{benchmark.missing}.png", + "ref.json", + "cmp.json", + ], + ) + + assert nvbench_compare.main() == 1 + assert "--plot-along-output supports template fields" in capsys.readouterr().out + + +def test_main_rejects_empty_plot_along_axis_before_comparing( + monkeypatch, capsys, nvbench_compare +): + def fail_compare_benches(*args, **kwargs): + del args, kwargs + raise AssertionError("compare_benches should not be called") + + monkeypatch.setattr(nvbench_compare, "compare_benches", fail_compare_benches) + monkeypatch.setattr( + sys, + "argv", + [ + "nvbench_compare", + "--plot-along", + "", + "ref.json", + "cmp.json", + ], + ) + + assert nvbench_compare.main() == 1 + assert "--plot-along requires a non-empty axis name" in capsys.readouterr().out + + +def test_main_rejects_mixed_plot_output_modes_when_summary_would_be_interactive( + monkeypatch, capsys, nvbench_compare +): + monkeypatch.setattr( + sys, + "argv", + [ + "nvbench_compare", + "--plot", + "--plot-along", + "A", + "--plot-along-output", + "plot-along.png", + "ref.json", + "cmp.json", + ], + ) + + assert nvbench_compare.main() == 1 + output = capsys.readouterr().out + assert "Cannot mix interactive and file-based plot output" in output + assert "Add --plot-output" in output + assert "omit --plot-along-output" in output + + +def test_main_rejects_mixed_plot_output_modes_when_plot_along_would_be_interactive( + monkeypatch, capsys, nvbench_compare +): + monkeypatch.setattr( + sys, + "argv", + [ + "nvbench_compare", + "--plot", + "--plot-output", + "compare.png", + "--plot-along", + "A", + "ref.json", + "cmp.json", + ], + ) + + assert nvbench_compare.main() == 1 + output = capsys.readouterr().out + assert "Cannot mix interactive and file-based plot output" in output + assert "Add --plot-along-output" in output + assert "omit --plot-output" in output + + +def make_directory_compare_inputs(tmp_path, nvbench_compare, filenames=None): + if filenames is None: + filenames = ["a.json", "b.json"] + ref_dir = tmp_path / "ref" + cmp_dir = tmp_path / "cmp" + ref_dir.mkdir() + cmp_dir.mkdir() + for filename in filenames: + (ref_dir / filename).write_text("{}", encoding="utf-8") + (cmp_dir / filename).write_text("{}", encoding="utf-8") + + devices = [{"id": 0, "name": "Test GPU"}] + + def make_timed_state(mean): + state = make_state(nvbench_compare, "state", mean=mean, axis_value=1) + state["summaries"].append( + make_summary(nvbench_compare, "GPU_SM_CLOCK_RATE_MEAN_TAG", "1.0") + ) + return state + + roots = {} + for filename in filenames: + benchmark_name = f"bench_{Path(filename).stem}" + roots[ref_dir / filename] = { + "devices": devices, + "benchmarks": [ + make_benchmark( + [make_timed_state("1.0")], + name=benchmark_name, + ) + ], + } + roots[cmp_dir / filename] = { + "devices": devices, + "benchmarks": [ + make_benchmark( + [make_timed_state("2.0")], + name=benchmark_name, + ) + ], + } + + def read_file(path): + try: + return roots[Path(path)] + except KeyError: + raise AssertionError(f"unexpected path: {path!r}") from None + + return ref_dir, cmp_dir, read_file + + +def test_main_disambiguates_plot_output_with_directory_inputs( + tmp_path, monkeypatch, capsys, nvbench_compare +): + ref_dir, cmp_dir, read_file = make_directory_compare_inputs( + tmp_path, nvbench_compare + ) + output = tmp_path / "compare.png" + pyplot = sys.modules["matplotlib.pyplot"] + + monkeypatch.setattr(nvbench_compare.reader, "read_file", read_file) + monkeypatch.setattr( + sys, + "argv", + [ + "nvbench_compare", + "--plot", + "--plot-output", + str(output), + str(ref_dir), + str(cmp_dir), + ], + ) + + assert nvbench_compare.main() == 0 + assert [call["args"][0] for call in pyplot.savefig_calls] == [ + str(output), + str(tmp_path / "compare-copy-1.png"), + ] + captured = capsys.readouterr() + assert "Warning: comparison plot output" in captured.err + assert " - Improvement (clear timing gap, %Diff < 0): 0" in captured.out + assert " - Regression (clear timing gap, %Diff > 0): 2" in captured.out + + +def test_main_allows_plot_output_when_directory_inputs_have_one_matching_json( + tmp_path, monkeypatch, capsys, nvbench_compare +): + ref_dir, cmp_dir, read_file = make_directory_compare_inputs( + tmp_path, nvbench_compare, filenames=["only.json"] + ) + output = tmp_path / "compare.png" + pyplot = sys.modules["matplotlib.pyplot"] + + monkeypatch.setattr(nvbench_compare.reader, "read_file", read_file) + monkeypatch.setattr( + sys, + "argv", + [ + "nvbench_compare", + "--plot", + "--plot-output", + str(output), + str(ref_dir), + str(cmp_dir), + ], + ) + + assert nvbench_compare.main() == 0 + assert [call["args"][0] for call in pyplot.savefig_calls] == [str(output)] + captured = capsys.readouterr() + assert " - Improvement (clear timing gap, %Diff < 0): 0" in captured.out + assert " - Regression (clear timing gap, %Diff > 0): 1" in captured.out + + +def test_main_disambiguates_plot_along_output_with_directory_inputs( + tmp_path, monkeypatch, capsys, nvbench_compare +): + ref_dir, cmp_dir, read_file = make_directory_compare_inputs( + tmp_path, nvbench_compare + ) + output = tmp_path / "plot-along.png" + pyplot = sys.modules["matplotlib.pyplot"] + + monkeypatch.setattr(nvbench_compare.reader, "read_file", read_file) + monkeypatch.setattr( + sys, + "argv", + [ + "nvbench_compare", + "--plot-along", + "A", + "--plot-along-output", + str(output), + str(ref_dir), + str(cmp_dir), + ], + ) + + assert nvbench_compare.main() == 0 + assert [call["args"][0] for call in pyplot.savefig_calls] == [ + str(output), + str(tmp_path / "plot-along-copy-1.png"), + ] + captured = capsys.readouterr() + assert "Warning: plot-along output" in captured.err + assert " - Improvement (clear timing gap, %Diff < 0): 0" in captured.out + assert " - Regression (clear timing gap, %Diff > 0): 2" in captured.out + + +def test_main_writes_bulk_debug_python_for_directory_inputs( + tmp_path, monkeypatch, nvbench_compare +): + ref_dir, cmp_dir, read_file = make_directory_compare_inputs( + tmp_path, nvbench_compare + ) + output = tmp_path / "bulk_debug.py" + + monkeypatch.setattr(nvbench_compare.reader, "read_file", read_file) + monkeypatch.setattr( + sys, + "argv", + [ + "nvbench_compare", + "--bulk-debug-python", + str(output), + str(ref_dir), + str(cmp_dir), + ], + ) + + assert nvbench_compare.main() == 0 + script = output.read_text(encoding="utf-8") + assert "# NVB-BULK-BEGIN" in script + assert script.count("'reference_json':") == 2 + assert str(ref_dir / "a.json") in script + assert str(ref_dir / "b.json") in script + + +@pytest.mark.parametrize( + "output_template", + [ + "plots/{benchmark}-{device}-{axis}.png", + "plots/{{literal}}-{axis}.png", + "plots/{benchmark}-pair{pair}-{axis}.png", + ], +) +def test_validate_plot_along_output_template_accepts_supported_fields( + nvbench_compare, output_template +): + nvbench_compare.plotting.validate_plot_along_output_template(output_template) + + +@pytest.mark.parametrize( + "output_template", + [ + "plots/{benchmark.missing}.png", + "plots/{benchmark[0]}.png", + "plots/{benchmark!r}.png", + "plots/{benchmark:s}.png", + "plots/{}.png", + "plots/{unknown}.png", + ], +) +def test_validate_plot_along_output_template_rejects_unsupported_fields( + nvbench_compare, output_template +): + with pytest.raises(ValueError, match=r"--plot-along-output supports"): + nvbench_compare.plotting.validate_plot_along_output_template(output_template) + + +def test_format_plot_along_output_path_sanitizes_template_fields( + tmp_path, nvbench_compare +): + output = nvbench_compare.plotting.format_plot_along_output_path( + str(tmp_path / "plots" / "{benchmark}-device{device}-{axis}.png"), + benchmark_name="../../workspace/bench{name}", + device_id=0, + axis_name="Elements{io}/../Time", + ) + + assert output == str( + tmp_path / "plots" / "workspace_bench_name-device0-Elements_io_Time.png" + ) + + +def test_format_plot_along_output_path_formats_pair_field(tmp_path, nvbench_compare): + output = nvbench_compare.plotting.format_plot_along_output_path( + str(tmp_path / "plots" / "{benchmark}-pair{pair}-{axis}.png"), + benchmark_name="bench", + device_id=0, + axis_name="A", + device_pair_index=3, + ) + + assert output == str(tmp_path / "plots" / "bench-pair3-A.png") + + +def test_sanitize_plot_output_component_uses_fallback_for_empty_values( + nvbench_compare, +): + assert nvbench_compare.plotting.sanitize_plot_output_component("../../") == "value" + + def test_main_converts_threshold_diff_percent_to_fraction(monkeypatch, nvbench_compare): devices = [{"id": 0, "name": "Test GPU"}] root = { diff --git a/python/test/test_nvbench_tooling_deps.py b/python/test/test_nvbench_tooling_deps.py index 39468955..8769ca9e 100644 --- a/python/test/test_nvbench_tooling_deps.py +++ b/python/test/test_nvbench_tooling_deps.py @@ -11,6 +11,7 @@ import pytest SCRIPT_SOURCE_FILES = [ + "_nvbench_compare_plotting.py", "nvbench_compare.py", "nvbench_compare_robust.py", "nvbench_histogram.py",