From 2ede07696bd8ac8f4cf2d72e93a1680c28eb26ff Mon Sep 17 00:00:00 2001 From: arose26 Date: Tue, 18 Aug 2026 04:51:30 -0400 Subject: [PATCH 1/3] Return nan instead of raising when ignore_index removes every sample Curve-based classification metrics indexed an empty tensor once ignore_index filtered out every sample of a class, which is reached in normal use when one label of a multilabel target carries no valid entries in a batch. --- CHANGELOG.md | 3 ++ .../classification/average_precision.py | 14 ++++++++- .../classification/precision_recall_curve.py | 16 ++++++++++ .../functional/classification/roc.py | 6 ++++ src/torchmetrics/utilities/compute.py | 8 +++++ tests/unittests/classification/test_auroc.py | 30 +++++++++++++++++++ .../classification/test_average_precision.py | 26 ++++++++++++++++ 7 files changed, 102 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9518750113a..7ed014f4e0f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Fixed malformed LaTeX in `CLIPScore` and `HausdorffDistance` docstring math so it renders correctly ([#3427](https://github.com/Lightning-AI/torchmetrics/pull/3427)) +- Fixed `IndexError` in curve-based classification metrics when `ignore_index` removed every sample of a class, which now score as `nan` ([#3444](https://github.com/Lightning-AI/torchmetrics/pull/3444)) + + --- ## [1.9.0] - 2026-03-05 diff --git a/src/torchmetrics/functional/classification/average_precision.py b/src/torchmetrics/functional/classification/average_precision.py index 7810c2352b5..a89c2a9c397 100644 --- a/src/torchmetrics/functional/classification/average_precision.py +++ b/src/torchmetrics/functional/classification/average_precision.py @@ -52,7 +52,14 @@ def _reduce_average_precision( recall = torch.where(torch.isnan(recall), torch.zeros_like(recall), recall) res = -torch.sum((recall[:, 1:] - recall[:, :-1]) * precision[:, :-1], 1) else: - res = torch.stack([-torch.sum((r[1:] - r[:-1]) * p[:-1]) for p, r in zip(precision, recall)]) + res = torch.stack([ + # A class with fewer than two curve points had no samples, so its score is undefined. + # Summing the empty slices below would otherwise report it as 0. + torch.tensor(float("nan"), device=r.device, dtype=r.dtype) + if r.numel() < 2 + else -torch.sum((r[1:] - r[:-1]) * p[:-1]) + for p, r in zip(precision, recall) + ]) if average is None or average == "none": return res if torch.isnan(res).any(): @@ -74,6 +81,11 @@ def _binary_average_precision_compute( thresholds: Optional[Tensor], ) -> Tensor: precision, recall, _ = _binary_precision_recall_curve_compute(state, thresholds) + if torch.isnan(precision).all() and torch.isnan(recall).all(): + # An undefined curve (no samples) must not collapse to 0.0 through the `nan` -> 0 filling + # below, which would be indistinguishable from a genuinely zero score. + return torch.tensor(float("nan"), device=precision.device) + precision = torch.where(torch.isnan(precision), torch.zeros_like(precision), precision) recall = torch.where(torch.isnan(recall), torch.zeros_like(recall), recall) return -torch.sum((recall[1:] - recall[:-1]) * precision[:-1]) diff --git a/src/torchmetrics/functional/classification/precision_recall_curve.py b/src/torchmetrics/functional/classification/precision_recall_curve.py index f6b09289692..a71cbd117eb 100644 --- a/src/torchmetrics/functional/classification/precision_recall_curve.py +++ b/src/torchmetrics/functional/classification/precision_recall_curve.py @@ -57,6 +57,16 @@ def _binary_clf_curve( # remove class dimension if necessary if preds.ndim > target.ndim: preds = preds[:, 0] + + if target.numel() == 0: + # Every sample was filtered out, e.g. an entire class was `ignore_index`. There is no + # curve to build; returning empty lets callers report the score as undefined instead of + # indexing into empty tensors below. + # Match the dtypes the non-empty branch below produces: the counts are default float + # (they come from `target * weight`) while the thresholds are taken from `preds`. + empty_counts = torch.empty(0, dtype=torch.get_default_dtype(), device=preds.device) + return empty_counts, empty_counts.clone(), torch.empty(0, dtype=preds.dtype, device=preds.device) + desc_score_indices = torch.argsort(preds, descending=True) preds = preds[desc_score_indices] @@ -274,6 +284,12 @@ def _binary_precision_recall_curve_compute( return precision, recall, thresholds fps, tps, thresholds = _binary_clf_curve(state[0], state[1], pos_label=pos_label) + if tps.numel() == 0: + # No samples left after filtering, so there is no curve. Report it as undefined rather than + # indexing `tps[-1]` below. + nan = torch.full((1,), float("nan"), dtype=tps.dtype, device=tps.device) + return nan, nan.clone(), nan.clone() + precision = tps / (tps + fps) recall = tps / tps[-1] if (state[1] == 0).all(): # all labels are negative, recall is undefined diff --git a/src/torchmetrics/functional/classification/roc.py b/src/torchmetrics/functional/classification/roc.py index 406443a397f..4f16ae684b2 100644 --- a/src/torchmetrics/functional/classification/roc.py +++ b/src/torchmetrics/functional/classification/roc.py @@ -52,6 +52,12 @@ def _binary_roc_compute( thres = thresholds.flip(0) else: fps, tps, thres = _binary_clf_curve(preds=state[0], target=state[1], pos_label=pos_label) + if tps.numel() == 0: + # No samples left after filtering, so there is no curve. Returning `nan` keeps the score + # undefined instead of letting the zero point added below turn it into a score of 0. + nan = torch.full((1,), float("nan"), dtype=tps.dtype, device=tps.device) + return nan, nan.clone(), nan.clone() + # Add an extra threshold position to make sure that the curve starts at (0, 0) tps = torch.cat([torch.zeros(1, dtype=tps.dtype, device=tps.device), tps]) fps = torch.cat([torch.zeros(1, dtype=fps.dtype, device=fps.device), fps]) diff --git a/src/torchmetrics/utilities/compute.py b/src/torchmetrics/utilities/compute.py index fe053fccf65..2fa589ceb31 100644 --- a/src/torchmetrics/utilities/compute.py +++ b/src/torchmetrics/utilities/compute.py @@ -115,6 +115,14 @@ def _auc_compute_without_check(x: Tensor, y: Tensor, direction: float, axis: int Assumes increasing or decreasing order of `x`. """ + if x.numel() < 2: + # A curve needs at least two points to have an area. Fewer means there were no samples to + # build it from, so the area is undefined rather than zero -- `torch.trapz` would return 0.0 + # here, which is indistinguishable from a genuinely zero score. Reductions over several + # classes already drop `nan` entries and warn about them. + # `x` may be integer (see `auc`), where a nan is not representable, so fall back to float + dtype = x.dtype if x.is_floating_point() else torch.get_default_dtype() + return torch.tensor(float("nan"), device=x.device, dtype=dtype) with torch.no_grad(): auc_score: Tensor = torch.trapz(y, x, dim=axis) * direction return auc_score diff --git a/tests/unittests/classification/test_auroc.py b/tests/unittests/classification/test_auroc.py index df12366b941..a8ccbdd4173 100644 --- a/tests/unittests/classification/test_auroc.py +++ b/tests/unittests/classification/test_auroc.py @@ -432,3 +432,33 @@ def test_wrapper_class(metric, kwargs, base_metric=AUROC): instance = base_metric(**kwargs) assert isinstance(instance, metric) assert isinstance(instance, Metric) + + +def test_corner_case_all_samples_ignored(): + """Check that a fully ignored input is undefined rather than an error. + + See issue https://github.com/Lightning-AI/torchmetrics/issues/2685. + + """ + preds = torch.tensor([0.1, 0.8, 0.4]) + target = torch.tensor([-1, -1, -1]) + assert torch.isnan(BinaryAUROC(ignore_index=-1)(preds, target)) + + +def test_corner_case_single_label_ignored(): + """Check that ignoring every sample of one label leaves the other labels untouched. + + The per-label curves are computed independently, so an empty label must not affect labels that + still have samples, and must not raise. See issue + https://github.com/Lightning-AI/torchmetrics/issues/2685. + + """ + preds = torch.tensor([[0.9, 0.5, 0.2], [0.1, 0.3, 0.8], [0.7, 0.6, 0.4]]) + target = torch.tensor([[1, -1, 0], [0, -1, 1], [1, -1, 0]]) + + res = MultilabelAUROC(num_labels=3, average=None, ignore_index=-1)(preds, target) + + assert torch.isnan(res[1]) + assert not torch.isnan(res[[0, 2]]).any() + # macro averaging already drops undefined labels, so it stays finite + assert not torch.isnan(MultilabelAUROC(num_labels=3, average="macro", ignore_index=-1)(preds, target)) diff --git a/tests/unittests/classification/test_average_precision.py b/tests/unittests/classification/test_average_precision.py index 1eca3aa6d83..da953a7d9b1 100644 --- a/tests/unittests/classification/test_average_precision.py +++ b/tests/unittests/classification/test_average_precision.py @@ -434,3 +434,29 @@ def test_wrapper_class(metric, kwargs, base_metric=AveragePrecision): instance = base_metric(**kwargs) assert isinstance(instance, metric) assert isinstance(instance, Metric) + + +def test_corner_case_all_samples_ignored(): + """Check that a fully ignored input is undefined rather than an error. + + See issue https://github.com/Lightning-AI/torchmetrics/issues/2685. + + """ + preds = torch.tensor([0.1, 0.8, 0.4]) + target = torch.tensor([-1, -1, -1]) + assert torch.isnan(BinaryAveragePrecision(ignore_index=-1)(preds, target)) + + +def test_corner_case_single_label_ignored(): + """Check that ignoring every sample of one label leaves the other labels untouched. + + See issue https://github.com/Lightning-AI/torchmetrics/issues/2685. + + """ + preds = torch.tensor([[0.9, 0.5, 0.2], [0.1, 0.3, 0.8], [0.7, 0.6, 0.4]]) + target = torch.tensor([[1, -1, 0], [0, -1, 1], [1, -1, 0]]) + + res = MultilabelAveragePrecision(num_labels=3, average=None, ignore_index=-1)(preds, target) + + assert torch.isnan(res[1]) + assert not torch.isnan(res[[0, 2]]).any() From 46ef567884cd3281c90a82f49b869f042c469c95 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 18 Aug 2026 08:59:11 +0000 Subject: [PATCH 2/3] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/unittests/classification/test_auroc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unittests/classification/test_auroc.py b/tests/unittests/classification/test_auroc.py index a8ccbdd4173..d0b1ca7500c 100644 --- a/tests/unittests/classification/test_auroc.py +++ b/tests/unittests/classification/test_auroc.py @@ -448,8 +448,8 @@ def test_corner_case_all_samples_ignored(): def test_corner_case_single_label_ignored(): """Check that ignoring every sample of one label leaves the other labels untouched. - The per-label curves are computed independently, so an empty label must not affect labels that - still have samples, and must not raise. See issue + The per-label curves are computed independently, so an empty label must not affect labels that still have samples, + and must not raise. See issue https://github.com/Lightning-AI/torchmetrics/issues/2685. """ From 047d231f98038faa188e98ccdb05ab0503e27852 Mon Sep 17 00:00:00 2001 From: arose26 Date: Tue, 18 Aug 2026 04:58:57 -0400 Subject: [PATCH 3/3] Point changelog entry at the PR number --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ed014f4e0f..c1f1baae253 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,7 +36,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Fixed malformed LaTeX in `CLIPScore` and `HausdorffDistance` docstring math so it renders correctly ([#3427](https://github.com/Lightning-AI/torchmetrics/pull/3427)) -- Fixed `IndexError` in curve-based classification metrics when `ignore_index` removed every sample of a class, which now score as `nan` ([#3444](https://github.com/Lightning-AI/torchmetrics/pull/3444)) +- Fixed `IndexError` in curve-based classification metrics when `ignore_index` removed every sample of a class, which now score as `nan` ([#3464](https://github.com/Lightning-AI/torchmetrics/pull/3464)) ---