diff --git a/CHANGELOG.md b/CHANGELOG.md index d33d477ef8d..4bc97e58972 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Fixed `Metric` ignoring an active `torch.device` context manager on torch 2.3-2.7 ([#3448](https://github.com/Lightning-AI/torchmetrics/pull/3448)) +- Fixed swapped hypothesis and reference arguments in `TER`, which made an empty hypothesis score a perfect 0.0 ([#3479](https://github.com/Lightning-AI/torchmetrics/pull/3479)) + + --- ## [1.9.0] - 2026-03-05 diff --git a/src/torchmetrics/functional/text/ter.py b/src/torchmetrics/functional/text/ter.py index 2d7a6211e0d..40e32fb96cf 100644 --- a/src/torchmetrics/functional/text/ter.py +++ b/src/torchmetrics/functional/text/ter.py @@ -444,7 +444,7 @@ def _compute_sentence_statistics(pred_words: list[str], target_words: list[list[ best_num_edits = tensor(2e16) for tgt_words in target_words: - num_edits = _translation_edit_rate(tgt_words, pred_words) + num_edits = _translation_edit_rate(pred_words, tgt_words) tgt_lengths += len(tgt_words) if num_edits < best_num_edits: best_num_edits = num_edits diff --git a/tests/unittests/text/test_ter.py b/tests/unittests/text/test_ter.py index 5b51d8da704..1a46ca97e29 100644 --- a/tests/unittests/text/test_ter.py +++ b/tests/unittests/text/test_ter.py @@ -176,3 +176,22 @@ def test_ter_return_sentence_level_class(): targets = _inputs_single_sentence_multiple_references.target _, sentence_ter = ter_metric(preds, targets) isinstance(sentence_ter, Tensor) + + +def test_ter_empty_hypothesis_is_not_a_perfect_score(): + """Test that a hypothesis producing nothing is charged one deletion per reference word.""" + assert translation_edit_rate([""], [["hello world foo"]]) == tensor(1.0) + + +def test_ter_matches_reference_when_shifts_are_asymmetric(): + """Test that edits are counted from hypothesis to reference, not the other way around. + + The shift search is not symmetric, so swapping the two sides changes the number of edits found. + + """ + preds = ["hello hello the a dog"] + targets = [["jumps dog lazy the"]] + expected = _reference_sacrebleu_ter( + preds, targets, normalized=False, no_punct=False, asian_support=False, case_sensitive=True + ) + assert translation_edit_rate(preds, targets, lowercase=False) == expected