From 3a4cf4c660e10eb293cea3cc60b4e550cc4f6aa0 Mon Sep 17 00:00:00 2001 From: Aditya Singh Date: Wed, 5 Aug 2026 05:33:34 -0700 Subject: [PATCH] Add a score on the lowest performing subjects compute_lowest_subject_scores ranks the subjects of every (dataset, pipeline) pair and averages only the lowest percentile of them, which is the family of metrics F1@20% belongs to. ERP, SSVEP and c-VEP saturate close to a perfect score on most subjects, so a mean over the whole cohort is dominated by subjects that no longer separate pipelines. The subjects a pipeline handles worst are where the remaining headroom is. Fixes #733 --- docs/source/api.rst | 1 + docs/source/whats_new.rst | 2 + moabb/analysis/meta_analysis.py | 61 ++++++++++++++++++++++++ moabb/tests/test_analysis.py | 82 +++++++++++++++++++++++++++++++++ 4 files changed, 146 insertions(+) diff --git a/docs/source/api.rst b/docs/source/api.rst index 4c8a5999a9..477811585c 100644 --- a/docs/source/api.rst +++ b/docs/source/api.rst @@ -551,6 +551,7 @@ Statistics meta_analysis.combine_effects meta_analysis.combine_pvalues meta_analysis.collapse_session_scores + meta_analysis.compute_lowest_subject_scores ----- Utils diff --git a/docs/source/whats_new.rst b/docs/source/whats_new.rst index b30e0c6e3f..dd8ac40cc3 100644 --- a/docs/source/whats_new.rst +++ b/docs/source/whats_new.rst @@ -44,6 +44,7 @@ Enhancements - Add :class:`moabb.datasets.preprocessing.EuclideanAlignment`, a trial-level Euclidean Alignment transformer (He & Wu 2020; Junqueira et al. 2024) that whitens each trial by the inverse square root of the Euclidean mean covariance to remove per-domain covariance shift before a (deep) model sees the data. Inductive and leakage-free by default (``fit`` learns the reference from training trials, ``transform`` re-applies it to unseen trials); ``fit_transform`` gives the transductive, per-recording form. Accepts an :class:`mne.BaseEpochs` or an ``(n_trials, n_channels, n_times)`` ndarray, uses a shrinkage covariance estimator (``"lwf"``) for robustness, and adds no new dependency (``pyriemann >= 0.11`` is already required). Distinct from :class:`pyriemann.transfer.TLCenter`, which recenters covariance *matrices* (:gh:`1108` by `Bruno Aristimunha`_). - Add an ``n_jobs`` parameter to :meth:`moabb.paradigms.base.BaseParadigm.get_data` and :meth:`moabb.datasets.base.BaseDataset.get_data` to load and preprocess subjects in parallel with :class:`joblib.Parallel`. Per-subject processing (reading, filtering, resampling, epoching) is independent, so this gives a near-linear speedup on datasets with many subjects, with identical numerical results. moabb's own patches to the shared BIDS cache files (``participants.tsv``/``.json``, ``dataset_description.json``) now take the mne-bids cross-process file lock, so parallel caching stays consistent (:gh:`1124` by `Bruno Aristimunha`_). - Drive cross-validation folds with any stock scikit-learn cross-validator passed as ``cv_class``, controlled by a ``groups`` argument — a metadata column name, a list of column names (compound key, e.g. ``["subject", "session"]``), or a callable ``metadata -> array`` — together with callable ``cv_kwargs`` resolved against the metadata (e.g. ``cv_class=PredefinedSplit`` with a ``test_fold`` callable to target a single fold). ``groups`` is exposed on :class:`moabb.evaluations.WithinSessionEvaluation`, :class:`moabb.evaluations.WithinSubjectEvaluation`, :class:`moabb.evaluations.CrossSessionEvaluation` and :class:`moabb.evaluations.CrossSubjectEvaluation` and threaded to their splitters; each splitter keeps its default grouping (``"subject"`` / ``"session"`` / labels) when ``groups`` is ``None``. :class:`moabb.evaluations.splitters.CrossDatasetSplitter` gains ``groups`` (its ``group_column`` argument is now a deprecated alias) (:gh:`1104` by `Bruno Aristimunha`_). +- Add :func:`moabb.analysis.meta_analysis.compute_lowest_subject_scores`, which scores every (dataset, pipeline) pair on the subjects it handles worst instead of on the whole cohort. Sessions are averaged per subject, the subjects are ranked, and the lowest ``percentile`` percent of them (rounded up, at least one) are averaged, giving the family of metrics that F1@20% belongs to. ERP, SSVEP and c-VEP saturate close to a perfect score on most subjects, so a cohort mean is dominated by subjects that no longer separate pipelines, while the worst subjects are where the headroom is (:gh:`733` by `Aditya Singh`_) API changes ~~~~~~~~~~~ @@ -923,4 +924,5 @@ API changes .. _Danae: https://github.com/dnplchrn .. _Henrique Lefundes: https://github.com/HenriqueLefundes .. _Paul-Adrien Graignic: https://github.com/pagraignic-yneuro +.. _Aditya Singh: https://github.com/adityasingh2400 .. _pre-commit-ci: https://github.com/apps/pre-commit-ci diff --git a/moabb/analysis/meta_analysis.py b/moabb/analysis/meta_analysis.py index 6572ab7297..cc5b1f3fc2 100644 --- a/moabb/analysis/meta_analysis.py +++ b/moabb/analysis/meta_analysis.py @@ -33,6 +33,67 @@ def collapse_session_scores(df): ) +def compute_lowest_subject_scores(df, percentile=20): + """Score of a pipeline on its lowest performing subjects. + + For every (dataset, pipeline) pair, rank the subjects by score and keep + only the ``percentile`` percent that score lowest, then average over them. + + Paradigms such as ERP, SSVEP and c-VEP saturate near a perfect score on + most subjects, so a mean over the whole cohort is dominated by subjects + that no longer discriminate between pipelines. The subjects a pipeline + handles worst are where the remaining differences are, which is what the + F1@20% metric of [1]_ reports. + + Scores are first averaged over the sessions of a subject, so every subject + weighs the same regardless of how many sessions it contributes. + + Parameters + ---------- + df: :class:`pandas.DataFrame` + results obtained by an evaluation, with at least the ``dataset``, + ``pipeline``, ``subject`` and ``score`` columns + percentile: float, default=20 + percentage of subjects to keep, in ``(0, 100]``. The number of + retained subjects is rounded up, and is at least one, so a value + small enough always falls back to the single worst subject. + + Returns + ------- + scores: :class:`pandas.DataFrame` + One row per (dataset, pipeline) pair, with the mean ``score`` over + the retained subjects and the ``n_subjects`` that were retained. + + References + ---------- + .. [1] Gnassounou, T., Collas, A., Flamary, R., Gramfort, A., 2025. + Multi-Source and Test-Time Domain Adaptation on Multivariate + Signals using Spatio-Temporal Monge Alignment. + https://arxiv.org/abs/2503.04582 + """ + if not 0 < percentile <= 100: + raise ValueError(f"percentile must be in (0, 100], got {percentile}") + + subject_scores = collapse_session_scores(df) + rows = [] + for (dataset, pipeline), group in subject_scores.groupby( + ["dataset", "pipeline"], sort=False + ): + n_keep = max(1, int(np.ceil(len(group) * percentile / 100))) + # Sorting on the subject as well keeps the selection stable when + # several subjects share the score at the cut-off. + lowest = group.sort_values(["score", "subject"], kind="stable").head(n_keep) + rows.append( + { + "dataset": dataset, + "pipeline": pipeline, + "score": lowest["score"].mean(), + "n_subjects": n_keep, + } + ) + return pd.DataFrame(rows, columns=["dataset", "pipeline", "score", "n_subjects"]) + + def compute_pvals_wilcoxon(df, order=None): """Compute Wilcoxon rank-sum test on aggregated results. diff --git a/moabb/tests/test_analysis.py b/moabb/tests/test_analysis.py index 42f77569d5..325db09af8 100644 --- a/moabb/tests/test_analysis.py +++ b/moabb/tests/test_analysis.py @@ -4,6 +4,7 @@ import numpy as np import pandas as pd +import pytest from matplotlib.pyplot import Figure import moabb.analysis.meta_analysis as ma @@ -167,6 +168,87 @@ def test_compute_pvals_random_cannot_be_zero(self): assert p1vsp2 >= 1 / n_perms, f"P-values cannot be zero {pvals}" +def _results_df(scores, dataset="d1", pipeline="p1", sessions=("0",)): + """Build an evaluation-like dataframe from per-subject scores.""" + return pd.DataFrame( + [ + { + "dataset": dataset, + "pipeline": pipeline, + "subject": subject, + "session": session, + "score": score, + } + for subject, score in enumerate(scores, start=1) + for session in sessions + ] + ) + + +class TestLowestSubjectScores: + def test_keeps_the_worst_subjects(self): + df = _results_df([0.9, 0.5, 1.0, 0.6, 0.95, 0.99, 0.98, 0.97, 0.96, 0.94]) + out = ma.compute_lowest_subject_scores(df, percentile=20) + assert len(out) == 1 + # 20% of 10 subjects, so the two worst: 0.5 and 0.6. + assert out.loc[0, "n_subjects"] == 2 + assert out.loc[0, "score"] == pytest.approx(0.55) + + def test_rounds_the_subject_count_up(self): + df = _results_df([0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7]) + out = ma.compute_lowest_subject_scores(df, percentile=20) + # 20% of 7 subjects is 1.4, rounded up to 2. + assert out.loc[0, "n_subjects"] == 2 + assert out.loc[0, "score"] == pytest.approx(0.15) + + def test_always_keeps_at_least_one_subject(self): + df = _results_df([0.3, 0.4, 0.5]) + out = ma.compute_lowest_subject_scores(df, percentile=1) + assert out.loc[0, "n_subjects"] == 1 + assert out.loc[0, "score"] == pytest.approx(0.3) + + def test_full_percentile_matches_the_plain_mean(self): + scores = [0.4, 0.6, 0.8, 1.0] + out = ma.compute_lowest_subject_scores(_results_df(scores), percentile=100) + assert out.loc[0, "n_subjects"] == len(scores) + assert out.loc[0, "score"] == pytest.approx(np.mean(scores)) + + def test_sessions_are_averaged_before_ranking(self): + # Subject 1 holds the single worst session (0.1) but averages 0.5, + # above subject 2. Ranking the raw rows would pick subject 1. + df = pd.concat( + [ + _results_df([0.1, 0.35], sessions=("0",)), + _results_df([0.9, 0.35], sessions=("1",)), + ], + ignore_index=True, + ) + out = ma.compute_lowest_subject_scores(df, percentile=50) + assert out.loc[0, "n_subjects"] == 1 + assert out.loc[0, "score"] == pytest.approx(0.35) + + def test_one_row_per_dataset_and_pipeline(self): + df = pd.concat( + [ + _results_df([0.2, 0.4], dataset="d1", pipeline="p1"), + _results_df([0.6, 0.8], dataset="d1", pipeline="p2"), + _results_df([0.1, 0.3], dataset="d2", pipeline="p1"), + ], + ignore_index=True, + ) + out = ma.compute_lowest_subject_scores(df, percentile=50) + assert len(out) == 3 + scores = out.set_index(["dataset", "pipeline"])["score"] + assert scores[("d1", "p1")] == pytest.approx(0.2) + assert scores[("d1", "p2")] == pytest.approx(0.6) + assert scores[("d2", "p1")] == pytest.approx(0.1) + + @pytest.mark.parametrize("percentile", [0, -5, 101]) + def test_rejects_an_invalid_percentile(self, percentile): + with pytest.raises(ValueError, match="percentile"): + ma.compute_lowest_subject_scores(_results_df([0.5]), percentile=percentile) + + class TestResults: def setup_method(self, method): self.obj = Results(