From beba7e02d1d1dd80802bce46e39cf6feafc1ebe5 Mon Sep 17 00:00:00 2001 From: Patricia Pampanelli Date: Mon, 24 Aug 2026 15:14:04 +0000 Subject: [PATCH 1/5] feat: narrow probe selection by intent: in run.spec Signed-off-by: Patricia Pampanelli --- docs/source/cas.rst | 13 +++++++ docs/source/configurable.rst | 34 ++++++++++------- docs/source/extending.probe.rst | 4 +- garak/_selection.py | 68 +++++++++++++++++++++++++++++++-- garak/cli.py | 12 ++++-- garak/command.py | 25 +++++++++--- 6 files changed, 130 insertions(+), 26 deletions(-) diff --git a/docs/source/cas.rst b/docs/source/cas.rst index 95661326c..60330ed46 100644 --- a/docs/source/cas.rst +++ b/docs/source/cas.rst @@ -136,9 +136,22 @@ A few things to note: * ``intent:S003productkey`` selects that single leaf behaviour. A *category* code such as ``intent:S003`` ("Illegal") instead expands to all of its leaves (``S003illegal``, ``S003instructions``, ``S003goods``, ``S003services``, ``S003productkeys``). +* An explicit ``intent:`` also **filters the ordinary probe selection** by + typology descendancy: ``garak --spec "probes.*,intent:S005hate"`` runs only the + probes carrying an intent beneath ``S005hate`` (plus any ``IntentProbe``, unless + its ``blocked_intent_spec`` covers every included code). The injected default + scope never filters. * If you give no ``intent:`` selector, the default scope ``S`` (the whole Safety branch) is injected at resolve time. +.. note:: + + The intent filter reads each probe's *class* intent from the plugin cache. An + intent that a probe only carries per payload group (see + :doc:`probes/encoding` and the payloads mechanism) is not visible to the + filter, so filtering by such an intent will not select that probe. This is + inherent to filtering before probes are instantiated. + Each intent carries a short imperative *stub* in the typology, which the technique expands into prompts. ``GrandmaIntent`` wraps each stub in a roleplay template, producing prompts such as: diff --git a/docs/source/configurable.rst b/docs/source/configurable.rst index a0a02456f..3ddbd6a0a 100644 --- a/docs/source/configurable.rst +++ b/docs/source/configurable.rst @@ -183,18 +183,24 @@ Selectors (a category prefix is mandatory): * ``tier:`` - filters probes by tier; **inclusive** ("log level"): ``tier:N`` admits tiers ``1..N`` (``tier:1`` is the most critical). Names work too (``tier:of_concern`` == ``tier:1``). -* ``intent:`` - selects intent typology codes for intent-based probes - (e.g. ``intent:S`` for the whole Safety branch, ``intent:S001`` for a category, - ``intent:S001mis`` for a leaf); ``intent:*`` or ``intent:all`` selects every - intent. This is a **separate axis** consumed by the - intent service: it does **not** add or remove probes. When no ``intent:`` is - given, the default scope ``S`` (the Safety branch) is injected at resolve - time. Typology - expansion and detectorless filtering are governed by the ``run.*`` intent - modifiers (``run.serve_detectorless_intents``). - Only ``IntentProbe`` - subclasses consume intents; selecting ``intent:`` without an ``IntentProbe`` - warns and proceeds. +* ``intent:`` - filters probes by intent typology code, and scopes the + intents a selected ``IntentProbe`` exercises. Matching is by typology + **descendancy**, not string prefix: a branch code keeps every probe whose + intent lies beneath it, while a leaf keeps only probes declaring that leaf + (``intent:S`` keeps the whole Safety branch, ``intent:S005`` a category, and + ``intent:S005hate`` only the ``S005hate`` leaf -- not a probe declaring the + parent ``S005``, nor the sibling leaf ``S005bully``). Two exemptions: the + injected default scope is never a filter, and ``IntentProbe`` subclasses are + pruned only when their ``blocked_intent_spec`` covers every included code -- + otherwise which intents they serve is settled later by the intent service. + ``intent:*`` or ``intent:all`` selects every intent and does not + filter. When no ``intent:`` is given, the default scope ``S`` (the Safety + branch) is injected at resolve time and, being the default, never prunes. + Typology expansion and detectorless filtering are governed by the ``run.*`` + intent modifiers (``run.serve_detectorless_intents``). Only ``IntentProbe`` + subclasses derive prompts from intents; giving an explicit ``intent:`` with no + ``IntentProbe`` in the selection warns and proceeds -- ordinary probes that + match the intent still run, but no intent-derived prompts are generated. Polarity: a bare selector (or ``+``) includes; a leading ``-`` removes. Note the asymmetry of ``tier``: ``tier:N`` is the inclusive filter, while ``-tier:N`` @@ -221,8 +227,10 @@ wildcard, so quote those specs (or use the ``all`` alias instead). garak --spec probes.all,probes.fitd.FITD # tiers {1,3}: tier:3 admits 1..3, then -tier:2 removes exactly tier 2 garak --spec "+probes.*,+tier:3,-tier:2" - # an intent probe over one intent category (intents are a separate axis) + # an intent probe over one intent category (intents scope the IntentProbe) garak --spec probes.grandma.GrandmaIntent,intent:S004 + # intent as a filter: only probes carrying the S005hate intent (plus IntentProbes) + garak --spec "probes.*,intent:S005hate" .. code-block:: yaml diff --git a/docs/source/extending.probe.rst b/docs/source/extending.probe.rst index 7665f8ad5..be6441c28 100644 --- a/docs/source/extending.probe.rst +++ b/docs/source/extending.probe.rst @@ -269,7 +269,9 @@ Two class attributes tune which intents the probe consumes: * ``skip_root_intents`` (default ``True``) -- skip single-letter root codes when gathering stubs, since a whole branch rarely has a meaningful prototypical stub. * ``blocked_intent_spec`` (default ``""``) -- intents this technique should never - exercise, even when in scope. + exercise, even when in scope. If it covers every intent an explicit ``intent:`` + include asked for, ``run.spec`` resolution drops the probe from the selection + entirely, since it would have nothing left to serve. If the active intent set is empty (for example the ``intent:`` axis was filtered to nothing), the probe is a graceful no-op: it sends no prompts and the run diff --git a/garak/_selection.py b/garak/_selection.py index 1fa0c9028..451803b24 100644 --- a/garak/_selection.py +++ b/garak/_selection.py @@ -17,6 +17,7 @@ from garak import _plugins from garak import _spec +from garak.cas import get_parent_name # Tier assigned to probes that do not declare one (Tier.UNLISTED). _DEFAULT_TIER = 9 @@ -78,6 +79,32 @@ def _tier_of(name: str) -> int: return int(_plugins.plugin_info(name).get("tier", _DEFAULT_TIER)) +def _intent_under(code: str, codes: List[str]) -> bool: + """True if ``code`` or a typology ancestor is in ``codes``; malformed codes never match.""" + current: str = code + while current: + if current in codes: + return True + try: + current = get_parent_name(current) + except ValueError: + return False + return False + + +def _intent_keeps(name: str, includes: List[str], excludes: List[str]) -> bool: + """True if ``name`` survives the intent filter. A probe declaring no intent + (an IntentProbe, by convention) is kept unless its ``blocked_intent_spec`` + covers every included code, in which case it has nothing left to serve.""" + info = _plugins.plugin_info(name) + code = info.get("intent") + if code is None: + blocked_spec = info.get("blocked_intent_spec", "") + blocked = [c.strip() for c in blocked_spec.split(",") if c.strip()] + return not (blocked and all(_intent_under(inc, blocked) for inc in includes)) + return _intent_under(code, includes) and not _intent_under(code, excludes) + + def _empty_reason(spec: _spec.Spec) -> str: """Best-effort explanation of why a spec resolved to no probes.""" tier_ceilings = [int(s.value) for s in spec.include if s.kind == "tier"] @@ -95,6 +122,17 @@ def _empty_reason(spec: _spec.Spec) -> str: f"probe '{name}' is tier {_tier_of(name)} but the spec restricts to " f"tiers 1..{ceiling}; widen the tier filter or drop the explicit probe" ) + intent_codes = [ + s.value + for s in spec.include + if s.kind == "intent" and s.value.lower() not in ("*", "all") + ] + if intent_codes: + codes = ", ".join(intent_codes) + return ( + f"no selected probe carries intent '{codes}'; widen the probe " + f"selection or drop the intent selector" + ) if any(s.kind in ("tag", "tier") for s in spec.include): return "no active probe matches the given tier/tag filters; widen the filters" return "every included probe was removed by an exclusion; adjust includes/excludes" @@ -104,7 +142,11 @@ def resolve_spec(spec: _spec.Spec, skip_unknown: bool = False) -> _spec.Resoluti """Resolve a :class:`garak._spec.Spec` to concrete probe and buff names. Selection happens against the live plugin registry (active state, tiers, - tags). This is the single entry point used by the CLI and harnesses. + tags). An explicit ``intent:`` include additionally filters probes by + typology descendancy; the injected default scope never filters, and + ``IntentProbe`` subclasses are pruned only when their + ``blocked_intent_spec`` covers every included code. This is the single + entry point used by the CLI and harnesses. """ rejected: List[str] = [] inactive_modules: List[str] = [] @@ -137,6 +179,28 @@ def resolve_spec(spec: _spec.Spec, skip_unknown: bool = False) -> _spec.Resoluti if tag_prefixes: candidate = {p for p in candidate if _has_any_tag(p, tag_prefixes)} + # Intent filter, mirroring the tag filter's OR-of-prefixes shape but matching + # by typology descendancy. Triggered only by explicit intent: + # includes (the injected DEFAULT_INTENT_SCOPE and exclude-only specs never + # prune); intent:* / intent:all are vacuous and do not filter. A probe that + # declares no intent (an IntentProbe, by convention) is pruned only when its + # blocked_intent_spec covers every included code; otherwise which intents it + # actually serves is decided later by IntentService. + intent_includes = [s.value for s in spec.include if s.kind == "intent"] + intent_excludes = [s.value for s in spec.exclude if s.kind == "intent"] + # Malformed codes must not drive pruning: they stay in ``rejected`` (raised + # below unless ``skip_unknown``), so a bad code never silently narrows the + # preview to IntentProbe-only under ``--list_probes``. + intent_filter = [ + c + for c in intent_includes + if c.lower() not in ("*", "all") and _spec.validate_intent_specifier(c) + ] + if intent_filter: + candidate = { + p for p in candidate if _intent_keeps(p, intent_filter, intent_excludes) + } + # Buffs: union of buffs.* includes (no implicit default) buff_includes = [ s for s in spec.include if s.kind == "plugin_path" and s.category == "buffs" @@ -172,8 +236,6 @@ def resolve_spec(spec: _spec.Spec, skip_unknown: bool = False) -> _spec.Resoluti # typology membership + expansion + detectorless filtering happen later in # IntentService. When no intent: selector is given, inject the default scope # (_spec.DEFAULT_INTENT_SCOPE) so the intent scope survives a run.spec override. - intent_includes = [s.value for s in spec.include if s.kind == "intent"] - intent_excludes = [s.value for s in spec.exclude if s.kind == "intent"] for code in intent_includes + intent_excludes: # ``*`` / ``all`` select every intent (IntentService expands the vacuous # sentinel); other codes must match the typology specifier format. diff --git a/garak/cli.py b/garak/cli.py index 60709397e..ac54005f5 100644 --- a/garak/cli.py +++ b/garak/cli.py @@ -550,9 +550,11 @@ def worker_count_validation(workers): selected_probes = None if _config.run.spec: - selected_probes = _selection.resolve_spec( + resolved = _selection.resolve_spec( parse_spec_file(_config.run.spec), skip_unknown=True - ).probes + ) + command.warn_rejected_selectors(resolved.rejected, "probes") + selected_probes = resolved.probes command.print_probes(selected_probes, verbose=_config.system.verbose) elif args.list_detectors: @@ -570,9 +572,11 @@ def worker_count_validation(workers): selected_buffs = None if _config.run.spec: - selected_buffs = _selection.resolve_spec( + resolved = _selection.resolve_spec( parse_spec_file(_config.run.spec), skip_unknown=True - ).buffs + ) + command.warn_rejected_selectors(resolved.rejected, "buffs") + selected_buffs = resolved.buffs command.print_buffs(selected_buffs) elif args.list_generators: diff --git a/garak/command.py b/garak/command.py index f61c2fe00..f3e9a9e86 100644 --- a/garak/command.py +++ b/garak/command.py @@ -353,9 +353,12 @@ def _selection_has_intent_probe(probe_names) -> bool: def warn_unconsumed_intents(probe_names) -> None: - """Warn once when ``intent:`` was given explicitly but no IntentProbe is in the - selection to consume it. The intent axis does not select probes, so without an - IntentProbe the intents are never exercised.""" + """Warn once when an ``intent:`` selector was given explicitly but no + IntentProbe is in the selection. An explicit ``intent:`` include also narrows + the probe set (acting as a filter, like ``tag:``), whereas exclude-only and + ``intent:*``/``intent:all`` selectors do not; either way, only an IntentProbe + derives prompts from the intent typology, so whenever none is selected no + intent-derived prompts are generated.""" from garak import _config if not getattr(_config.transient, "intents_explicit", False): @@ -363,13 +366,25 @@ def warn_unconsumed_intents(probe_names) -> None: if _selection_has_intent_probe(probe_names): return msg = ( - "intent: selector(s) given but no IntentProbe is selected; intents will " - "not be exercised (select an IntentProbe, e.g. probes.grandma.GrandmaIntent)" + "intent: selector(s) given but no IntentProbe is selected, so no " + "intent-derived prompts will be generated (add an IntentProbe, " + "e.g. probes.grandma.GrandmaIntent)" ) logging.warning(msg) print(f"⚠️ {msg}") +def warn_rejected_selectors(rejected, namespace: str) -> None: + """Warn when ``run.spec`` selectors (e.g. a malformed ``intent:`` code) were + rejected and silently dropped from a preview such as ``--list_probes``, + mirroring the reporting the run path already does via ``_check_selection``.""" + if not rejected: + return + msg = f"unusable {namespace} selector(s), skipped: {', '.join(rejected)}" + logging.warning(msg) + print(f"⚠️ {msg}") + + # do a run def probewise_run(generator, probe_names, evaluator, buffs): import garak.harnesses.probewise From 239973c803f599ecf108b091292614b1a1cb5e06 Mon Sep 17 00:00:00 2001 From: Patricia Pampanelli Date: Mon, 24 Aug 2026 18:26:23 +0000 Subject: [PATCH 2/5] test: add intent-selection coverage for run.spec narrowing Signed-off-by: Patricia Pampanelli --- tests/cas/test_intent_run_spec.py | 45 +++++++++- tests/cli/test_cli_list_filtering.py | 42 +++++++++ tests/test_spec.py | 128 ++++++++++++++++++++++++++- 3 files changed, 210 insertions(+), 5 deletions(-) diff --git a/tests/cas/test_intent_run_spec.py b/tests/cas/test_intent_run_spec.py index d839fa1ce..72a51e337 100644 --- a/tests/cas/test_intent_run_spec.py +++ b/tests/cas/test_intent_run_spec.py @@ -90,9 +90,15 @@ def test_warn_unconsumed_intents_fires_without_intent_probe(capsys): garak._config.transient.intents_explicit = True command.warn_unconsumed_intents(["probes.dan.DanInTheWild"]) + out = capsys.readouterr().out assert ( - "no IntentProbe is selected" in capsys.readouterr().out + "no IntentProbe is selected" in out ), "explicit intent: with no IntentProbe in the selection must warn" + assert ( + "no intent-derived prompts will be generated" in out + ), "the warning states no intent-derived prompts result without an IntentProbe" + # the message must not assert narrowing, which is false for exclude-only and */all specs + assert "narrowed" not in out, "the warning must not claim narrowing" def test_warn_unconsumed_intents_silent_with_mixed_selection(capsys): @@ -113,3 +119,40 @@ def test_warn_unconsumed_intents_silent_when_default(capsys): garak._config.transient.intents_explicit = False command.warn_unconsumed_intents(["probes.dan.DanInTheWild"]) assert capsys.readouterr().out == "", "the injected default (not explicit) must not warn" + + +def test_warn_rejected_selectors_reports_each(capsys): + # generic reporter for any run.spec selector dropped under skip_unknown=True + # (e.g. a malformed intent: code in a --list_probes/--list_buffs preview). + import garak.command as command + + command.warn_rejected_selectors(["intent:zzz", "probes.nonexistent"], "probes") + out = capsys.readouterr().out + assert "intent:zzz" in out, "each rejected selector must be named in the warning" + assert ( + "probes.nonexistent" in out + ), "each rejected selector must be named in the warning" + + +def test_warn_rejected_selectors_silent_when_empty(capsys): + import garak.command as command + + command.warn_rejected_selectors([], "buffs") + assert capsys.readouterr().out == "", "no rejected selectors must print nothing" + + +def test_probe_pruning_does_not_change_active_intent_set(): + # the probe-selection filter and the IntentService active set are independent + # axes: filtering the probe set must not change which intents become active. + from garak._selection import resolve_spec + from garak._spec import parse_spec_string + + res = resolve_spec(parse_spec_string("probes.*,intent:S005hate")) + assert res.intents == [ + "S005hate" + ], "the intent axis carries the requested code regardless of probe pruning" + active = _load("S005hate") + assert ( + "S005hate" in active + ), "the active intent set derives from the code, not from surviving probes" + diff --git a/tests/cli/test_cli_list_filtering.py b/tests/cli/test_cli_list_filtering.py index 5188b865c..b2f0bafad 100644 --- a/tests/cli/test_cli_list_filtering.py +++ b/tests/cli/test_cli_list_filtering.py @@ -70,6 +70,48 @@ def test_list_probes_with_detector_spec(capsys, options): assert any("🌟" in ln for ln in lines) +def test_list_probes_warns_on_rejected_intent_selector(capsys): + """A malformed intent: code must not be silently dropped from the preview + (the run path already reports it via _check_selection; --list_probes must too).""" + cli.main(["--list_probes", "--spec", "probes.dan,intent:zzz"]) + out = capsys.readouterr().out + assert "intent:zzz" in out, "the rejected selector must be named in the warning" + + lines = _plugin_lines(out) + # print_plugins strips the "probes." category prefix before printing each name + expected = { + name.removeprefix("probes.") + for name, active in _plugins.enumerate_plugins(category="probes") + if active and name.startswith("probes.dan.") + } + assert expected, "fixture expects at least one active probes.dan.* plugin" + assert all( + any(name in ln for ln in lines) for name in expected + ), "the malformed intent: code must leave the whole dan family unfiltered" + + +def test_list_buffs_warns_on_rejected_intent_selector(capsys): + """The --list_buffs preview must warn on a malformed intent: code and still + show the surviving buff, mirroring the --list_probes behaviour above.""" + active_buffs = { + name for name, active in _plugins.enumerate_plugins(category="buffs") if active + } + assert ( + "buffs.lowercase.Lowercase" in active_buffs + ), "fixture expects buffs.lowercase.Lowercase to be active" + + cli.main(["--list_buffs", "--spec", "buffs.lowercase,intent:zzz"]) + out = capsys.readouterr().out + assert "intent:zzz" in out, "the rejected selector must be named in the warning" + + lines = _plugin_lines(out) + assert all(ln.startswith("buffs: ") for ln in lines), "expected all 'buffs:' lines" + # print_plugins strips the "buffs." category prefix before printing each name + assert any( + "lowercase.Lowercase" in ln for ln in lines + ), "the malformed intent: code must leave the requested buff in the preview" + + def test_list_probes_verbose_table(capsys): """Test that --list_probes -v outputs a markdown table with tier and description.""" cli.main(["--list_probes", "-v"]) diff --git a/tests/test_spec.py b/tests/test_spec.py index 35f51c15b..82aa93ee1 100644 --- a/tests/test_spec.py +++ b/tests/test_spec.py @@ -315,10 +315,122 @@ def test_intent_explicit_overrides_default(): assert res.intents_explicit is True, "user-supplied intent: marks the selection explicit" -def test_intent_does_not_filter_probe_set(): - with_intent = set(resolve("probes.dan,intent:S004").probes) - without = set(resolve("probes.dan").probes) - assert with_intent == without, "intent: must not add or remove probes (separate axis)" +def test_intent_default_does_not_filter_probe_set(): + # the injected default scope (no explicit intent:) and an exclude-only spec + # must both leave the probe set untouched -- only explicit includes filter. + # probes.dan carries T009ignore; a naive filter would erase the whole family. + family = {p for p in _active("probes") if p.startswith("probes.dan.")} + injected_default = set(resolve("probes.dan").probes) + exclude_only = set(resolve("probes.dan,-intent:S004").probes) + assert ( + injected_default == family + ), "the injected default scope must not prune probes" + assert exclude_only == family, "an exclude-only intent spec must not prune probes" + + +def test_intent_descendancy_keeps_child_of_branch(): + probes = set(resolve("probes.*,intent:S005").probes) + assert ( + "probes.lmrc.Bullying" in probes + ), "intent:S005 keeps a probe declaring the child leaf S005bully" + + +def test_intent_leaf_drops_more_generic_probe(): + probes = set(resolve("probes.*,intent:S005hate").probes) + assert ( + "probes.lmrc.SlurUsage" in probes + ), "intent:S005hate keeps the probe declaring S005hate" + assert ( + "probes.realtoxicityprompts.RTPBlank" not in probes + ), "a leaf intent must not keep a probe declaring only the parent code S005" + + +def test_intent_siblings_do_not_match_by_prefix(): + # anti-regression guard: string-prefix would wrongly capture the sibling leaf + # S001fabperson under intent:S001fab. Descendancy walks parents, so it does not. + fab = set(resolve("probes.*,intent:S001fab").probes) + assert ( + "probes.packagehallucination.Python" in fab + ), "intent:S001fab keeps a probe declaring S001fab" + assert ( + "probes.goodside.WhoIsRiley" not in fab + ), "intent:S001fab must NOT keep the sibling leaf S001fabperson" + parent = set(resolve("probes.*,intent:S001").probes) + assert { + "probes.packagehallucination.Python", + "probes.goodside.WhoIsRiley", + } <= parent, "intent:S001 keeps both sibling leaves S001fab and S001fabperson" + + +@pytest.mark.parametrize("token", ["intent:*", "intent:all"]) +def test_intent_all_does_not_filter(token): + assert set(resolve(f"probes.*,{token}").probes) == _active( + "probes" + ), "intent:* / intent:all is vacuous and must not prune the probe set" + + +def test_intent_exclude_subtracts_within_explicit_include(): + probes = set(resolve("probes.*,intent:S005,-intent:S005hate").probes) + assert "probes.lmrc.Bullying" in probes, "S005bully stays under intent:S005" + assert ( + "probes.lmrc.SlurUsage" not in probes + ), "-intent:S005hate subtracts the S005hate probe while the include survives" + + +def test_intent_never_prunes_intent_probe_with_empty_blocked_spec(): + # GrandmaIntent's blocked_intent_spec is "" (blocks nothing), so it can never + # cover an include code: which intents it serves is IntentService's decision, + # made after resolution. Also the #1889 self-cancelling-axis boundary case. + probes = set(resolve("probes.grandma.GrandmaIntent,intent:M010degrade").probes) + assert ( + "probes.grandma.GrandmaIntent" in probes + ), "an IntentProbe with no blocked_intent_spec is never removed by the intent filter" + + +def test_intent_probe_pruned_when_blocked_spec_covers_every_include(monkeypatch): + # a synthetic IntentProbe whose blocked_intent_spec fully covers the sole + # requested include has nothing left to serve, so it drops out of selection. + from garak import _selection + + def fake_plugin_info(name): + if name == "probes.fake.FullyBlockedIntent": + return {"intent": None, "blocked_intent_spec": "S005"} + return plugin_info(name) + + monkeypatch.setattr(_selection._plugins, "plugin_info", fake_plugin_info) + assert not _selection._intent_keeps( + "probes.fake.FullyBlockedIntent", ["S005hate"], [] + ), "blocked_intent_spec S005 covers the include S005hate, so the probe drops out" + assert _selection._intent_keeps( + "probes.fake.FullyBlockedIntent", ["S004"], [] + ), "blocked_intent_spec S005 does not cover the unrelated include S004" + + +def test_intent_collapse_names_codes_in_empty_reason(): + res = resolve("probes.lmrc,intent:M010degrade") + assert ( + not res.probes + ), "no lmrc probe carries M010degrade, so the selection collapses" + assert ( + res.empty_reason and "M010degrade" in res.empty_reason + ), "empty_reason names the requested intent code" + assert ( + "intent" in res.empty_reason + ), "reason must read as an intent collapse, not a tier collapse" + + +def test_intent_partial_unmatched_is_silent(): + # a valid explicit intent: that matches nothing, with the selection surviving + # via another code, produces no field/warning -- by symmetry with tag:. + matched = set(resolve("probes.*,intent:S005hate").probes) + partial = resolve("probes.*,intent:S005hate,intent:M010rep") + assert ( + set(partial.probes) == matched + ), "an unmatched extra intent code leaves the surviving selection unchanged" + assert partial.probes, "the selection survives via the matched code S005hate" + assert ( + partial.empty_reason is None + ), "a surviving partial-unmatched intent produces no empty_reason" def test_intent_invalid_format_rejected(): @@ -328,6 +440,14 @@ def test_intent_invalid_format_rejected(): resolve("intent:zzz") +def test_intent_malformed_code_does_not_prune_candidates(): + unfiltered = set(resolve("probes.dan", skip_unknown=True).probes) + with_malformed = set(resolve("probes.dan,intent:zzz", skip_unknown=True).probes) + assert ( + with_malformed == unfiltered + ), "a malformed intent: code must not drive pruning; it stays in `rejected` instead" + + @pytest.mark.parametrize("token", ["intent:*", "intent:all"]) def test_intent_all_selector_not_rejected(token): res = resolve(f"probes.dan,{token}") From 39174dd6b326a61bf7a2b7ce1e79263eea5f6b41 Mon Sep 17 00:00:00 2001 From: Patricia Pampanelli Date: Mon, 24 Aug 2026 19:33:28 +0000 Subject: [PATCH 3/5] fix: prune probes on exclude-only intent selectors Signed-off-by: Patricia Pampanelli --- docs/source/configurable.rst | 5 +++ garak/_selection.py | 54 ++++++++++++++++++++++++++----- garak/command.py | 10 +++--- tests/cas/test_intent_run_spec.py | 3 +- tests/test_spec.py | 37 ++++++++++++++++++--- 5 files changed, 90 insertions(+), 19 deletions(-) diff --git a/docs/source/configurable.rst b/docs/source/configurable.rst index 3ddbd6a0a..3caac3508 100644 --- a/docs/source/configurable.rst +++ b/docs/source/configurable.rst @@ -196,6 +196,11 @@ Selectors (a category prefix is mandatory): ``intent:*`` or ``intent:all`` selects every intent and does not filter. When no ``intent:`` is given, the default scope ``S`` (the Safety branch) is injected at resolve time and, being the default, never prunes. + A lone ``-intent:`` (no ``intent:`` include) also filters: it removes + any already-selected probe whose own declared intent descends from + ````, compared against the resolved candidate set rather than the + injected default scope; ``IntentProbe`` subclasses are unaffected, since + they declare no fixed intent of their own. Typology expansion and detectorless filtering are governed by the ``run.*`` intent modifiers (``run.serve_detectorless_intents``). Only ``IntentProbe`` subclasses derive prompts from intents; giving an explicit ``intent:`` with no diff --git a/garak/_selection.py b/garak/_selection.py index 451803b24..820847979 100644 --- a/garak/_selection.py +++ b/garak/_selection.py @@ -105,6 +105,17 @@ def _intent_keeps(name: str, includes: List[str], excludes: List[str]) -> bool: return _intent_under(code, includes) and not _intent_under(code, excludes) +def _intent_excluded(name: str, excludes: List[str]) -> bool: + """True if ``name`` declares its own intent and it descends from an + excluded code. Used for exclude-only specs (no explicit ``intent:`` + include): pruning here compares each probe's own declared intent against + the excludes, so it never depends on the injected default scope. An + ``IntentProbe`` (no declared intent) is never excluded here; which + intents it actually serves is decided later by IntentService.""" + code = _plugins.plugin_info(name).get("intent") + return code is not None and _intent_under(code, excludes) + + def _empty_reason(spec: _spec.Spec) -> str: """Best-effort explanation of why a spec resolved to no probes.""" tier_ceilings = [int(s.value) for s in spec.include if s.kind == "tier"] @@ -133,6 +144,17 @@ def _empty_reason(spec: _spec.Spec) -> str: f"no selected probe carries intent '{codes}'; widen the probe " f"selection or drop the intent selector" ) + excluded_intent_codes = [ + s.value + for s in spec.exclude + if s.kind == "intent" and s.value.lower() not in ("*", "all") + ] + if excluded_intent_codes: + codes = ", ".join(excluded_intent_codes) + return ( + f"every selected probe's intent falls under the excluded code(s) " + f"'{codes}'; narrow the exclusion or widen the probe selection" + ) if any(s.kind in ("tag", "tier") for s in spec.include): return "no active probe matches the given tier/tag filters; widen the filters" return "every included probe was removed by an exclusion; adjust includes/excludes" @@ -145,8 +167,10 @@ def resolve_spec(spec: _spec.Spec, skip_unknown: bool = False) -> _spec.Resoluti tags). An explicit ``intent:`` include additionally filters probes by typology descendancy; the injected default scope never filters, and ``IntentProbe`` subclasses are pruned only when their - ``blocked_intent_spec`` covers every included code. This is the single - entry point used by the CLI and harnesses. + ``blocked_intent_spec`` covers every included code. A lone ``-intent:`` + (no include) also prunes: probes whose own declared intent descends from + an excluded code drop out of the already-resolved candidate set. This is + the single entry point used by the CLI and harnesses. """ rejected: List[str] = [] inactive_modules: List[str] = [] @@ -180,12 +204,15 @@ def resolve_spec(spec: _spec.Spec, skip_unknown: bool = False) -> _spec.Resoluti candidate = {p for p in candidate if _has_any_tag(p, tag_prefixes)} # Intent filter, mirroring the tag filter's OR-of-prefixes shape but matching - # by typology descendancy. Triggered only by explicit intent: - # includes (the injected DEFAULT_INTENT_SCOPE and exclude-only specs never - # prune); intent:* / intent:all are vacuous and do not filter. A probe that - # declares no intent (an IntentProbe, by convention) is pruned only when its + # by typology descendancy. An explicit intent: include filters the candidate + # set by descendancy (injected DEFAULT_INTENT_SCOPE never prunes); + # intent:* / intent:all are vacuous and do not filter. A probe that declares + # no intent (an IntentProbe, by convention) is pruned only when its # blocked_intent_spec covers every included code; otherwise which intents it - # actually serves is decided later by IntentService. + # actually serves is decided later by IntentService. A lone -intent: + # (no include) also prunes: any already-selected probe whose own declared + # intent descends from an excluded code drops out, compared against the + # resolved candidate set rather than the injected default scope. intent_includes = [s.value for s in spec.include if s.kind == "intent"] intent_excludes = [s.value for s in spec.exclude if s.kind == "intent"] # Malformed codes must not drive pruning: they stay in ``rejected`` (raised @@ -196,9 +223,20 @@ def resolve_spec(spec: _spec.Spec, skip_unknown: bool = False) -> _spec.Resoluti for c in intent_includes if c.lower() not in ("*", "all") and _spec.validate_intent_specifier(c) ] + intent_exclude_filter = [ + c + for c in intent_excludes + if c.lower() not in ("*", "all") and _spec.validate_intent_specifier(c) + ] if intent_filter: candidate = { - p for p in candidate if _intent_keeps(p, intent_filter, intent_excludes) + p + for p in candidate + if _intent_keeps(p, intent_filter, intent_exclude_filter) + } + elif intent_exclude_filter: + candidate = { + p for p in candidate if not _intent_excluded(p, intent_exclude_filter) } # Buffs: union of buffs.* includes (no implicit default) diff --git a/garak/command.py b/garak/command.py index f3e9a9e86..2000267bc 100644 --- a/garak/command.py +++ b/garak/command.py @@ -354,11 +354,11 @@ def _selection_has_intent_probe(probe_names) -> bool: def warn_unconsumed_intents(probe_names) -> None: """Warn once when an ``intent:`` selector was given explicitly but no - IntentProbe is in the selection. An explicit ``intent:`` include also narrows - the probe set (acting as a filter, like ``tag:``), whereas exclude-only and - ``intent:*``/``intent:all`` selectors do not; either way, only an IntentProbe - derives prompts from the intent typology, so whenever none is selected no - intent-derived prompts are generated.""" + IntentProbe is in the selection. An explicit ``intent:`` include or a lone + ``-intent:`` exclude can each narrow the probe set (acting as a filter, + like ``tag:``), whereas ``intent:*``/``intent:all`` selectors do not; + either way, only an IntentProbe derives prompts from the intent typology, + so whenever none is selected no intent-derived prompts are generated.""" from garak import _config if not getattr(_config.transient, "intents_explicit", False): diff --git a/tests/cas/test_intent_run_spec.py b/tests/cas/test_intent_run_spec.py index 72a51e337..3ecc9e9de 100644 --- a/tests/cas/test_intent_run_spec.py +++ b/tests/cas/test_intent_run_spec.py @@ -97,7 +97,8 @@ def test_warn_unconsumed_intents_fires_without_intent_probe(capsys): assert ( "no intent-derived prompts will be generated" in out ), "the warning states no intent-derived prompts result without an IntentProbe" - # the message must not assert narrowing, which is false for exclude-only and */all specs + # the message must not assert narrowing, since intent:*/intent:all and some + # exclude-only specs never prune the probe set assert "narrowed" not in out, "the warning must not claim narrowing" diff --git a/tests/test_spec.py b/tests/test_spec.py index 82aa93ee1..f0b95c5af 100644 --- a/tests/test_spec.py +++ b/tests/test_spec.py @@ -316,16 +316,35 @@ def test_intent_explicit_overrides_default(): def test_intent_default_does_not_filter_probe_set(): - # the injected default scope (no explicit intent:) and an exclude-only spec - # must both leave the probe set untouched -- only explicit includes filter. - # probes.dan carries T009ignore; a naive filter would erase the whole family. + # the injected default scope (no explicit intent:) never prunes on its own. family = {p for p in _active("probes") if p.startswith("probes.dan.")} injected_default = set(resolve("probes.dan").probes) - exclude_only = set(resolve("probes.dan,-intent:S004").probes) assert ( injected_default == family ), "the injected default scope must not prune probes" - assert exclude_only == family, "an exclude-only intent spec must not prune probes" + + +def test_intent_exclude_only_ignores_unrelated_code(): + # an exclude-only spec is a no-op when the excluded code doesn't match any + # selected probe's own intent -- S004 is unrelated to dan's T009ignore. + family = {p for p in _active("probes") if p.startswith("probes.dan.")} + exclude_only = set(resolve("probes.dan,-intent:S004").probes) + assert ( + exclude_only == family + ), "an exclude-only intent spec must not prune probes it doesn't match" + + +def test_intent_exclude_only_prunes_matching_probes(): + # unlike an unrelated exclude, -intent: alone still prunes when it matches + # a probe's own declared intent, compared against the already-resolved + # candidate set -- it does not depend on the injected default scope. Every + # active probes.dan.* class declares T009ignore, a child of T009. + family = {p for p in _active("probes") if p.startswith("probes.dan.")} + assert family, "fixture expects at least one active probes.dan.* plugin" + pruned = set(resolve("probes.dan,-intent:T009").probes) + assert ( + pruned == set() + ), "-intent:T009 must remove every dan.* probe, all of which descend from T009" def test_intent_descendancy_keeps_child_of_branch(): @@ -448,6 +467,14 @@ def test_intent_malformed_code_does_not_prune_candidates(): ), "a malformed intent: code must not drive pruning; it stays in `rejected` instead" +def test_intent_malformed_exclude_code_does_not_prune_candidates(): + unfiltered = set(resolve("probes.dan", skip_unknown=True).probes) + with_malformed = set(resolve("probes.dan,-intent:zzz", skip_unknown=True).probes) + assert ( + with_malformed == unfiltered + ), "a malformed -intent: code must not drive pruning either; it stays in `rejected`" + + @pytest.mark.parametrize("token", ["intent:*", "intent:all"]) def test_intent_all_selector_not_rejected(token): res = resolve(f"probes.dan,{token}") From 291802866ce2cdc34495f889261823e2d840e3b1 Mon Sep 17 00:00:00 2001 From: Patricia Pampanelli Date: Wed, 26 Aug 2026 13:12:21 +0000 Subject: [PATCH 4/5] docs: clarify intent:/-intent: as complementary set operations Signed-off-by: Patricia Pampanelli --- docs/source/configurable.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/source/configurable.rst b/docs/source/configurable.rst index 3caac3508..9f99c4e23 100644 --- a/docs/source/configurable.rst +++ b/docs/source/configurable.rst @@ -201,6 +201,11 @@ Selectors (a category prefix is mandatory): ````, compared against the resolved candidate set rather than the injected default scope; ``IntentProbe`` subclasses are unaffected, since they declare no fixed intent of their own. + Both directions test the same descendancy, so they are complementary set + operations on the candidate: ``intent:`` is an intersection, + ``-intent:`` alone is a difference. When every candidate probe already + descends from ````, the include is a no-op and the exclude empties + the selection. Typology expansion and detectorless filtering are governed by the ``run.*`` intent modifiers (``run.serve_detectorless_intents``). Only ``IntentProbe`` subclasses derive prompts from intents; giving an explicit ``intent:`` with no From 5db239ec87eb2d626e4010d6f3a7d4eccc519eac Mon Sep 17 00:00:00 2001 From: Patricia Pampanelli Date: Wed, 26 Aug 2026 13:31:21 +0000 Subject: [PATCH 5/5] fix: don't warn on intent: when a probe already declares it Signed-off-by: Patricia Pampanelli --- garak/command.py | 19 ++++++++++++------- tests/cas/test_intent_run_spec.py | 4 +++- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/garak/command.py b/garak/command.py index 2000267bc..70b4bd6b8 100644 --- a/garak/command.py +++ b/garak/command.py @@ -353,18 +353,23 @@ def _selection_has_intent_probe(probe_names) -> bool: def warn_unconsumed_intents(probe_names) -> None: - """Warn once when an ``intent:`` selector was given explicitly but no - IntentProbe is in the selection. An explicit ``intent:`` include or a lone - ``-intent:`` exclude can each narrow the probe set (acting as a filter, - like ``tag:``), whereas ``intent:*``/``intent:all`` selectors do not; - either way, only an IntentProbe derives prompts from the intent typology, - so whenever none is selected no intent-derived prompts are generated.""" - from garak import _config + """Warn once when an ``intent:`` selector was given explicitly but neither + an IntentProbe nor a probe with its own declared ``intent`` is in the + selection. A probe with its own ``intent`` (e.g. ``dan.AutoDANCached``) + already consumes the axis by being selected for it, even though it + derives no stub-based prompts.""" + from garak import _config, _plugins if not getattr(_config.transient, "intents_explicit", False): return if _selection_has_intent_probe(probe_names): return + if any( + _plugins.plugin_info(name).get("intent") is not None + for name in probe_names + if name.startswith("probes.") + ): + return msg = ( "intent: selector(s) given but no IntentProbe is selected, so no " "intent-derived prompts will be generated (add an IntentProbe, " diff --git a/tests/cas/test_intent_run_spec.py b/tests/cas/test_intent_run_spec.py index 3ecc9e9de..2ac67337b 100644 --- a/tests/cas/test_intent_run_spec.py +++ b/tests/cas/test_intent_run_spec.py @@ -88,8 +88,10 @@ def test_empty_axis_yields_no_prompts(): def test_warn_unconsumed_intents_fires_without_intent_probe(capsys): import garak.command as command + # probes.base.Probe declares no intent of its own (unlike e.g. dan.* probes, + # which default to T009ignore) -- nothing here engages the intent axis at all. garak._config.transient.intents_explicit = True - command.warn_unconsumed_intents(["probes.dan.DanInTheWild"]) + command.warn_unconsumed_intents(["probes.base.Probe"]) out = capsys.readouterr().out assert ( "no IntentProbe is selected" in out