diff --git a/CHANGELOG.md b/CHANGELOG.md index 87ceeb4..c32f97c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Changed +- **Breaking (experimental section-mode JSON only):** section mode now scores each candidate with an independent Jev Noul question instead of one Choice question (owner decision D2, openspec `update-jev-blind-validation`). In `fix-gaps.json` the page `diagnosis` reports `scorer: "noul"` and `scores` in place of `probabilities` / `confidence`; `verification` reports `score_before` / `score_after` in place of `p_before` / `p_after`. `already_best` and `won` require the best own section to score strictly above every source (ties count as losing). Page mode, the default, is unchanged. +- Jev now sees neutral candidate ids (`c01`, `c02`, …) in an order derived from the query and the texts, so it cannot tell your sections from the cited sources. Diagnosis and verification use the same ids. +- `jev_validation` reports the pre-registered v4 validation of the configuration section mode runs (`protocol: "v4"`, `scorer: "noul"`, `ids: "blind"`, `unit: "sections"`): mean AUC 0.559 (95% CI 0.489–0.637, 30 queries, 2026-10-15), below the 0.65 gate, so `status` stays `"experimental"` and every Jev diagnosis and verification keeps `"experimental": true`. The 2.2.0 figures move to `jev_validation.previous`, labeled `choice, leaky ids, page excerpts`. +- `eval/jev_selection` scores with the product function on sections (`own_source`), adds `--ids {blind,leaky}`, `--prereg-sha` (required for live runs) and `--repo-root`, and drops `--scorer`. +- The offline mock answers Noul questions by query-word overlap instead of a constant 0.9. + +### Added +- `eval/jev_selection/PREREG-v4.md` (registered 2026-10-08, commit `56ef2db`) and `queries-v3.json`: the v4 validation protocol, the query list and its hash, the gate and the consequences of passing and failing. The leak diagnostic and the v4 result are in `eval/jev_selection/README.md`. + +### Known limitations +- Jev diagnosis and verification in section mode **failed** their pre-registered validation (mean AUC 0.56 against a 0.65 gate). When Jev judged the own section best, Perplexity cited the page in 1 of 6 queries. Treat `already_best` as a weak signal and confirm with `remeasure`. Section mode stays experimental and opt-in; page mode, the default, does not use Jev. + ## [2.2.0] - 2026-09-26 ### Known limitations diff --git a/egeo/gaps.py b/egeo/gaps.py index 33893f5..1d6b382 100644 --- a/egeo/gaps.py +++ b/egeo/gaps.py @@ -261,35 +261,60 @@ def run_fix_gaps( "rewriter_error", # the rewriter LLM failed for this page (LLMError); nothing written ) -# The figures below are the pre-registered v2 run of the *choice* scorer: the same single Choice question -# (judge.select_best) that section mode uses to diagnose and verify. The gate-deciding noul scorer -# (AUC 0.637) is not what section mode runs, so its numbers are not quoted here. +# v4 (openspec update-jev-blind-validation): section mode scores every candidate with an independent Noul +# question under blind ids (c01, c02, ...). Its pre-registered validation (2026-10-15) failed the gate. The +# 2.2.0 number measured a different configuration (single Choice question, ids that revealed ownership, +# full-page excerpts), so it is quoted only as an earlier configuration, never as this version's result. SECTIONS_NOTE = ( "Section mode: the Jev comparison between your sections and the sources the tracker says were cited is " - "EXPERIMENTAL. It scores text competitiveness (which candidate's text best answers the query) and does not " - "model authority or links, so it is not a prediction of citation. In a pre-registered test (30 queries, " - "2026-09-25) the same single-choice comparison separated cited from uncited sources with mean AUC 0.62 " - "(95% CI 0.56-0.67), below the 0.65 bar, and matched the engine's cited/not-cited outcome for the own page " - "in only 17% of the queries that had one. That test compared full-page excerpts, not sections " - "(eval/jev_selection/README.md). Confirm any change by re-checking the queries in `remeasure` with your tracker." + "EXPERIMENTAL. It scores text competitiveness (how well each candidate's text answers the query) and does " + "not model authority or links, so it is not a prediction of citation. The configuration this version runs " + "(an independent score per candidate, neutral ids that hide which text is yours, sections) scored mean AUC " + "0.56 (95% CI 0.49-0.64, 30 queries, 2026-10-15) in a pre-registered test, below the 0.65 bar " + "(eval/jev_selection/README.md). Treat `already_best` and the re-check as weak signals. An earlier " + "configuration (single-choice comparison, ids that revealed ownership, full-page excerpts) scored mean AUC " + "0.62 (95% CI 0.56-0.67, 30 queries, 2026-09-25). Confirm any change by re-checking the queries in " + "`remeasure` with your tracker." ) -# Copied verbatim into every section-mode report as "jev_validation"; every non-null page "diagnosis" and -# "verification" also carries "experimental": True (owner decision 2026-09-25, plan v3). +# Copied verbatim into every section-mode report as "jev_validation". While "status" != "validated", every +# non-null page "diagnosis" and "verification" carries "experimental": True (one switch, flipped only by the +# pre-registered outcome, plan v4 §4). The v4 run failed the gate, so the status stays "experimental". JEV_VALIDATION = { "status": "experimental", - "scorer": "choice", - "mean_auc": 0.619, - "ci95": [0.561, 0.674], - "own_accuracy": 0.167, + "protocol": "v4", + "scorer": "noul", + "ids": "blind", + "unit": "sections", + "mean_auc": 0.559, + "ci95": [0.489, 0.637], + "own_accuracy": 0.5625, "n_queries": 30, - "unit": "full-page excerpts (<= 1500 chars), not sections", "gate_auc": 0.65, + "gate_min_queries": 25, "gate_passed": False, - "date": "2026-09-25", + "prereg_sha": "56ef2db55125c691bd72c7f350ce8dc3dae31d27", + "date": "2026-10-15", "details": "eval/jev_selection/README.md", + "previous": { + "config": "choice, leaky ids, page excerpts", + "mean_auc": 0.619, + "ci95": [0.561, 0.674], + "own_accuracy": 0.167, + "n_queries": 30, + "gate_passed": False, + "date": "2026-09-25", + }, } + +def _flag(d): + """Add "experimental": True to a diagnosis/verification dict while Jev is not validated.""" + if JEV_VALIDATION["status"] != "validated": + d["experimental"] = True + return d + + # Every page dict gets an "offpage" key (plan v2). For status "already_best" it is # {"reason": OFFPAGE_REASON, "message": OFFPAGE_MESSAGE, "sources": [url of every ok fetched doc, in fetch order]} # and for every other status it is None. The CLI line for an already_best page is @@ -327,11 +352,11 @@ def _section_page(page, *, out_dir, jev_client, rewrite_fn, fetch_fn, exclude_do "offpage": None, "reasons": []} try: diag = judge.diagnose(page.query, own, srcs, jev_client) - if diag.selection is None or diag.status == "already_best": - sel = diag.selection - res["diagnosis"] = None if sel is None else {"status": diag.status, "winner": sel.winner, "winner_kind": sel.winner_kind, - "confidence": sel.confidence, "probabilities": dict(sel.probabilities), - "target_section": None, "experimental": True} + if diag.scored is not None: + lead = judge.leader(diag.scored, own, srcs) + res["diagnosis"] = _flag({"status": diag.status, "winner": lead.id, "winner_kind": lead.kind, "scorer": "noul", + "scores": dict(diag.scored.scores), "target_section": None}) + if diag.scored is None or diag.status == "already_best": res["status"] = diag.status if diag.status == "already_best": res["offpage"] = {"reason": OFFPAGE_REASON, "message": OFFPAGE_MESSAGE, @@ -339,10 +364,7 @@ def _section_page(page, *, out_dir, jev_client, rewrite_fn, fetch_fn, exclude_do return res sid = diag.target_id[len("own_"):] sec = next(s for s in secs if s.id == sid) - sel = diag.selection - res["diagnosis"] = {"status": diag.status, "winner": sel.winner, "winner_kind": sel.winner_kind, "confidence": sel.confidence, - "probabilities": dict(sel.probabilities), "target_section": {"id": sid, "heading": sec.heading}, - "experimental": True} + res["diagnosis"]["target_section"] = {"id": sid, "heading": sec.heading} try: new_text = rewrite_fn(page.query, sec.text, title) except LLMError as exc: @@ -358,9 +380,10 @@ def _section_page(page, *, out_dir, jev_client, rewrite_fn, fetch_fn, exclude_do if not v.accepted: res["status"] = "rejected_fidelity_judge"; return res own_after = [judge.Candidate(c.id, c.kind, c.label, new_text if c.id == diag.target_id else c.text) for c in own] - ver = judge.verify(page.query, own_after, srcs, diag.target_id, sel, jev_client) - res["verification"] = {"p_before": ver.p_before, "p_after": ver.p_after, "winner_after": ver.winner_after, - "winner_after_kind": ver.winner_after_kind, "outcome": ver.outcome, "experimental": True} + ver = judge.verify(page.query, own_after, srcs, diag.target_id, diag.scored, jev_client) + res["verification"] = _flag({"score_before": ver.score_before, "score_after": ver.score_after, + "winner_after": ver.winner_after, "winner_after_kind": ver.winner_after_kind, + "outcome": ver.outcome}) if ver.outcome == "worse": res["status"] = "rejected_worse"; return res except (JevProviderError, JevConfigError) as exc: diff --git a/egeo/jev.py b/egeo/jev.py index 4e93746..eb19a82 100644 --- a/egeo/jev.py +++ b/egeo/jev.py @@ -37,7 +37,9 @@ to 4 decimals. ``choice`` = highest probability (ties -> first option in criteria order). ``confidence = max(0, min(1, (n * peak - 1) / (n - 1)))`` with n options (n == 1 -> 1.0), where ``peak`` is the UNROUNDED highest probability; round the confidence to 4 decimals. -- noul: 0.9. +- noul: the share of query words found in the candidate the question points at: + ``round(|words(query) ∩ words(candidates[qid])| / max(1, |words(query)|), 4)`` when ``state`` is a + dict with ``"query"`` and a dict ``"candidates"`` containing the question id ``qid``; else 0.0. - model "mock", usage zeros. Returns status 200. ``make_client(mock=..., model=...)``: mock -> ``JevClient(MockJevTransport(), model="mock")``; @@ -151,7 +153,12 @@ def post_json(self, url, body): answers[qid] = {"type": "choice", "choice": best, "confidence": round(conf, 4), "probabilities": {o: round(p, 4) for o, p in zip(opts, raw)}} else: - answers[qid] = {"type": "noul", "noul": 0.9} + cands = state.get("candidates") if isinstance(state, dict) else None + score = 0.0 + if isinstance(cands, dict) and qid in cands: + qw = _words(state.get("query", "")) + score = round(len(qw & _words(cands[qid])) / max(1, len(qw)), 4) + answers[qid] = {"type": "noul", "noul": score} return 200, {"model": "mock", "answers": answers, "usage": {"input_tokens": 0, "output_tokens": 0}} diff --git a/egeo/judge.py b/egeo/judge.py index 8714420..3375cf4 100644 --- a/egeo/judge.py +++ b/egeo/judge.py @@ -1,56 +1,71 @@ """Jev decisions for section-mode ``fix-gaps``: diagnose, fidelity judge, verify. Jev judges; it never writes. All thresholds are the constants below and must not be tuned per run. - -``select_best(query, candidates, client)``: -- Requires >= 2 candidates with unique ids, else ``ValueError``. -- One Choice question with id ``SELECT_QID`` and ``SELECT_INSTRUCTIONS``. -- state = ``{"query": query, "candidates": {c.id: c.text[:MAX_CANDIDATE_CHARS] for c}}``. -- criteria = ``{c.id: f"The candidate whose text is candidates.{c.id}"}`` (neutral labels; the - model must not learn which candidates are ours). -- Returns ``Selection(winner=answer.choice, winner_kind=, - probabilities=answer.probabilities, confidence=answer.confidence)``. +v4 (openspec ``update-jev-blind-validation``; owner decision D2 = Noul, 2026-10-07). + +``blind_ids(query, candidates) -> {original id: "cNN"}``: sort candidates by +``sha256(f"{query}\n{c.text}".encode()).hexdigest()``, ties by original id, and number them +``c01``, ``c02``, ... The order depends only on the query and the texts, never on ownership. + +``score_candidates(query, candidates, client, *, id_map=None, ids="blind") -> Scored``: +- Requires >= 2 candidates with unique ids, else ``ValueError``; ``ids`` is ``"blind"`` or + ``"leaky"``, else ``ValueError``. +- Sent id per candidate: ``id_map[c.id]`` when ``id_map`` is given (it must cover every candidate, + else ``ValueError``); else ``blind_ids(query, candidates)``; with ``ids="leaky"`` (eval leak + diagnostic only) the original id. +- ONE request. state = ``{"query": query, "candidates": {sent: c.text[:MAX_CANDIDATE_CHARS]}}`` in sent-id + order; questions = ``{sent: noul_question(NOUL_INSTRUCTIONS.format(cid=sent))}``. Nothing else (no + label, host or kind) reaches Jev. +- Returns ``Scored(scores={original id: noul}, id_map={original id: sent id}, model=)``. + +``leader(scored, own, sources) -> Candidate``: the best own candidate (ties -> first in ``own``) when +its score is strictly above every source score; otherwise the best source (ties -> first in +``sources``). A tie is never an own win: ``already_best`` is an off-page claim, so it must be earned. ``diagnose(query, own, sources, client)``: -- no own candidates -> ``Diagnosis(selection=None, target_id=None, status="no_own_candidates")``. +- no own candidates -> ``Diagnosis(scored=None, target_id=None, status="no_own_candidates")``. - no sources -> ``Diagnosis(None, None, "no_competitor_sources")`` (Jev is not called). -- otherwise ``sel = select_best(query, [*own, *sources], client)``; - winner_kind == "own" -> status ``"already_best"``, target None; - else status ``"loses"``, target = own candidate with the highest probability (ties -> the one - that comes first in ``own``). +- otherwise ``scored = score_candidates(query, [*own, *sources], client)``; when the best own score + is strictly above the best source score -> ``"already_best"``, target None; else ``"loses"``, + target = own candidate with the highest score (ties -> first in ``own``). ``judge_fidelity(original, rewrite, client)``: - One Choice question ``FIDELITY_QID`` with ``FIDELITY_INSTRUCTIONS`` and ``FIDELITY_CRITERIA``; state = ``{"original": original, "rewrite": rewrite}``. - ``accepted = verdict == "faithful" and confidence >= FIDELITY_GATE``. -- Not validated directly: ``eval/jev_selection`` measured source *selection*, not fidelity. The judge +- Not validated directly: ``eval/jev_selection`` measures source *scoring*, not fidelity. The judge can only reject, and it runs after the deterministic rules in ``egeo.fidelity``. ``verify(query, own_after, sources, target_id, before, client)``: -- ``after = select_best(query, [*own_after, *sources], client)`` (same ids as before). -- ``p_before = before.probabilities.get(target_id, 0.0)``; ``p_after`` likewise from ``after``. -- outcome: winner_kind of ``after`` == "own" -> ``"won"``; elif p_after - p_before >= IMPROVE_DELTA - -> ``"improved"``; elif p_before - p_after >= IMPROVE_DELTA -> ``"worse"``; else ``"no_change"``. - Compare with a 1e-9 tolerance so that exactly 0.05 counts. +- ``after = score_candidates(query, [*own_after, *sources], client, id_map=before.id_map)``: the + ids are the diagnosis ids, computed once from the original texts. +- ``score_before = before.scores[target_id]``; ``score_after`` likewise from ``after``. +- outcome: best own strictly above best source -> ``"won"``; elif score_after - score_before >= + IMPROVE_DELTA -> ``"improved"``; elif score_before - score_after >= IMPROVE_DELTA -> ``"worse"``; + else ``"no_change"``. Compare with a 1e-9 tolerance so that exactly 0.05 counts. ``"won"`` means ANY own section wins after the rewrite, not necessarily ``target_id``. + ``winner_after`` / ``winner_after_kind`` come from ``leader``. ``own_candidates(sections)``: sections with ``word_count >= MIN_SECTION_WORDS`` become ``Candidate(id=f"own_{s.id}", kind="own", label=s.heading or "(intro)", text=s.text)``. ``source_candidates(docs)``: docs with ``ok`` become ``Candidate(id=f"src_{i}", kind="source", label=host, text=doc.text)`` with i = 1, 2, ... counting only ok docs, host = ``sources.host_of(url)``. +These ids and labels stay in code: Jev only ever sees the blind ids. """ from __future__ import annotations from dataclasses import dataclass +import hashlib from typing import Dict, List, Optional, Sequence from .jev import JevClient -SELECT_QID = "best_source" -SELECT_INSTRUCTIONS = ( - "Which candidate best answers `query`, so that an AI answer engine would quote or cite it " - "as a source? Judge only the candidate text given in state." +# Moved verbatim from eval/jev_selection/run.py (v2), so the eval and the product ask the same question. +NOUL_INSTRUCTIONS = ( + "Would an AI answer engine cite candidates.{cid} as a source when answering `query`? " + "Answer yes only if that candidate's own text directly and credibly answers `query`." ) +IDS_MODES = ("blind", "leaky") FIDELITY_QID = "fidelity" FIDELITY_INSTRUCTIONS = "Compare `rewrite` with `original`. How does the rewrite relate to the original?" FIDELITY_CRITERIA: Dict[str, str] = { @@ -76,16 +91,15 @@ class Candidate: @dataclass(frozen=True) -class Selection: - winner: str - winner_kind: str - probabilities: Dict[str, float] - confidence: float +class Scored: + scores: Dict[str, float] # original candidate id -> Noul score + id_map: Dict[str, str] # original candidate id -> id Jev saw + model: str = "" # model string the API returned @dataclass(frozen=True) class Diagnosis: - selection: Optional[Selection] + scored: Optional[Scored] target_id: Optional[str] status: str # "loses" | "already_best" | "no_own_candidates" | "no_competitor_sources" @@ -99,14 +113,14 @@ class FidelityVerdict: @dataclass(frozen=True) class Verification: - p_before: float - p_after: float + score_before: float + score_after: float winner_after: str winner_after_kind: str outcome: str # "won" | "improved" | "no_change" | "worse" -from .jev import choice_question +from .jev import choice_question, noul_question from .sources import host_of @@ -122,15 +136,40 @@ def source_candidates(docs): return out -def select_best(query, candidates, client): - ids = [c.id for c in candidates] - if len(ids) < 2 or len(set(ids)) != len(ids): +def blind_ids(query, candidates): + order = sorted(candidates, key=lambda c: (hashlib.sha256(f"{query}\n{c.text}".encode()).hexdigest(), c.id)) + return {c.id: f"c{i:02d}" for i, c in enumerate(order, 1)} + + +def score_candidates(query, candidates, client, *, id_map=None, ids="blind"): + orig = [c.id for c in candidates] + if len(orig) < 2 or len(set(orig)) != len(orig): raise ValueError("need >= 2 unique candidates") - state = {"query": query, "candidates": {c.id: c.text[:MAX_CANDIDATE_CHARS] for c in candidates}} - q = choice_question(SELECT_INSTRUCTIONS, {c.id: f"The candidate whose text is candidates.{c.id}" for c in candidates}) - a = client.ask(state, {SELECT_QID: q}).answers[SELECT_QID] - kind = {c.id: c.kind for c in candidates}[a.choice] - return Selection(a.choice, kind, dict(a.probabilities), a.confidence) + if ids not in IDS_MODES: + raise ValueError(f"unknown ids mode: {ids}") + if id_map is not None: + if set(id_map) != set(orig) or len(set(id_map.values())) != len(orig): + raise ValueError("id_map must map every candidate to a unique id") + m = dict(id_map) + elif ids == "leaky": + m = {i: i for i in orig} + else: + m = blind_ids(query, candidates) + sent = sorted(candidates, key=lambda c: m[c.id]) if ids == "blind" or id_map is not None else list(candidates) + state = {"query": query, "candidates": {m[c.id]: c.text[:MAX_CANDIDATE_CHARS] for c in sent}} + questions = {m[c.id]: noul_question(NOUL_INSTRUCTIONS.format(cid=m[c.id])) for c in sent} + resp = client.ask(state, questions) + return Scored({c.id: resp.answers[m[c.id]].noul for c in candidates}, m, resp.model) + + +def _best(cands, scores): + return max(cands, key=lambda c: (scores.get(c.id, 0.0), -cands.index(c))) + + +def leader(scored, own, sources): + sc = scored.scores + bo, bs = _best(own, sc), _best(sources, sc) + return bo if sc.get(bo.id, 0.0) > sc.get(bs.id, 0.0) else bs def diagnose(query, own, sources, client): @@ -138,11 +177,10 @@ def diagnose(query, own, sources, client): return Diagnosis(None, None, "no_own_candidates") if not sources: return Diagnosis(None, None, "no_competitor_sources") - sel = select_best(query, [*own, *sources], client) - if sel.winner_kind == "own": - return Diagnosis(sel, None, "already_best") - best = max(own, key=lambda c: (sel.probabilities.get(c.id, 0.0), -own.index(c))) - return Diagnosis(sel, best.id, "loses") + scored = score_candidates(query, [*own, *sources], client) + if leader(scored, own, sources).kind == "own": + return Diagnosis(scored, None, "already_best") + return Diagnosis(scored, _best(own, scored.scores).id, "loses") def judge_fidelity(original, rewrite, client): @@ -152,15 +190,16 @@ def judge_fidelity(original, rewrite, client): def verify(query, own_after, sources, target_id, before, client): - after = select_best(query, [*own_after, *sources], client) - pb, pa = before.probabilities.get(target_id, 0.0), after.probabilities.get(target_id, 0.0) + after = score_candidates(query, [*own_after, *sources], client, id_map=before.id_map) + sb, sa = before.scores.get(target_id, 0.0), after.scores.get(target_id, 0.0) + lead = leader(after, own_after, sources) eps = 1e-9 - if after.winner_kind == "own": + if lead.kind == "own": o = "won" - elif pa - pb >= IMPROVE_DELTA - eps: + elif sa - sb >= IMPROVE_DELTA - eps: o = "improved" - elif pb - pa >= IMPROVE_DELTA - eps: + elif sb - sa >= IMPROVE_DELTA - eps: o = "worse" else: o = "no_change" - return Verification(pb, pa, after.winner, after.winner_kind, o) + return Verification(sb, sa, lead.id, lead.kind, o) diff --git a/eval/jev_selection/PREREG-v4.md b/eval/jev_selection/PREREG-v4.md new file mode 100644 index 0000000..6a4abfd --- /dev/null +++ b/eval/jev_selection/PREREG-v4.md @@ -0,0 +1,65 @@ +# Pre-registration — Jev validation v4 + +**Status: REGISTERED** (owner approval 2026-10-08). The SHA of the commit that set this line is the +`--prereg-sha` of every v4 run. No live v4 run (leak diagnostic included) happened before this commit. +Nothing in this file changes after it. + +Change: `openspec/changes/update-jev-blind-validation` (design D3–D5). +Plan: `docs/plans/2026-09-26-jev-blind-validation-plan-v4.md` §3–§4. + +## What is measured + +Whether `egeo.judge.score_candidates`, the function section mode runs to diagnose and verify, scores +the sources Perplexity cites above the sources it does not cite, when Jev cannot tell which +candidates are ours. + +| Setting | Value | +|---|---| +| Scorer | `noul` (one independent Noul question per candidate, `egeo.judge.NOUL_INSTRUCTIONS`) | +| Ids | `blind` (`c01`, `c02`, … ordered by `sha256(query + "\n" + text)`) | +| Unit | `sections` for items with `own_source`, `page_excerpt` otherwise (2 items) | +| Candidate cap | `egeo.judge.MAX_CANDIDATE_CHARS` = 1500 chars; `MIN_SECTION_WORDS` = 30 | +| Sources per class | `MAX_PER_CLASS` = 8 cited, 8 uncited | +| Metric | mean per-query ROC AUC over sources (cited = 1, uncited SERP = 0), Mann-Whitney, ties 0.5 | +| Interval | bootstrap 95% CI, 2000 resamples, seed 0 | +| Jev model | the request asks for `jev-latest`; the exact model string the API returns is recorded per query (`jev_model`) and per run (`jev_models_returned`) | + +## Queries + +`eval/jev_selection/queries-v3.json`: the same 30 queries as v2 +(`~/Sync/mkt/egeo-expansion/jev-selection/queries-v2.json`), in the same order, with the same +`own_url`, plus `own_source` (the page's Markdown under `site/src/content/docs/`) for the 16 queries +whose own page has a local source. The 2 own pages without one (`https://egeoagents.com/` and the +GitHub repository) stay `page_excerpt` and are excluded from own metrics. + +sha256 of `queries-v3.json`: `094e056c7ee12b4f75af36b57a8cfb5a7639f303bf526d8d03c0e2f5d4fb1718` + +`own_source` is read from a checkout of `main` at the last commit before the v3 collection started +(`git rev-list -1 --before="" main`). That SHA is recorded in the README next +to the result, so the sections scored are the ones that were live when Perplexity answered. + +## Gate + +`gate_passed = mean_auc >= 0.65 AND n_scored >= 25`, on the blind-id run of dataset v3, with this +file's commit SHA passed as `--prereg-sha`. Unchanged from v2, for comparability. + +Own metrics (`own_wins` × `egeo_cited` confusion table, `own_accuracy` with a Wilson 95% interval, +sections items only) are reported and **do not gate**. + +## Steps, each run once + +1. **Leak diagnostic (non-gating)** on `dataset-v2-2026-09-25.json`: `--ids leaky`, then `--ids blind`, + same day. Report Δ own-win rate, Δ mean own rank and Δ mean AUC. It changes nothing in this file. +2. **Collect dataset v3 once**, on a date after the registering commit, with + `~/.hermes/scripts/egeo_jev_selection_collect.py --queries eval/jev_selection/queries-v3.json` + (the collector must copy `own_source` into each item). Save it as `dataset-v3-.json`. + No re-collection. +3. **Evaluate once:** + `python -m eval.jev_selection.run --dataset …/dataset-v3-.json --out …/results-v3-.json --prereg-sha --repo-root `. + A run that dies before writing a result (network, quota) may be restarted. A completed run is + final. No re-runs, prompt edits or threshold changes after seeing numbers. + +## Consequences, fixed in advance (design D5, verbatim) + +- **PASS:** `JEV_VALIDATION` gets the v4 numbers with `"status": "validated"` and `"gate_passed": true`; the `experimental` flags and the 2.2.0 "Known limitations" entry are removed. The report still states that Jev judges text competitiveness and is not a prediction of citation. The `fix-gaps` default does **not** change (that needs `remeasure` evidence). +- **FAIL:** `JEV_VALIDATION` gets the v4 numbers, `"status": "experimental"`, and the labels stay. The owner then chooses, without another run: keep section mode experimental, or remove the Jev diagnosis and verification and keep only the deterministic fidelity rules. diff --git a/eval/jev_selection/README.md b/eval/jev_selection/README.md index ea854a3..d1eaeee 100644 --- a/eval/jev_selection/README.md +++ b/eval/jev_selection/README.md @@ -5,6 +5,74 @@ validation gate for section-mode `fix-gaps` (`openspec/specs/citation-gap-fixing Section mode uses Jev to judge **text competitiveness** — whose text best answers the query — *not* to predict citations (Jev sees text only; engines also weigh authority and links). +## v4 — the path section mode actually runs + +v2 (below) measured a configuration the product does not run: its own copy of the Noul question, on +full-page excerpts, with ids that told Jev which candidate was ours (`own` next to `src_N`). From v4 +the harness calls the product function `egeo.judge.score_candidates`: + +- **Blind ids.** Jev sees `c01`, `c02`, … ordered by `sha256(query + "\n" + text)`; the mapping back + to own section or source stays in code. `--ids leaky` reproduces the 2.2.0 ids for the leak + diagnostic only and never gates. +- **Sections.** Items with `own_source` (repo-relative Markdown) are scored as section mode builds + them: frontmatter removed, sections of ≥ 30 words. Other items keep one `page_excerpt` candidate. +- **Own metrics (not gating):** `own_wins` (best own section strictly above every source, the + product's `already_best` rule), an `own_wins` × `egeo_cited` confusion table and `own_accuracy` + with a Wilson 95% interval, on `sections` items only. +- **Pre-registration:** `PREREG-v4.md` and `queries-v3.json`. A live run needs `--prereg-sha`; a run + without one never passes the gate. + +The pre-registered v4 run is done (2026-10-15, GATE FAIL, below). Section mode reports it in +`jev_validation` (`"status": "experimental"`, `"gate_passed": false`) and quotes the v2 Choice +figures only as an earlier configuration, under `previous`. + +### v4 step 1 — leak diagnostic (non-gating), 2026-10-08 + +Dataset v2 as collected (page excerpts, 18 queries with an own page), product scorer, `--prereg-sha 56ef2db`, +one run per mode, same day. Model returned: `jev-1.13.0`. Cost: 30 requests and $0.0071 per run. + +| | leaky ids (2.2.0) | blind ids | Δ (blind − leaky) | +|---|---|---|---| +| mean AUC (sources) | 0.639 [0.564, 0.709] | 0.626 [0.538, 0.712] | −0.013 | +| own wins (best own > every source) | 4/18 (0.222) | 3/18 (0.167) | −0.056 | +| mean own rank | 5.72 | 5.28 | −0.44 | + +Reading: hiding ownership barely moves source AUC, and the intervals overlap almost fully. Own wins +drop by one query. The mean own rank also improves a little, so the leak does not show a consistent +bias in either direction at this sample size. As pre-registered, this changes nothing: the blind run +here is on dataset v2, so its `gate_passed` field is not the v4 gate. Files: +`results-v4-leak-{leaky,blind}-2026-10-08.json`. + +### v4 steps 2–3 — dataset v3, pre-registered run, 2026-10-15 — GATE FAIL + +Collected once with `queries-v3.json` (sha256 `094e056c…1718`, as registered), collection start +2026-10-15T23:39:02Z: 30 queries, 0 empty Perplexity answers, 10 cited and 10 SERP URLs per query, +E-GEO named in 5/30 answers. `own_source` read from `main` at +`ffd5b49802c800ac1c82ea3df3a95a2c50323c62` (`git rev-list -1 --before= main`). Evaluated +once: `--ids blind --prereg-sha 56ef2db55125c691bd72c7f350ce8dc3dae31d27`. Model returned: +`jev-1.13.0`. Cost: 30 requests, 196,800 in-tokens, $0.0083. + +| Metric | v4 (blind, sections) | gate | +|---|---|---| +| mean AUC (sources) | **0.559** [0.489, 0.637] | ≥ 0.65 | +| n_scored | 30/30 | ≥ 25 | +| units | 16 `sections`, 2 `page_excerpt` | | +| own wins × egeo_cited (sections, n=16) | tp 1, fp 5, fn 2, tn 8 | not gating | +| own_accuracy (sections) | 0.5625 [0.332, 0.769] | not gating | + +Worst AUCs: `AI search visibility tracking tools` 0.20, `how to self-host an AI agent on a VPS` +0.22, `best open source AI coding agents 2026` 0.23. Best: `eGEOagents` 1.00, `open source tool to +rank in ChatGPT and Perplexity` 1.00, `what is generative engine optimization` 0.90. + +Reading: the whole 95% interval sits below the 0.65 gate, and the point estimate is lower than v2's +0.637 on page excerpts. When Jev picks our section as best, Perplexity cites us in 1 of 6 queries. +As pre-registered (design D5, FAIL), the result is final: no re-run, prompt or threshold change. +`JEV_VALIDATION` gets these numbers with `"status": "experimental"` and the labels stay; the owner +then chooses, without another run, to keep section mode experimental or remove the Jev diagnosis and +verification. Owner decision (2026-10-16): keep section mode experimental; demoting `already_best` +from a verdict to a warning goes to a separate change. Files: `~/Sync/mkt/egeo-expansion/jev-selection/dataset-v3-2026-10-15.json`, +`results-v3-2026-10-15.json`. + ## v1 → v2 (2026-09-25 review) v1 (one Choice question over all candidates, n=10) reported mean AUC 0.6493 against a 0.65 gate and @@ -46,10 +114,12 @@ set -a; . <(grep -E '^TYPESAFE_API_KEY=' ~/.hermes/.env); set +a python -m eval.jev_selection.run \ --dataset ~/Sync/mkt/egeo-expansion/jev-selection/dataset-v2-2026-09-25.json \ --out ~/Sync/mkt/egeo-expansion/jev-selection/results-v2-2026-09-25.json -# comparison only (does not decide the gate): -python -m eval.jev_selection.run --dataset … --out … --scorer choice ``` +The v2 commands above predate v4: since v4 the harness has no `--scorer` flag (Noul only, via +`egeo.judge.score_candidates`), and a live run needs `--prereg-sha`. The v4 commands are in +`PREREG-v4.md`. + ## Result — 2026-09-25 — GATE FAIL | Metric | Noul (gate) | Choice (comparison) | diff --git a/eval/jev_selection/queries-v3.json b/eval/jev_selection/queries-v3.json new file mode 100644 index 0000000..e762ea2 --- /dev/null +++ b/eval/jev_selection/queries-v3.json @@ -0,0 +1,138 @@ +[ + { + "query": "What are the best open-source generative engine optimization (GEO) tools on GitHub in 2026?", + "own_url": "https://egeoagents.com/compare/geo-tools-2026/", + "own_source": "site/src/content/docs/compare/geo-tools-2026.md" + }, + { + "query": "best GEO tools 2026", + "own_url": "https://egeoagents.com/compare/geo-tools-2026/", + "own_source": "site/src/content/docs/compare/geo-tools-2026.md" + }, + { + "query": "open source AEO tools", + "own_url": "https://egeoagents.com/compare/geo-tools-2026/", + "own_source": "site/src/content/docs/compare/geo-tools-2026.md" + }, + { + "query": "how to optimize content for Perplexity", + "own_url": "https://egeoagents.com/guides/rank-in-perplexity/", + "own_source": "site/src/content/docs/guides/rank-in-perplexity.md" + }, + { + "query": "GEO tool with MCP server", + "own_url": "https://egeoagents.com/docs/mcp-server/", + "own_source": "site/src/content/docs/docs/mcp-server.md" + }, + { + "query": "eGEOagents GitHub", + "own_url": "https://github.com/mverab/eGEOagents" + }, + { + "query": "eGEOagents", + "own_url": "https://egeoagents.com/" + }, + { + "query": "best answer engine optimization tools for developers", + "own_url": "https://egeoagents.com/compare/geo-tools-2026/", + "own_source": "site/src/content/docs/compare/geo-tools-2026.md" + }, + { + "query": "open source tool to rank in ChatGPT and Perplexity", + "own_url": "https://egeoagents.com/guides/rank-in-chatgpt-search/", + "own_source": "site/src/content/docs/guides/rank-in-chatgpt-search.md" + }, + { + "query": "GEO evaluation harness open source", + "own_url": "https://egeoagents.com/docs/evaluation/", + "own_source": "site/src/content/docs/docs/evaluation.md" + }, + { + "query": "what is generative engine optimization", + "own_url": "https://egeoagents.com/concepts/what-is-geo/", + "own_source": "site/src/content/docs/concepts/what-is-geo.md" + }, + { + "query": "what is answer engine optimization", + "own_url": "https://egeoagents.com/concepts/what-is-aeo/", + "own_source": "site/src/content/docs/concepts/what-is-aeo.md" + }, + { + "query": "GEO vs SEO differences", + "own_url": "https://egeoagents.com/concepts/geo-vs-seo/", + "own_source": "site/src/content/docs/concepts/geo-vs-seo.md" + }, + { + "query": "llms.txt examples", + "own_url": "https://egeoagents.com/concepts/what-is-llms-txt/", + "own_source": "site/src/content/docs/concepts/what-is-llms-txt.md" + }, + { + "query": "does llms.txt help SEO", + "own_url": "https://egeoagents.com/concepts/what-is-llms-txt/", + "own_source": "site/src/content/docs/concepts/what-is-llms-txt.md" + }, + { + "query": "how to get cited by ChatGPT", + "own_url": "https://egeoagents.com/guides/rank-in-chatgpt-search/", + "own_source": "site/src/content/docs/guides/rank-in-chatgpt-search.md" + }, + { + "query": "how to track brand mentions in AI search", + "own_url": "https://egeoagents.com/guides/track-ai-citations/", + "own_source": "site/src/content/docs/guides/track-ai-citations.md" + }, + { + "query": "how do AI overviews choose sources", + "own_url": "" + }, + { + "query": "schema markup for AI search", + "own_url": "" + }, + { + "query": "AI search visibility tracking tools", + "own_url": "https://egeoagents.com/guides/track-ai-citations/", + "own_source": "site/src/content/docs/guides/track-ai-citations.md" + }, + { + "query": "best open source AI coding agents 2026", + "own_url": "" + }, + { + "query": "open source alternatives to Ahrefs", + "own_url": "" + }, + { + "query": "best MCP servers for developers", + "own_url": "" + }, + { + "query": "how to self-host an AI agent on a VPS", + "own_url": "" + }, + { + "query": "cheapest LLM API for coding", + "own_url": "" + }, + { + "query": "open source LLM evaluation frameworks", + "own_url": "" + }, + { + "query": "AI agent memory frameworks open source", + "own_url": "" + }, + { + "query": "best vector database for RAG", + "own_url": "" + }, + { + "query": "how to reduce LLM API costs", + "own_url": "" + }, + { + "query": "open source web scraping tools for AI", + "own_url": "" + } +] diff --git a/eval/jev_selection/run.py b/eval/jev_selection/run.py index 63f6b43..9e331b9 100644 --- a/eval/jev_selection/run.py +++ b/eval/jev_selection/run.py @@ -1,48 +1,52 @@ """Does Jev judge sources like Perplexity cites them? (validation gate for section-mode fix-gaps) -v2 CONTRACT (2026-09-25 review of the v1 run). Implement until ``tests/test_jev_selection_eval.py`` -passes. Plan: docs/plans/2026-09-25-fix-gaps-sections-plan-v2.md. +v4 CONTRACT (openspec ``update-jev-blind-validation``, plan docs/plans/2026-09-26-jev-blind-validation-plan-v4.md). +Implement until ``tests/test_jev_selection_eval.py`` passes. -Why v2: v1 scored candidates with ONE Choice question, which spreads probability over 1-2 candidates; -44% of candidates got p < 0.01, so AUC was dominated by ties at ~0 and ``own_predicted_cited`` -(compared against the MINIMUM cited probability, 0.0 in 9/10 queries) was always True. A re-run on -the same data moved mean AUC from 0.6493 to 0.6272, i.e. the 0.65 gate sat inside run-to-run noise. +Why v4: v2 scored with its own copy of the Noul question, on full-page excerpts, with ids that told Jev which +candidate was ours (``own`` next to ``src_N``). v4 scores with the product's function, +``egeo.judge.score_candidates``, on the product's unit (sections) and with blind ids, so the gate measures +what section mode ships. Dataset (JSON list), one item per query, produced on the VPS by the collector described in the plan: {"query": str, "egeo_cited": bool, "own_url": str | "" | null, + "own_source": str (optional), # repo-relative .md of the own page -> unit "sections" "cited_urls": [str, ...], # URLs Perplexity cited in its answer -> label 1 "serp_urls": [str, ...]} # top web results for the same query -> label 0 unless also cited -- ``build_query_candidates``: unchanged from v1 (see tests). -- ``score_candidates_noul(query, candidates, client) -> {id: float}``: ONE request with one Noul - question per candidate (question id = candidate id, instructions = - ``NOUL_INSTRUCTIONS.format(cid=)``). State = ``{"query": query, "candidates": - {id: text[:egeo.judge.MAX_CANDIDATE_CHARS]}}``. Returns each candidate's ``noul``. - ``ValueError`` when ``candidates`` is empty. -- ``score_query(item, docs, client, own_domain, scorer="noul")``: - ``scorer`` is ``"noul"`` (per-candidate scores above) or ``"choice"`` (probabilities from - ``egeo.judge.select_best``); anything else -> ``ValueError``. Fewer than 2 candidates -> - ``{"query", "scorer", "skipped": "too_few_candidates"}``. Otherwise, with ``score = {id: float}``: - auc = auc(scores of label-1 sources, scores of label-0 sources) (None if a class is empty) - own_score = score["own"] if an own candidate exists else None - own_rank = 1 + number of candidates with a strictly higher score (None without own) - own_predicted_cited = own_rank <= n_pos # "would own make the top-k", k = cited sources present - (None without own or when n_pos == 0) - winner = id with the highest score (ties -> earliest candidate) - Return ``{"query", "scorer", "auc", "n_pos", "n_neg", "own_score", "own_rank", - "own_predicted_cited", "egeo_cited", "winner"}``. -- ``bootstrap_ci(values, *, n=2000, seed=0, alpha=0.05)``: None when fewer than 2 values. Otherwise - ``rng = random.Random(seed)``; ``n`` times take ``rng.choices(values, k=len(values))`` and record its - mean; sort; return ``(means[int(alpha / 2 * n)], means[int((1 - alpha / 2) * n) - 1])``. -- ``run(dataset, client, fetch, own_domain="egeoagents.com", scorer="noul")``: as v1 (one ``fetch`` - call for every unique normalized URL) plus ``"scorer"`` and ``"mean_auc_ci"`` = - ``bootstrap_ci()`` (a list ``[low, high]`` or None). - ``gate_passed = mean_auc is not None and mean_auc >= GATE_MIN_AUC and n_scored >= GATE_MIN_QUERIES``. +- ``build_query_candidates(item, docs, own_domain, repo_root=None) -> (candidates, labels, unit)``: + sources as in v2 (cited first, then uncited SERP, own domain dropped, at most MAX_PER_CLASS each, + ``src_N`` over ok docs). Own: with ``own_source``, read ``repo_root / own_source`` (missing file -> + ``FileNotFoundError``; never a silent fallback), drop frontmatter (``pipeline._extract_frontmatter``), + ``judge.own_candidates(split_sections(body))``, unit ``"sections"``. Else, when ``own_url`` was fetched + ok, one ``Candidate("own", "own", host, text)``, unit ``"page_excerpt"``. Else unit None. +- ``score_query(item, docs, client, own_domain, *, ids="blind", repo_root=None)``: ``ids`` not in + ``judge.IDS_MODES`` -> ``ValueError``. Fewer than 2 candidates -> ``{"query", "scorer", "ids", + "skipped": "too_few_candidates"}``. Otherwise ``scored = judge.score_candidates(query, cands, client, + ids=ids)`` and, with ``score = scored.scores``: + auc = auc(scores of label-1 sources, scores of label-0 sources) (None if a class is empty) + own_best_score = max own score; src_best_score = max source score (None when absent) + own_rank = 1 + number of sources scoring strictly above own_best_score (None without own) + own_wins = own_best_score > src_best_score, strictly: the product's ``already_best`` rule + (None without own or without sources) + winner = id with the highest score (ties -> earliest candidate) + Return ``{"query", "scorer": "noul", "ids", "unit", "auc", "n_pos", "n_neg", "own_best_score", + "src_best_score", "own_rank", "own_wins", "egeo_cited", "winner", "jev_model"}`` (``jev_model`` = the + model string the API returned for that request). +- ``bootstrap_ci``: unchanged. ``wilson_ci(k, n, z=1.96)``: Wilson score interval, None when n == 0. +- ``run(dataset, client, fetch, own_domain="egeoagents.com", *, ids="blind", repo_root=None, + prereg_sha=None)``: one ``fetch`` call for every unique normalized URL (``own_url`` is skipped for + items with ``own_source``). Adds ``ids``, ``gating = ids == "blind" and bool(prereg_sha)`` (a run + without a pre-registration can never pass), ``prereg_sha``, ``unit_counts`` + (scored items per unit), ``own_confusion`` (``own_wins`` x ``egeo_cited``: tp/fp/fn/tn; ``sections`` + items only), ``own_n``, ``own_accuracy`` and ``own_accuracy_ci95`` (Wilson), ``jev_models_returned``. + ``gate_passed = gating and mean_auc is not None and mean_auc >= GATE_MIN_AUC and n_scored >= + GATE_MIN_QUERIES``. Own metrics never gate. - ``auc``: unchanged (Mann-Whitney with 0.5 for ties). -- CLI: add ``--scorer {noul,choice}`` (default ``noul``); also print the CI. +- CLI: ``--ids {blind,leaky}`` (default blind), ``--prereg-sha SHA`` (required unless ``--mock``; exit 2 + without it), ``--repo-root PATH`` (default ``.``, for ``own_source``). -Pre-registration (do not change after seeing results): scorer ``noul``, GATE_MIN_AUC 0.65, -GATE_MIN_QUERIES 25, dataset v2 collected once, evaluated once. +Pre-registration lives in ``eval/jev_selection/PREREG-v4.md`` and is committed before any live v4 run. """ from __future__ import annotations @@ -51,20 +55,16 @@ MAX_PER_CLASS = 8 GATE_MIN_AUC = 0.65 GATE_MIN_QUERIES = 25 -NOUL_INSTRUCTIONS = ( - "Would an AI answer engine cite candidates.{cid} as a source when answering `query`? " - "Answer yes only if that candidate's own text directly and credibly answers `query`." -) -SCORERS = ("noul", "choice") import argparse import datetime import json +import math import random +from pathlib import Path from egeo import judge as _judge -from egeo.jev import noul_question from egeo.sources import host_of, normalize_source @@ -89,7 +89,15 @@ def _uniq(urls, dom): return out -def build_query_candidates(item, docs, own_domain): +def _own_sections(path): + from egeo.pipeline import _extract_frontmatter + from egeo.sections import split_sections + + body = _extract_frontmatter(Path(path).read_text(encoding="utf-8"))[1] + return _judge.own_candidates(split_sections(body)) + + +def build_query_candidates(item, docs, own_domain, repo_root=None): cited = _uniq(item.get("cited_urls", []), own_domain)[:MAX_PER_CLASS] unc = [u for u in _uniq(item.get("serp_urls", []), own_domain) if u not in cited][:MAX_PER_CLASS] cands, labels = [], {} @@ -99,19 +107,14 @@ def build_query_candidates(item, docs, own_domain): continue cid = f"src_{len(cands) + 1}" cands.append(_judge.Candidate(cid, "source", host_of(u), d.text)); labels[cid] = lab + if item.get("own_source"): + cands.extend(_own_sections(Path(repo_root or ".") / item["own_source"])) + return cands, labels, "sections" ou = normalize_source(item.get("own_url", "")) if ou and docs.get(ou) is not None and docs[ou].ok: cands.append(_judge.Candidate("own", "own", host_of(ou), docs[ou].text)) - return cands, labels - - -def score_candidates_noul(query: str, candidates: Sequence[Any], client: Any) -> Dict[str, float]: - if not candidates: - raise ValueError("need at least one candidate") - state = {"query": query, "candidates": {c.id: c.text[:_judge.MAX_CANDIDATE_CHARS] for c in candidates}} - questions = {c.id: noul_question(NOUL_INSTRUCTIONS.format(cid=c.id)) for c in candidates} - answers = client.ask(state, questions).answers - return {c.id: answers[c.id].noul for c in candidates} + return cands, labels, "page_excerpt" + return cands, labels, None def bootstrap_ci(values: Sequence[float], *, n: int = 2000, seed: int = 0, alpha: float = 0.05) -> Optional[Tuple[float, float]]: @@ -122,64 +125,93 @@ def bootstrap_ci(values: Sequence[float], *, n: int = 2000, seed: int = 0, alpha return means[int(alpha / 2 * n)], means[int((1 - alpha / 2) * n) - 1] -def score_query(item, docs, client, own_domain, scorer="noul"): - if scorer not in SCORERS: - raise ValueError(f"unknown scorer: {scorer}") - cands, labels = build_query_candidates(item, docs, own_domain) +def wilson_ci(k: int, n: int, z: float = 1.96) -> Optional[Tuple[float, float]]: + if n == 0: + return None + p = k / n + den = 1 + z * z / n + centre = (p + z * z / (2 * n)) / den + half = z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) / den + return centre - half, centre + half + + +def score_query(item, docs, client, own_domain, *, ids="blind", repo_root=None): + if ids not in _judge.IDS_MODES: + raise ValueError(f"unknown ids mode: {ids}") + cands, labels, unit = build_query_candidates(item, docs, own_domain, repo_root=repo_root) if len(cands) < 2: - return {"query": item["query"], "scorer": scorer, "skipped": "too_few_candidates"} - if scorer == "noul": - score = score_candidates_noul(item["query"], cands, client) - else: - score = dict(_judge.select_best(item["query"], cands, client).probabilities) - pos = [score.get(i, 0.0) for i, l in labels.items() if l == 1] - neg = [score.get(i, 0.0) for i, l in labels.items() if l == 0] - has_own = any(c.id == "own" for c in cands) - if has_own: - own_score = score.get("own", 0.0) - own_rank = 1 + sum(1 for c in cands if score.get(c.id, 0.0) > own_score) - pred = (own_rank <= len(pos)) if pos else None - else: - own_score = own_rank = pred = None + return {"query": item["query"], "scorer": "noul", "ids": ids, "skipped": "too_few_candidates"} + scored = _judge.score_candidates(item["query"], cands, client, ids=ids) + score = scored.scores + pos = [score[i] for i, l in labels.items() if l == 1] + neg = [score[i] for i, l in labels.items() if l == 0] + own = [score[c.id] for c in cands if c.kind == "own"] + src = [score[c.id] for c in cands if c.kind == "source"] + own_best = max(own) if own else None + src_best = max(src) if src else None + own_rank = 1 + sum(1 for v in src if v > own_best) if own else None + own_wins = own_best > src_best if own and src else None winner = None for c in cands: - if winner is None or score.get(c.id, 0.0) > score.get(winner, 0.0): + if winner is None or score[c.id] > score[winner]: winner = c.id - return {"query": item["query"], "scorer": scorer, "auc": auc(pos, neg), "n_pos": len(pos), "n_neg": len(neg), - "own_score": own_score, "own_rank": own_rank, "own_predicted_cited": pred, - "egeo_cited": item.get("egeo_cited"), "winner": winner} + return {"query": item["query"], "scorer": "noul", "ids": ids, "unit": unit, "auc": auc(pos, neg), + "n_pos": len(pos), "n_neg": len(neg), "own_best_score": own_best, "src_best_score": src_best, + "own_rank": own_rank, "own_wins": own_wins, "egeo_cited": item.get("egeo_cited"), "winner": winner, + "jev_model": scored.model} -def run(dataset, client, fetch, own_domain="egeoagents.com", scorer="noul"): +def run(dataset, client, fetch, own_domain="egeoagents.com", *, ids="blind", repo_root=None, prereg_sha=None): urls = [] for it in dataset: - for v in [*it.get("cited_urls", []), *it.get("serp_urls", []), it.get("own_url", "")]: + own_url = [] if it.get("own_source") else [it.get("own_url", "")] + for v in [*it.get("cited_urls", []), *it.get("serp_urls", []), *own_url]: u = normalize_source(v) if u and u not in urls: urls.append(u) docs = fetch(urls) - per = [score_query(it, docs, client, own_domain, scorer=scorer) for it in dataset] + per = [score_query(it, docs, client, own_domain, ids=ids, repo_root=repo_root) for it in dataset] + models = sorted({p["jev_model"] for p in per if p.get("jev_model")}) aucs = [p["auc"] for p in per if p.get("auc") is not None] mean = sum(aucs) / len(aucs) if aucs else None ci = bootstrap_ci(aucs) - owns = [p for p in per if p.get("own_predicted_cited") is not None] - acc = sum(1 for p in owns if p["own_predicted_cited"] == p["egeo_cited"]) / len(owns) if owns else None - return {"per_query": per, "scorer": scorer, "mean_auc": mean, "mean_auc_ci": list(ci) if ci else None, - "n_scored": len(aucs), "own_accuracy": acc, - "gate_passed": mean is not None and mean >= GATE_MIN_AUC and len(aucs) >= GATE_MIN_QUERIES, - "jev_model": client.model, "jev_usage": dict(client.totals), "date": datetime.date.today().isoformat()} + units = {} + for p in per: + if "skipped" not in p and p["unit"]: + units[p["unit"]] = units.get(p["unit"], 0) + 1 + conf = {"tp": 0, "fp": 0, "fn": 0, "tn": 0} + for p in per: + if p.get("unit") == "sections" and p["own_wins"] is not None and isinstance(p["egeo_cited"], bool): + conf[("t" if p["own_wins"] == p["egeo_cited"] else "f") + ("p" if p["own_wins"] else "n")] += 1 + own_n = sum(conf.values()) + acc_ci = wilson_ci(conf["tp"] + conf["tn"], own_n) + gating = ids == "blind" and bool(prereg_sha) + return {"per_query": per, "scorer": "noul", "ids": ids, "gating": gating, "prereg_sha": prereg_sha, + "unit_counts": units, "mean_auc": mean, "mean_auc_ci": list(ci) if ci else None, "n_scored": len(aucs), + "own_confusion": conf, "own_n": own_n, + "own_accuracy": (conf["tp"] + conf["tn"]) / own_n if own_n else None, + "own_accuracy_ci95": list(acc_ci) if acc_ci else None, + "gate_passed": gating and mean is not None and mean >= GATE_MIN_AUC and len(aucs) >= GATE_MIN_QUERIES, + "jev_model": client.model, "jev_models_returned": models, "jev_usage": dict(client.totals), + "date": datetime.date.today().isoformat()} def main(argv=None): + import sys + parser = argparse.ArgumentParser(description="Does Jev choose sources like Perplexity does?") parser.add_argument("--dataset", required=True, help="Dataset JSON (see module docstring).") parser.add_argument("--out", required=True, help="Where to write the results JSON.") parser.add_argument("--mock", action="store_true", help="Offline deterministic run (no TypeSafe, no network).") - parser.add_argument("--scorer", default="noul", choices=list(SCORERS), help="Candidate scoring (default: noul).") + parser.add_argument("--ids", default="blind", choices=list(_judge.IDS_MODES), + help="blind (default, gating) or leaky (2.2.0 ids, leak diagnostic only, never gates).") + parser.add_argument("--prereg-sha", default=None, help="Commit SHA of PREREG-v4.md (required unless --mock).") + parser.add_argument("--repo-root", default=".", help="Root that dataset `own_source` paths are relative to.") parser.add_argument("--own-domain", default="egeoagents.com") args = parser.parse_args(argv) - - from pathlib import Path + if not args.mock and not args.prereg_sha: + print("error: a live run needs --prereg-sha ", file=sys.stderr) + return 2 from egeo import jev as _jev from egeo import sources as _sources @@ -188,12 +220,16 @@ def main(argv=None): client = _jev.make_client(mock=args.mock) source_fn = _sources.placeholder_sources if args.mock else _sources.fetch_sources fetch = lambda urls: {d.url: d for d in source_fn(urls, limit=len(urls))} # noqa: E731 - result = run(dataset, client, fetch, own_domain=args.own_domain, scorer=args.scorer) + result = run(dataset, client, fetch, own_domain=args.own_domain, ids=args.ids, repo_root=Path(args.repo_root), + prereg_sha=args.prereg_sha) Path(args.out).write_text(json.dumps(result, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") - print(f"scorer: {result['scorer']}") + print(f"ids: {result['ids']} ({'gating' if result['gating'] else 'NON-GATING diagnostic'})") + print(f"unit_counts: {result['unit_counts']}") print(f"mean_auc: {result['mean_auc']}") print(f"mean_auc_ci: {result['mean_auc_ci']}") print(f"n_scored: {result['n_scored']}") + print(f"own_accuracy (sections): {result['own_accuracy']} {result['own_accuracy_ci95']}") + print(f"jev_models_returned: {result['jev_models_returned']}") print("GATE PASS" if result["gate_passed"] else "GATE FAIL") return 0 diff --git a/openspec/changes/update-jev-blind-validation/specs/citation-gap-fixing/spec.md b/openspec/changes/update-jev-blind-validation/specs/citation-gap-fixing/spec.md index 66359bf..47f3ba3 100644 --- a/openspec/changes/update-jev-blind-validation/specs/citation-gap-fixing/spec.md +++ b/openspec/changes/update-jev-blind-validation/specs/citation-gap-fixing/spec.md @@ -8,7 +8,7 @@ Every Jev request that compares candidates (diagnosis, verification and the vali - **WHEN** section mode sends the diagnosis request for a page with own sections and fetched sources - **THEN** every candidate key in the request state matches `^c\d{2}$` -- **AND** no question text contains `own` or `src` +- **AND** no request contains an original candidate id (`own_…`, `src_…`), host or section heading #### Scenario: Stable ids across diagnosis and verification diff --git a/openspec/changes/update-jev-blind-validation/tasks.md b/openspec/changes/update-jev-blind-validation/tasks.md index e7555a3..a161463 100644 --- a/openspec/changes/update-jev-blind-validation/tasks.md +++ b/openspec/changes/update-jev-blind-validation/tasks.md @@ -1,22 +1,22 @@ Implementation plan with lanes, contracts and the run protocol: `docs/plans/2026-09-26-jev-blind-validation-plan-v4.md`. ## 0. Approval -- [ ] 0.1 Owner approves this proposal and decides D2 (Noul, recommended, vs Choice) +- [x] 0.1 Owner approves this proposal and decides D2 (Noul, recommended, vs Choice) — approved 2026-10-07, D2 = Noul ## 1. Contracts (RED) -- [ ] 1.1 Contract tests for blinding, `score_candidates`, Noul diagnose and verify, mock Noul, the eval product path and the report keys; stubs with signatures in docstrings +- [x] 1.1 Contract tests for blinding, `score_candidates`, Noul diagnose and verify, mock Noul, the eval product path and the report keys; stubs with signatures in docstrings ## 2. Implementation -- [ ] 2.1 `egeo/judge.py` + `egeo/jev.py` (lane J1) -- [ ] 2.2 `eval/jev_selection/run.py` (lane G4) -- [ ] 2.3 `egeo/gaps.py` report keys (lane F4) -- [ ] 2.4 Full suite green; reviewer reads the diff +- [x] 2.1 `egeo/judge.py` + `egeo/jev.py` (lane J1) +- [x] 2.2 `eval/jev_selection/run.py` (lane G4) +- [x] 2.3 `egeo/gaps.py` report keys (lane F4) +- [x] 2.4 Full suite green; reviewer reads the diff ## 3. Validation (pre-registered; design D4) -- [ ] 3.1 Commit `eval/jev_selection/PREREG-v4.md` (queries sha256, scorer, ids, unit, gate, consequences) before any live run -- [ ] 3.2 Leak diagnostic on dataset v2: `--ids leaky` vs `--ids blind` (non-gating) -- [ ] 3.3 Collect dataset v3 once (≥ 30 queries, `own_source` mapped) -- [ ] 3.4 Evaluate v3 once; record the result in the README +- [x] 3.1 Commit `eval/jev_selection/PREREG-v4.md` (registered 2026-10-08; its registering commit SHA is `--prereg-sha`) (queries sha256, scorer, ids, unit, gate, consequences) before any live run +- [x] 3.2 Leak diagnostic on dataset v2: `--ids leaky` vs `--ids blind` (non-gating) +- [x] 3.3 Collect dataset v3 once (≥ 30 queries, `own_source` mapped) — 2026-10-15, 30 queries, 0 empty +- [x] 3.4 Evaluate v3 once; record the result in the README — GATE FAIL, mean AUC 0.559 [0.489, 0.637] ## 4. Outcome (design D5) -- [ ] 4.1 Update `JEV_VALIDATION`, docs and CHANGELOG per the pre-registered consequence; owner decision if FAIL +- [x] 4.1 Update `JEV_VALIDATION`, docs and CHANGELOG per the pre-registered consequence; owner decision if FAIL — FAIL; owner decision 2026-10-16: keep section mode experimental (demote `already_best` in a separate change) diff --git a/site/src/content/docs/docs/cli.md b/site/src/content/docs/docs/cli.md index f57a1e3..d5273d2 100644 --- a/site/src/content/docs/docs/cli.md +++ b/site/src/content/docs/docs/cli.md @@ -152,7 +152,7 @@ egeo fix-gaps gaps.json --project project.yaml --out-dir fix-gaps-output --mode | `rewriter_error` | the rewriter LLM failed for this page (other pages continue) | :::caution[Experimental: the Jev comparison] -The two Jev steps (which section loses, and the before/after re-check) measure **text competitiveness** — whether a candidate's *text* directly answers the query. They do **not** model authority, links or brand, and they are **not a prediction of citation**. In a pre-registered validation (30 queries, 2026-09-25) the same single-choice comparison separated Perplexity-cited from uncited sources with mean AUC **0.62 (95% CI 0.56–0.67)** — weak, and below the 0.65 bar we set — and matched the engine's cited/not-cited outcome for the own page in only **17%** of queries. That validation used full-page excerpts, not sections. The Jev fidelity judge was not validated directly (the eval measured selection); it can only reject, after the deterministic rules. A re-check outcome of `won` means any own section now wins, not necessarily the rewritten one. Numbers and method: `eval/jev_selection/README.md`. Every `fix-gaps.json` carries the same figures in `jev_validation`, and every diagnosis/verification is marked `"experimental": true`. **Proof of impact never comes from Jev:** re-run your tracker on the `remeasure` queries. +The two Jev steps (which section loses, and the before/after re-check) measure **text competitiveness** — whether a candidate's *text* directly answers the query. They do **not** model authority, links or brand, and they are **not a prediction of citation**. In the pre-registered validation of the configuration this version runs (independent score per candidate, neutral ids that hide which text is yours, sections; 30 queries, 2026-10-15), Jev separated Perplexity-cited from uncited sources with mean AUC **0.56 (95% CI 0.49–0.64)**: weak, and below the 0.65 bar we set. When Jev judged your own section the best, Perplexity cited you in only **1 of 6** of those queries, so treat `already_best` as a weak signal, not a reason to stop. An earlier configuration (single-choice comparison, ids that revealed ownership, full-page excerpts) scored 0.62 (95% CI 0.56–0.67, 2026-09-25). The Jev fidelity judge was not validated directly (the eval measured selection); it can only reject, after the deterministic rules. A re-check outcome of `won` means any own section now wins, not necessarily the rewritten one. Numbers and method: `eval/jev_selection/README.md`. Every `fix-gaps.json` carries the same figures in `jev_validation`, and every diagnosis/verification is marked `"experimental": true`. **Proof of impact never comes from Jev:** re-run your tracker on the `remeasure` queries. ::: **`already_best` → off-page recommendation:** when your text already wins the comparison, rewriting it is unlikely to help — the gap is probably off-page. The report lists the sources the engine cited so you can get your page mentioned or linked by them. diff --git a/tests/test_gaps_sections.py b/tests/test_gaps_sections.py index b1c804f..1acbb8b 100644 --- a/tests/test_gaps_sections.py +++ b/tests/test_gaps_sections.py @@ -2,13 +2,14 @@ from __future__ import annotations import json +import re from pathlib import Path import pytest from egeo import gaps, judge, sources from egeo.cli import main -from egeo.jev import ChoiceAnswer, JevProviderError, JevResponse +from egeo.jev import ChoiceAnswer, JevProviderError, JevResponse, NoulAnswer from egeo.sources import SourceDoc PROJECT_YAML = """schema_version: 1 @@ -50,7 +51,15 @@ GAPS = [{"query": "open source AEO tools", "cited": False, "sources": ["https://toolradar.com/x", "https://demo.dev/y"]}] +SRC_TEXT = "Best AEO tools list" +# label of every candidate text Jev can see (the rewritten Tools section keeps the label own_s01) +LABEL_OF = {TOOLS: "own_s01", GOOD_REWRITE: "own_s01", FAQ: "own_s02", SRC_TEXT: "src_1"} + + class ScriptedJev: + """Queue items: a dict ``{label: noul score}`` answers a Noul (scoring) request, a ChoiceAnswer answers + the fidelity judge, an Exception is raised.""" + def __init__(self, *answers) -> None: self.queue = list(answers) self.requests = [] @@ -63,13 +72,18 @@ def ask(self, state, questions): nxt = self.queue.pop(0) if isinstance(nxt, Exception): raise nxt - (qid,) = questions.keys() - return JevResponse(answers={qid: nxt}, model="fake", input_tokens=1, output_tokens=0, cost_usd=0.0) + if isinstance(nxt, dict): + labels = {qid: LABEL_OF[state["candidates"][qid]] for qid in questions} + answers = {qid: NoulAnswer(nxt[label]) for qid, label in labels.items()} + else: + (qid,) = questions.keys() + answers = {qid: nxt} + return JevResponse(answers=answers, model="fake", input_tokens=1, output_tokens=0, cost_usd=0.0) -LOSES = ChoiceAnswer("src_1", 0.3, {"own_s01": 0.30, "own_s02": 0.10, "src_1": 0.60}) +LOSES = {"own_s01": 0.30, "own_s02": 0.10, "src_1": 0.60} FAITHFUL = ChoiceAnswer("faithful", 0.8, {"faithful": 0.9, "drops_facts": 0.05, "adds_claims": 0.05}) -WON = ChoiceAnswer("own_s01", 0.5, {"own_s01": 0.70, "own_s02": 0.10, "src_1": 0.20}) +WON = {"own_s01": 0.70, "own_s02": 0.10, "src_1": 0.20} @pytest.fixture() @@ -87,7 +101,7 @@ def _plan(project: Path): def fetch_ok(values, *, exclude_domain, limit): - return [SourceDoc(url="https://toolradar.com/x", ok=True, title="toolradar.com", text="Best AEO tools list")] + return [SourceDoc(url="https://toolradar.com/x", ok=True, title="toolradar.com", text=SRC_TEXT)] def _run(project: Path, jev, rewrite, fetch=fetch_ok): @@ -116,10 +130,15 @@ def rewrite(query, text, title): assert page["status"] == "rewritten" and page["reasons"] == [] assert page["diagnosis"]["winner"] == "src_1" and page["diagnosis"]["winner_kind"] == "source" assert page["diagnosis"]["target_section"] == {"id": "s01", "heading": "Tools"} + assert page["diagnosis"]["scorer"] == "noul" + assert page["diagnosis"]["scores"] == {"own_s01": 0.30, "own_s02": 0.10, "src_1": 0.60} + assert "probabilities" not in page["diagnosis"] and "confidence" not in page["diagnosis"] assert page["fidelity"]["rules"] == {"passed": True, "violations": []} assert page["fidelity"]["judge"] == {"verdict": "faithful", "confidence": 0.8, "accepted": True} assert page["verification"]["outcome"] == "won" - assert page["verification"]["p_before"] == 0.30 and page["verification"]["p_after"] == 0.70 + assert page["verification"]["score_before"] == 0.30 and page["verification"]["score_after"] == 0.70 + assert page["verification"]["winner_after"] == "own_s01" and page["verification"]["winner_after_kind"] == "own" + assert "p_before" not in page["verification"] assert page["sources"] == [{"url": "https://toolradar.com/x", "ok": True, "error": ""}] written = project / "out" / "compare" / "compare.md" @@ -135,7 +154,7 @@ def rewrite(query, text, title): assert saved["mode"] == "sections" and saved["jev_model"] == "fake" assert saved["note"] == gaps.SECTIONS_NOTE assert "not a prediction of citation" in saved["note"] - assert "EXPERIMENTAL" in saved["note"] and "0.62" in saved["note"] + assert "EXPERIMENTAL" in saved["note"] and "below the 0.65 bar" in saved["note"] assert saved["jev_validation"] == gaps.JEV_VALIDATION assert saved["jev_validation"]["status"] == "experimental" and saved["jev_validation"]["gate_passed"] is False assert page["diagnosis"]["experimental"] is True @@ -150,13 +169,17 @@ def rewrite(query, text, title): for key in ("format", "unmatched", "skipped", "remeasure"): assert key in saved - # the Jev diagnosis saw own sections (intro is < 30 words so excluded) and the fetched source - state, _ = jev.requests[0] - assert set(state["candidates"]) == {"own_s01", "own_s02", "src_1"} - # verification used the same ids with the rewritten text + # the Jev diagnosis saw own sections (intro is < 30 words so excluded) and the fetched source, blind + state, questions = jev.requests[0] + assert sorted(LABEL_OF[t] for t in state["candidates"].values()) == ["own_s01", "own_s02", "src_1"] + assert all(re.match(r"^c\d{2}$", k) for k in [*state["candidates"], *questions]) + blob = repr((state, questions)) + assert "own_" not in blob and "src_" not in blob and "toolradar" not in blob.replace(SRC_TEXT, "") + # verification used the same ids, with the rewritten text under the target's id state_after, _ = jev.requests[2] - assert set(state_after["candidates"]) == {"own_s01", "own_s02", "src_1"} - assert state_after["candidates"]["own_s01"] == GOOD_REWRITE[: judge.MAX_CANDIDATE_CHARS] + assert list(state_after["candidates"]) == list(state["candidates"]) + target_cid = next(k for k, t in state["candidates"].items() if t == TOOLS) + assert state_after["candidates"][target_cid] == GOOD_REWRITE[: judge.MAX_CANDIDATE_CHARS] def _no_output(project: Path) -> None: @@ -220,7 +243,7 @@ def test_judge_rejection(project: Path) -> None: def test_worse_is_rejected(project: Path) -> None: - worse = ChoiceAnswer("src_1", 0.5, {"own_s01": 0.20, "own_s02": 0.10, "src_1": 0.70}) + worse = {"own_s01": 0.20, "own_s02": 0.10, "src_1": 0.70} report = _run(project, ScriptedJev(LOSES, FAITHFUL, worse), lambda *a: GOOD_REWRITE) page = report["pages"][0] assert page["status"] == "rejected_worse" and page["verification"]["outcome"] == "worse" @@ -236,12 +259,30 @@ def test_jev_error_marks_page(project: Path) -> None: _no_output(project) -def test_jev_validation_reports_choice_scorer_on_page_excerpts() -> None: +def test_jev_validation_records_the_failed_v4_run() -> None: + # The pre-registered v4 run (PREREG-v4.md, 2026-10-15) failed the gate. Per design D5 the numbers are + # recorded, the status stays "experimental" and the 2.2.0 figures stay under "previous". v = gaps.JEV_VALIDATION - assert v["scorer"] == "choice" and v["mean_auc"] == 0.619 and v["ci95"] == [0.561, 0.674] - assert v["own_accuracy"] == 0.167 and v["n_queries"] == 30 and v["gate_passed"] is False - assert "not sections" in v["unit"] - assert "0.637" not in gaps.SECTIONS_NOTE and "full-page excerpts, not sections" in gaps.SECTIONS_NOTE + assert (v["protocol"], v["scorer"], v["ids"], v["unit"]) == ("v4", "noul", "blind", "sections") + assert v["mean_auc"] == 0.559 and v["ci95"] == [0.489, 0.637] and v["n_queries"] == 30 + assert v["own_accuracy"] == 0.5625 and v["date"] == "2026-10-15" + assert v["prereg_sha"] == "56ef2db55125c691bd72c7f350ce8dc3dae31d27" + assert v["status"] == "experimental" and v["gate_passed"] is False + assert v["gate_auc"] == 0.65 and v["gate_min_queries"] == 25 + prev = v["previous"] + assert prev["config"] == "choice, leaky ids, page excerpts" + assert prev["mean_auc"] == 0.619 and prev["ci95"] == [0.561, 0.674] and prev["own_accuracy"] == 0.167 + current = gaps.SECTIONS_NOTE.split("earlier configuration")[0] + assert "0.56" in current and "2026-10-15" in current and "0.62" not in current + + +def test_experimental_flags_follow_the_validation_status(project: Path, monkeypatch) -> None: + monkeypatch.setitem(gaps.JEV_VALIDATION, "status", "validated") + report = _run(project, ScriptedJev(LOSES, FAITHFUL, WON), lambda *a: GOOD_REWRITE) + page = report["pages"][0] + assert page["status"] == "rewritten" + assert "experimental" not in page["diagnosis"] and "experimental" not in page["verification"] + assert "not a prediction of citation" in report["note"] def test_rewriter_error_marks_page_and_run_continues(tmp_path: Path) -> None: diff --git a/tests/test_jev.py b/tests/test_jev.py index 2a51fe6..f281030 100644 --- a/tests/test_jev.py +++ b/tests/test_jev.py @@ -127,11 +127,24 @@ def test_mock_transport_is_deterministic_word_overlap() -> None: assert best["choice"] == "own_s01" assert best["probabilities"] == {"own_s01": 0.8333, "src_1": 0.1667} assert best["confidence"] == pytest.approx(0.6667, abs=1e-4) - assert payload["answers"]["ok"] == {"type": "noul", "noul": 0.9} + # noul question "ok" points at no candidate -> 0.0 + assert payload["answers"]["ok"] == {"type": "noul", "noul": 0.0} assert payload["model"] == "mock" assert MockJevTransport().post_json(jev.TYPESAFE_URL, {"model": "mock", "state": state, "questions": questions}) == (status, payload) +def test_mock_noul_is_query_word_overlap_share() -> None: + state = {"query": "open source geo tools", + "candidates": {"c01": "Open source GEO tools compared", "c02": "open recipes", "c03": "cooking"}} + questions = {cid: noul_question(f"cite candidates.{cid}?") for cid in ("c01", "c02", "c03")} + status, payload = MockJevTransport().post_json(jev.TYPESAFE_URL, {"model": "mock", "state": state, "questions": questions}) + assert status == 200 + # 4 query words: c01 overlaps 4 -> 1.0, c02 overlaps 1 -> 0.25, c03 none -> 0.0 + assert {q: a["noul"] for q, a in payload["answers"].items()} == {"c01": 1.0, "c02": 0.25, "c03": 0.0} + _, plain = MockJevTransport().post_json(jev.TYPESAFE_URL, {"model": "mock", "state": "s", "questions": {"x": noul_question("?")}}) + assert plain["answers"]["x"] == {"type": "noul", "noul": 0.0} + + def test_make_client() -> None: mock = jev.make_client(mock=True) assert isinstance(mock.transport, MockJevTransport) and mock.model == "mock" diff --git a/tests/test_jev_selection_eval.py b/tests/test_jev_selection_eval.py index 32d5a9e..5e0539c 100644 --- a/tests/test_jev_selection_eval.py +++ b/tests/test_jev_selection_eval.py @@ -1,6 +1,10 @@ -"""Contract tests for eval/jev_selection/run.py — v2 (noul scorer, rank-based own metric, CI, n>=25). No network.""" +"""Contract tests for eval/jev_selection/run.py — v4 (product scoring path, blind ids, sections). No network.""" from __future__ import annotations +import json +import re +from pathlib import Path + import pytest from egeo import judge @@ -21,37 +25,33 @@ def _doc(url: str, ok: bool = True) -> SourceDoc: return SourceDoc(url=url, ok=ok, title=url, text=f"text of {url}", error="" if ok else "http_404") +OWN_URL = "https://egeoagents.com/compare/geo-tools-2026/" ITEM = { "query": "best geo tools 2026", "egeo_cited": False, - "own_url": "https://egeoagents.com/compare/geo-tools-2026/", + "own_url": OWN_URL, "cited_urls": ["https://libhunt.com/", "https://egeoagents.com/x", "https://toolradar.com/best", "https://libhunt.com/"], "serp_urls": ["https://toolradar.com/best", "https://frase.io/blog", "https://down.example.com/", "https://www.egeoagents.com/"], } -DOCS = {u: _doc(u) for u in [ - "https://libhunt.com/", "https://toolradar.com/best", "https://frase.io/blog", - "https://egeoagents.com/compare/geo-tools-2026/", -]} +DOCS = {u: _doc(u) for u in ["https://libhunt.com/", "https://toolradar.com/best", "https://frase.io/blog", OWN_URL]} DOCS["https://down.example.com/"] = _doc("https://down.example.com/", ok=False) -# candidates built from ITEM: src_1 libhunt (cited), src_2 toolradar (cited), src_3 frase (uncited), own - +# candidates built from ITEM: src_1 libhunt (cited), src_2 toolradar (cited), src_3 frase (uncited), own (excerpt) +TEXT_OF = {"src_1": DOCS["https://libhunt.com/"].text, "src_2": DOCS["https://toolradar.com/best"].text, + "src_3": DOCS["https://frase.io/blog"].text, "own": DOCS[OWN_URL].text} -def test_build_query_candidates() -> None: - cands, labels = ev.build_query_candidates(ITEM, DOCS, "egeoagents.com") - assert [(c.id, c.kind, c.label) for c in cands] == [ - ("src_1", "source", "libhunt.com"), ("src_2", "source", "toolradar.com"), - ("src_3", "source", "frase.io"), ("own", "own", "egeoagents.com"), - ] - assert labels == {"src_1": 1, "src_2": 1, "src_3": 0} +SEC_A = "## Tools\n\n" + " ".join(["tools"] * 35) + "\n\n" +SEC_B = "## FAQ\n\n" + " ".join(["faq"] * 35) + "\n" +PAGE_MD = "---\ntitle: GEO tools\n---\nShort intro.\n\n" + SEC_A + SEC_B class FixedJev: - """Answers Choice questions with the given probabilities and Noul questions with the same numbers.""" - - model = "fake" + """Answers each Noul question with the score of the candidate *text* it points at: ``scores`` is keyed + by a label (``src_1``, ``own``, ``own_s01`` …) and ``texts`` maps label -> text.""" - def __init__(self, scores): + def __init__(self, scores, texts=None, model="fake"): + self.model = model self.scores = scores + self.by_text = {t: scores.get(k, 0.0) for k, t in (texts or TEXT_OF).items()} self.requests = [] self.totals = {"requests": 0, "input_tokens": 0, "output_tokens": 0, "cost_usd": 0.0} @@ -61,63 +61,98 @@ def ask(self, state, questions): answers = {} for qid, q in questions.items(): if q["type"] == "noul": - answers[qid] = NoulAnswer(self.scores.get(qid, 0.0)) + answers[qid] = NoulAnswer(self.by_text.get(state["candidates"][qid], 0.0)) else: - ids = list(q["criteria"]) - probs = {i: self.scores.get(i, 0.0) for i in ids} - answers[qid] = ChoiceAnswer(max(ids, key=lambda i: probs[i]), 0.5, probs) - return JevResponse(answers=answers, model="fake", input_tokens=0, output_tokens=0, cost_usd=0.0) + answers[qid] = ChoiceAnswer(next(iter(q["criteria"])), 0.5, {}) + return JevResponse(answers=answers, model="jev-1.13.0", input_tokens=0, output_tokens=0, cost_usd=0.0) -def test_score_candidates_noul_one_request_one_question_per_candidate() -> None: - cands = [judge.Candidate("src_1", "source", "a.com", "x" * 3000), judge.Candidate("own", "own", "b.com", "own text")] - fake = FixedJev({"src_1": 0.8, "own": 0.3}) - assert ev.score_candidates_noul("best geo tools", cands, fake) == {"src_1": 0.8, "own": 0.3} - assert len(fake.requests) == 1 - state, questions = fake.requests[0] - assert state == {"query": "best geo tools", "candidates": {"src_1": "x" * judge.MAX_CANDIDATE_CHARS, "own": "own text"}} - assert questions == { - "src_1": {"type": "noul", "instructions": ev.NOUL_INSTRUCTIONS.format(cid="src_1")}, - "own": {"type": "noul", "instructions": ev.NOUL_INSTRUCTIONS.format(cid="own")}, - } +def _write_page(root: Path) -> str: + (root / "content").mkdir(exist_ok=True) + (root / "content" / "geo-tools.md").write_text(PAGE_MD, encoding="utf-8") + return "content/geo-tools.md" -def test_score_candidates_noul_needs_candidates() -> None: - with pytest.raises(ValueError): - ev.score_candidates_noul("q", [], FixedJev({})) +def _sections_texts(root: Path): + cands, _, _ = ev.build_query_candidates(dict(ITEM, own_source="content/geo-tools.md"), DOCS, "egeoagents.com", repo_root=root) + return {c.id: c.text for c in cands} -def test_score_query_noul() -> None: - res = ev.score_query(ITEM, DOCS, FixedJev({"src_1": 0.9, "src_2": 0.2, "src_3": 0.5, "own": 0.6}), "egeoagents.com") - assert res == { - "query": ITEM["query"], "scorer": "noul", "auc": 0.5, "n_pos": 2, "n_neg": 1, - "own_score": 0.6, "own_rank": 2, "own_predicted_cited": True, # rank 2 <= 2 cited sources present - "egeo_cited": False, "winner": "src_1", - } +def test_build_query_candidates_page_excerpt() -> None: + cands, labels, unit = ev.build_query_candidates(ITEM, DOCS, "egeoagents.com") + assert [(c.id, c.kind, c.label) for c in cands] == [ + ("src_1", "source", "libhunt.com"), ("src_2", "source", "toolradar.com"), + ("src_3", "source", "frase.io"), ("own", "own", "egeoagents.com"), + ] + assert labels == {"src_1": 1, "src_2": 1, "src_3": 0} + assert unit == "page_excerpt" + +def test_build_query_candidates_own_source_uses_product_sections(tmp_path: Path) -> None: + rel = _write_page(tmp_path) + cands, labels, unit = ev.build_query_candidates(dict(ITEM, own_source=rel), DOCS, "egeoagents.com", repo_root=tmp_path) + assert unit == "sections" + assert [(c.id, c.kind, c.label) for c in cands] == [ + ("src_1", "source", "libhunt.com"), ("src_2", "source", "toolradar.com"), ("src_3", "source", "frase.io"), + ("own_s01", "own", "Tools"), ("own_s02", "own", "FAQ"), # intro < MIN_SECTION_WORDS, frontmatter removed + ] + assert all("title:" not in c.text for c in cands) + assert labels == {"src_1": 1, "src_2": 1, "src_3": 0} -def test_own_not_in_top_k_is_predicted_uncited() -> None: - res = ev.score_query(ITEM, DOCS, FixedJev({"src_1": 0.9, "src_2": 0.2, "src_3": 0.5, "own": 0.4}), "egeoagents.com") - assert res["own_rank"] == 3 and res["own_predicted_cited"] is False +def test_build_query_candidates_without_own(tmp_path: Path) -> None: + _, _, unit = ev.build_query_candidates(dict(ITEM, own_url=""), DOCS, "egeoagents.com") + assert unit is None -def test_score_query_choice_scorer() -> None: - res = ev.score_query(ITEM, DOCS, FixedJev({"src_1": 0.4, "src_2": 0.1, "src_3": 0.2, "own": 0.3}), - "egeoagents.com", scorer="choice") - assert res["scorer"] == "choice" and res["auc"] == 0.5 - assert res["own_score"] == 0.3 and res["own_rank"] == 2 and res["own_predicted_cited"] is True - assert res["winner"] == "src_1" +def test_missing_own_source_fails_loudly(tmp_path: Path) -> None: + with pytest.raises(FileNotFoundError): + ev.build_query_candidates(dict(ITEM, own_source="content/nope.md"), DOCS, "egeoagents.com", repo_root=tmp_path) -def test_unknown_scorer() -> None: - with pytest.raises(ValueError): - ev.score_query(ITEM, DOCS, FixedJev({}), "egeoagents.com", scorer="vibes") + +def test_score_query_blind_uses_product_scoring() -> None: + fake = FixedJev({"src_1": 0.9, "src_2": 0.2, "src_3": 0.5, "own": 0.6}) + res = ev.score_query(ITEM, DOCS, fake, "egeoagents.com") + assert res == { + "query": ITEM["query"], "scorer": "noul", "ids": "blind", "unit": "page_excerpt", + "auc": 0.5, "n_pos": 2, "n_neg": 1, + "own_best_score": 0.6, "src_best_score": 0.9, "own_rank": 2, "own_wins": False, + "egeo_cited": False, "winner": "src_1", "jev_model": "jev-1.13.0", + } + state, questions = fake.requests[0] + assert len(fake.requests) == 1 + assert all(re.match(r"^c\d{2}$", k) for k in [*state["candidates"], *questions]) + assert {q["instructions"] for q in questions.values()} == {judge.NOUL_INSTRUCTIONS.format(cid=k) for k in questions} + + +def test_score_query_leaky_sends_original_ids() -> None: + fake = FixedJev({"src_1": 0.9}) + res = ev.score_query(ITEM, DOCS, fake, "egeoagents.com", ids="leaky") + assert res["ids"] == "leaky" + assert list(fake.requests[0][0]["candidates"]) == ["src_1", "src_2", "src_3", "own"] + + +def test_own_wins_is_strict_and_uses_best_section(tmp_path: Path) -> None: + rel = _write_page(tmp_path) + texts = {**TEXT_OF, **_sections_texts(tmp_path)} + item = dict(ITEM, own_source=rel) + win = ev.score_query(item, DOCS, FixedJev({"src_1": 0.7, "own_s01": 0.2, "own_s02": 0.8}, texts), "egeoagents.com", + repo_root=tmp_path) + assert win["unit"] == "sections" and win["own_best_score"] == 0.8 and win["own_wins"] is True + assert win["own_rank"] == 1 and win["winner"] == "own_s02" + tie = ev.score_query(item, DOCS, FixedJev({"src_1": 0.8, "own_s01": 0.8}, texts), "egeoagents.com", repo_root=tmp_path) + assert tie["own_wins"] is False and tie["own_rank"] == 1 def test_score_query_too_few_candidates() -> None: item = dict(ITEM, cited_urls=[], serp_urls=[], own_url="") assert ev.score_query(item, DOCS, FixedJev({}), "egeoagents.com") == { - "query": ITEM["query"], "scorer": "noul", "skipped": "too_few_candidates"} + "query": ITEM["query"], "scorer": "noul", "ids": "blind", "skipped": "too_few_candidates"} + + +def test_unknown_ids_mode() -> None: + with pytest.raises(ValueError): + ev.score_query(ITEM, DOCS, FixedJev({}), "egeoagents.com", ids="vibes") def test_bootstrap_ci() -> None: @@ -129,6 +164,14 @@ def test_bootstrap_ci() -> None: assert low <= sum(values) / len(values) <= high and low < high +def test_wilson_ci() -> None: + assert ev.wilson_ci(0, 0) is None + low, high = ev.wilson_ci(5, 10) + assert low == pytest.approx(0.2366, abs=1e-4) and high == pytest.approx(0.7634, abs=1e-4) + low, high = ev.wilson_ci(0, 4) + assert low == pytest.approx(0.0, abs=1e-12) and 0 < high < 1 + + def _fetch_factory(calls): def fetch(urls): calls.append(sorted(urls)) @@ -136,20 +179,57 @@ def fetch(urls): return fetch -def test_run_noul_aggregates_and_gate() -> None: +def test_run_aggregates_and_gate() -> None: calls = [] dataset = [dict(ITEM, query=f"q{i}") for i in range(25)] client = FixedJev({"src_1": 0.9, "src_2": 0.6, "src_3": 0.1, "own": 0.2}) - out = ev.run(dataset, client, _fetch_factory(calls), own_domain="egeoagents.com") + out = ev.run(dataset, client, _fetch_factory(calls), own_domain="egeoagents.com", prereg_sha="abc123") assert len(calls) == 1 - assert out["scorer"] == "noul" + assert out["scorer"] == "noul" and out["ids"] == "blind" and out["gating"] is True + assert out["prereg_sha"] == "abc123" assert out["mean_auc"] == 1.0 and out["n_scored"] == 25 and out["gate_passed"] is True assert out["mean_auc_ci"] == [1.0, 1.0] - assert out["own_accuracy"] == 1.0 # own rank 3 > 2 cited -> predicted uncited == egeo_cited False + assert out["unit_counts"] == {"page_excerpt": 25} + # own metrics count sections items only + assert out["own_n"] == 0 and out["own_accuracy"] is None and out["own_accuracy_ci95"] is None + assert out["own_confusion"] == {"tp": 0, "fp": 0, "fn": 0, "tn": 0} assert out["jev_usage"]["requests"] == 25 and out["jev_model"] == "fake" + assert out["jev_models_returned"] == ["jev-1.13.0"] assert len(out["per_query"]) == 25 and "date" in out +def test_run_own_confusion_on_sections(tmp_path: Path) -> None: + rel = _write_page(tmp_path) + texts = {**TEXT_OF, **_sections_texts(tmp_path)} + dataset = ([dict(ITEM, query=f"w{i}", own_source=rel, egeo_cited=True) for i in range(3)] + + [dict(ITEM, query="l", own_source=rel, egeo_cited=True)]) + calls = [] + client = FixedJev({"src_1": 0.5, "src_2": 0.4, "src_3": 0.1, "own_s01": 0.9}, texts) + # the last query loses: make its own text score low by giving it a different page + (tmp_path / "content" / "low.md").write_text("## Other\n\n" + " ".join(["other"] * 35) + "\n", encoding="utf-8") + dataset[-1]["own_source"] = "content/low.md" + out = ev.run(dataset, client, _fetch_factory(calls), own_domain="egeoagents.com", repo_root=tmp_path, prereg_sha="s") + assert OWN_URL not in calls[0] # own_url is not fetched when the page comes from own_source + assert out["unit_counts"] == {"sections": 4} + assert out["own_confusion"] == {"tp": 3, "fp": 0, "fn": 1, "tn": 0} + assert out["own_n"] == 4 and out["own_accuracy"] == 0.75 + low, high = out["own_accuracy_ci95"] + assert low < 0.75 < high + + +def test_leaky_run_never_gates() -> None: + dataset = [dict(ITEM, query=f"q{i}") for i in range(25)] + out = ev.run(dataset, FixedJev({"src_1": 0.9, "src_2": 0.6, "src_3": 0.1}), _fetch_factory([]), + own_domain="egeoagents.com", ids="leaky", prereg_sha="abc123") + assert out["mean_auc"] == 1.0 and out["gating"] is False and out["gate_passed"] is False + + +def test_run_without_prereg_never_gates() -> None: + dataset = [dict(ITEM, query=f"q{i}") for i in range(25)] + out = ev.run(dataset, FixedJev({"src_1": 0.9, "src_2": 0.6, "src_3": 0.1}), _fetch_factory([]), own_domain="egeoagents.com") + assert out["mean_auc"] == 1.0 and out["prereg_sha"] is None and out["gating"] is False and out["gate_passed"] is False + + def test_gate_needs_25_queries() -> None: dataset = [dict(ITEM, query=f"q{i}") for i in range(24)] out = ev.run(dataset, FixedJev({"src_1": 0.9, "src_2": 0.6, "src_3": 0.1}), _fetch_factory([]), own_domain="egeoagents.com") @@ -162,8 +242,16 @@ def test_gate_fails_on_low_auc() -> None: assert out["mean_auc"] == 0.0 and out["gate_passed"] is False -def test_run_choice_scorer() -> None: - dataset = [dict(ITEM, query=f"q{i}") for i in range(3)] - out = ev.run(dataset, FixedJev({"src_1": 0.5, "src_2": 0.3, "src_3": 0.1}), _fetch_factory([]), - own_domain="egeoagents.com", scorer="choice") - assert out["scorer"] == "choice" and out["n_scored"] == 3 and out["gate_passed"] is False +def test_main_live_run_requires_prereg_sha(tmp_path: Path, capsys) -> None: + ds = tmp_path / "ds.json"; ds.write_text(json.dumps([ITEM]), encoding="utf-8") + out = tmp_path / "out.json" + assert ev.main(["--dataset", str(ds), "--out", str(out)]) == 2 + assert "--prereg-sha" in capsys.readouterr().err and not out.exists() + + +def test_main_mock_writes_result(tmp_path: Path) -> None: + ds = tmp_path / "ds.json"; ds.write_text(json.dumps([ITEM]), encoding="utf-8") + out = tmp_path / "out.json" + assert ev.main(["--dataset", str(ds), "--out", str(out), "--mock", "--ids", "leaky"]) == 0 + res = json.loads(out.read_text(encoding="utf-8")) + assert res["ids"] == "leaky" and res["gating"] is False and res["jev_model"] == "mock" diff --git a/tests/test_judge.py b/tests/test_judge.py index 60b9c44..4098f09 100644 --- a/tests/test_judge.py +++ b/tests/test_judge.py @@ -1,75 +1,147 @@ -"""Contract tests for egeo.judge. A scripted fake Jev client; never the network.""" +"""Contract tests for egeo.judge (v4: blind ids, Noul scoring). A scripted fake Jev client; never the network.""" from __future__ import annotations +import hashlib +import re + import pytest from egeo import judge -from egeo.jev import ChoiceAnswer, JevResponse -from egeo.judge import Candidate, Selection, diagnose, judge_fidelity, select_best, verify +from egeo.jev import ChoiceAnswer, JevResponse, NoulAnswer +from egeo.judge import Candidate, Scored, blind_ids, diagnose, judge_fidelity, score_candidates, verify from egeo.sections import split_sections from egeo.sources import SourceDoc +BLIND_ID = re.compile(r"^c\d{2}$") + class ScriptedJev: - """Returns queued ChoiceAnswers in order and records every request.""" + """Choice questions: pops the next queued ChoiceAnswer. Noul questions: looks up the score of the + candidate text the question points at, via ``by_text`` (a function text -> score). Records requests.""" - def __init__(self, *answers: ChoiceAnswer) -> None: + def __init__(self, *answers: ChoiceAnswer, by_text=None) -> None: self.queue = list(answers) + self.by_text = by_text or (lambda text: 0.0) self.requests = [] def ask(self, state, questions): self.requests.append((state, questions)) - (qid,) = questions.keys() - return JevResponse(answers={qid: self.queue.pop(0)}, model="fake", input_tokens=10, output_tokens=1, cost_usd=0.0) + answers = {} + for qid, q in questions.items(): + if q["type"] == "noul": + answers[qid] = NoulAnswer(self.by_text(state["candidates"][qid])) + else: + answers[qid] = self.queue.pop(0) + return JevResponse(answers=answers, model="jev-test", input_tokens=10, output_tokens=1, cost_usd=0.0) + + +def scores_by_text(table): + """Fake Noul: score = table[text] (texts are unique in these tests).""" + return lambda text: table[text] OWN = [Candidate("own_s01", "own", "Intro", "E-GEO rewrites pages."), Candidate("own_s02", "own", "Tools", "A table of tools.")] SRC = [Candidate("src_1", "source", "libhunt.com", "LibHunt list"), Candidate("src_2", "source", "toolradar.com", "x" * 3000)] +ALL = [*OWN, *SRC] + + +def _expected_order(query, cands): + return sorted(cands, key=lambda c: (hashlib.sha256(f"{query}\n{c.text}".encode()).hexdigest(), c.id)) + + +def test_blind_ids_order_depends_only_on_query_and_text() -> None: + m = blind_ids("best geo tools", ALL) + assert m == {c.id: f"c{i:02d}" for i, c in enumerate(_expected_order("best geo tools", ALL), 1)} + assert blind_ids("best geo tools", list(reversed(ALL))) == m # input order does not matter + assert sorted(m.values()) == ["c01", "c02", "c03", "c04"] + + +def test_blind_ids_ties_break_on_original_id() -> None: + twins = [Candidate("src_2", "source", "b", "same"), Candidate("src_1", "source", "a", "same")] + assert blind_ids("q", twins) == {"src_1": "c01", "src_2": "c02"} -def test_select_best_request_shape_and_result() -> None: - fake = ScriptedJev(ChoiceAnswer("src_1", 0.3, {"own_s01": 0.1, "own_s02": 0.3, "src_1": 0.5, "src_2": 0.1})) - sel = select_best("best geo tools", [*OWN, *SRC], fake) +def test_score_candidates_blind_request_shape() -> None: + table = {c.text: s for c, s in zip(ALL, (0.1, 0.3, 0.5, 0.2))} + fake = ScriptedJev(by_text=scores_by_text({**table, "x" * judge.MAX_CANDIDATE_CHARS: 0.2})) + scored = score_candidates("best geo tools", ALL, fake) + assert len(fake.requests) == 1 state, questions = fake.requests[0] + m = blind_ids("best geo tools", ALL) + assert scored == Scored({"own_s01": 0.1, "own_s02": 0.3, "src_1": 0.5, "src_2": 0.2}, m, "jev-test") assert state["query"] == "best geo tools" - assert state["candidates"]["src_2"] == "x" * judge.MAX_CANDIDATE_CHARS - q = questions[judge.SELECT_QID] - assert q["type"] == "choice" and q["instructions"] == judge.SELECT_INSTRUCTIONS - assert q["criteria"] == {c.id: f"The candidate whose text is candidates.{c.id}" for c in [*OWN, *SRC]} - assert sel == Selection("src_1", "source", {"own_s01": 0.1, "own_s02": 0.3, "src_1": 0.5, "src_2": 0.1}, 0.3) + assert all(BLIND_ID.match(k) for k in state["candidates"]) + assert list(state["candidates"]) == sorted(state["candidates"]) # sent in blind order + assert state["candidates"][m["src_2"]] == "x" * judge.MAX_CANDIDATE_CHARS + assert questions == {m[c.id]: {"type": "noul", "instructions": judge.NOUL_INSTRUCTIONS.format(cid=m[c.id])} for c in ALL} + # no original id, host or heading reaches Jev + blob = repr((state, questions)) + for c in ALL: + assert c.id not in blob and c.label not in blob + + +def test_score_candidates_leaky_sends_original_ids() -> None: + fake = ScriptedJev(by_text=lambda t: 0.5) + scored = score_candidates("q", ALL, fake, ids="leaky") + state, questions = fake.requests[0] + assert list(state["candidates"]) == [c.id for c in ALL] and list(questions) == [c.id for c in ALL] + assert scored.id_map == {c.id: c.id for c in ALL} + +def test_score_candidates_reuses_given_id_map() -> None: + m = {"own_s01": "c02", "src_1": "c01"} + fake = ScriptedJev(by_text=lambda t: 0.4) + scored = score_candidates("q", [OWN[0], SRC[0]], fake, id_map=m) + assert set(fake.requests[0][0]["candidates"]) == {"c01", "c02"} + assert scored.id_map == m + with pytest.raises(ValueError): + score_candidates("q", [OWN[0], SRC[0]], fake, id_map={"own_s01": "c01"}) # map misses a candidate -def test_select_best_needs_two_unique_candidates() -> None: + +def test_score_candidates_validates_input() -> None: + with pytest.raises(ValueError): + score_candidates("q", OWN[:1], ScriptedJev()) with pytest.raises(ValueError): - select_best("q", OWN[:1], ScriptedJev()) + score_candidates("q", [OWN[0], OWN[0]], ScriptedJev()) with pytest.raises(ValueError): - select_best("q", [OWN[0], OWN[0]], ScriptedJev()) + score_candidates("q", ALL, ScriptedJev(), ids="vibes") def test_diagnose_loses_targets_best_own_section() -> None: - fake = ScriptedJev(ChoiceAnswer("src_1", 0.3, {"own_s01": 0.1, "own_s02": 0.3, "src_1": 0.5, "src_2": 0.1})) + fake = ScriptedJev(by_text=scores_by_text({OWN[0].text: 0.1, OWN[1].text: 0.3, SRC[0].text: 0.5, "x" * 1500: 0.1})) d = diagnose("q", OWN, SRC, fake) - assert d.status == "loses" and d.target_id == "own_s02" and d.selection.winner == "src_1" + assert d.status == "loses" and d.target_id == "own_s02" + assert d.scored.scores == {"own_s01": 0.1, "own_s02": 0.3, "src_1": 0.5, "src_2": 0.1} def test_diagnose_tie_prefers_first_own() -> None: - fake = ScriptedJev(ChoiceAnswer("src_1", 0.3, {"own_s01": 0.2, "own_s02": 0.2, "src_1": 0.6})) + fake = ScriptedJev(by_text=scores_by_text({OWN[0].text: 0.2, OWN[1].text: 0.2, SRC[0].text: 0.6})) assert diagnose("q", OWN, SRC[:1], fake).target_id == "own_s01" -def test_diagnose_already_best() -> None: - fake = ScriptedJev(ChoiceAnswer("own_s01", 0.5, {"own_s01": 0.7, "own_s02": 0.1, "src_1": 0.2})) +def test_diagnose_already_best_needs_strictly_higher_score() -> None: + fake = ScriptedJev(by_text=scores_by_text({OWN[0].text: 0.7, OWN[1].text: 0.1, SRC[0].text: 0.2})) d = diagnose("q", OWN, SRC[:1], fake) assert d.status == "already_best" and d.target_id is None + tie = ScriptedJev(by_text=scores_by_text({OWN[0].text: 0.6, OWN[1].text: 0.1, SRC[0].text: 0.6})) + d = diagnose("q", OWN, SRC[:1], tie) + assert d.status == "loses" and d.target_id == "own_s01" def test_diagnose_without_own_or_sources_does_not_call_jev() -> None: fake = ScriptedJev() assert diagnose("q", [], SRC, fake).status == "no_own_candidates" assert diagnose("q", OWN, [], fake).status == "no_competitor_sources" + assert diagnose("q", [], SRC, fake).scored is None assert fake.requests == [] +def test_leader() -> None: + scores = {"own_s01": 0.4, "own_s02": 0.4, "src_1": 0.4, "src_2": 0.1} + assert judge.leader(Scored(scores, {}), OWN, SRC) == SRC[0] # tie is not an own win + assert judge.leader(Scored(dict(scores, own_s02=0.5), {}), OWN, SRC) == OWN[1] + + @pytest.mark.parametrize( "answer,accepted", [ @@ -88,23 +160,34 @@ def test_judge_fidelity_gate(answer: ChoiceAnswer, accepted: bool) -> None: assert (v.verdict, v.confidence, v.accepted) == (answer.choice, answer.confidence, accepted) -BEFORE = Selection("src_1", "source", {"own_s01": 0.1, "own_s02": 0.30, "src_1": 0.6}, 0.3) +REWRITE = "A comparison table of the best GEO tools." @pytest.mark.parametrize( - "after,outcome", + "after_score,src_score,outcome", [ - (ChoiceAnswer("own_s02", 0.5, {"own_s01": 0.1, "own_s02": 0.7, "src_1": 0.2}), "won"), - (ChoiceAnswer("src_1", 0.2, {"own_s01": 0.1, "own_s02": 0.35, "src_1": 0.55}), "improved"), - (ChoiceAnswer("src_1", 0.3, {"own_s01": 0.1, "own_s02": 0.32, "src_1": 0.58}), "no_change"), - (ChoiceAnswer("src_1", 0.5, {"own_s01": 0.1, "own_s02": 0.25, "src_1": 0.65}), "worse"), + (0.70, 0.60, "won"), + (0.60, 0.60, "improved"), # a tie is not a win + (0.35, 0.60, "improved"), # exactly +IMPROVE_DELTA + (0.32, 0.60, "no_change"), + (0.25, 0.60, "worse"), # exactly -IMPROVE_DELTA ], ) -def test_verify_outcomes(after: ChoiceAnswer, outcome: str) -> None: - v = verify("q", OWN, SRC[:1], "own_s02", BEFORE, ScriptedJev(after)) +def test_verify_outcomes(after_score: float, src_score: float, outcome: str) -> None: + before_fake = ScriptedJev(by_text=scores_by_text({OWN[0].text: 0.1, OWN[1].text: 0.30, SRC[0].text: 0.6})) + before = diagnose("q", OWN, SRC[:1], before_fake).scored + own_after = [OWN[0], Candidate("own_s02", "own", "Tools", REWRITE)] + fake = ScriptedJev(by_text=scores_by_text({OWN[0].text: 0.1, REWRITE: after_score, SRC[0].text: src_score})) + v = verify("q", own_after, SRC[:1], "own_s02", before, fake) assert v.outcome == outcome - assert v.p_before == 0.30 and v.p_after == after.probabilities["own_s02"] - assert v.winner_after == after.choice + assert v.score_before == 0.30 and v.score_after == after_score + assert v.winner_after == ("own_s02" if outcome == "won" else "src_1") + assert v.winner_after_kind == ("own" if outcome == "won" else "source") + # same ids as the diagnosis, even though the target text changed + state_before, _ = before_fake.requests[0] + state_after, _ = fake.requests[0] + assert list(state_after["candidates"]) == list(state_before["candidates"]) + assert state_after["candidates"][before.id_map["own_s02"]] == REWRITE def test_candidate_builders() -> None: @@ -115,3 +198,8 @@ def test_candidate_builders() -> None: SourceDoc("https://c.com/", True, "t", "text c")] assert [(c.id, c.label, c.text) for c in judge.source_candidates(docs)] == [ ("src_1", "libhunt.com", "text a"), ("src_2", "c.com", "text c")] + + +def test_select_best_is_gone() -> None: + # D2 = Noul (owner decision 2026-10-07): the single Choice comparison no longer exists. + assert not hasattr(judge, "select_best") and not hasattr(judge, "Selection")