From c65ba4581c871813449ec6f16c3a621f6a6ce121 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 16 Aug 2026 19:07:59 +0000 Subject: [PATCH 1/3] Scan record: name the tree the after-campaign counts hold at (0.34.0) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The record carried one commit field and it named the before state, so every after-state number a campaign writes into `summary` — `testsAfter` and anything else measured post-landing — held at a tree the schema never named. That is not wrong-and-detectable, it is outside the schema's reach: `rain.sol.codegen` committed `testsAfter: 102`, a count occurring at no commit in the range its record covers, and neither a reader nor a tool had anything to check it against. `testsAfterCommit` is the tree the run's own output landed at, immediately after `commit`, both full 40-character SHAs. It is a must-have, not optional, and a run that landed nothing sets it equal to `commit` rather than null: "nothing landed" and "nobody recorded where it landed" have to stay distinguishable, and an optional field's absence means both at once. The schema is documentation, so until now nothing in this repo executed it. `.github/scripts/check-scan-record-schema.sh` does: the README template parses as JSON, carries every must-have, keeps `testsAfterCommit` adjacent to `commit`, holds full SHAs in both, states the rule in prose outside the fence, and SKILL.md's closing section names the field. Co-Authored-By: Claude Opus 5 (1M context) --- .claude-plugin/marketplace.json | 2 +- .claude-plugin/plugin.json | 2 +- .github/scripts/check-scan-record-schema.sh | 89 +++++++++++++++++++++ .github/workflows/schema-hygiene.yaml | 17 ++++ README.md | 30 +++++-- flake.nix | 3 + skills/adversarial-mutation-test/SKILL.md | 17 ++-- 7 files changed, 148 insertions(+), 12 deletions(-) create mode 100644 .github/scripts/check-scan-record-schema.sh create mode 100644 .github/workflows/schema-hygiene.yaml diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 5590e75..24535ed 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -10,7 +10,7 @@ "name": "adversarial-mutation-test", "source": "./", "description": "Find BUGS and harden the test suite for a whole repository — adversarial (spec as oracle, code as suspect; surface candidates for triage) + mutation (break each line, prove a test catches it). Whole-repo, resumable, language-agnostic.", - "version": "0.33.0", + "version": "0.34.0", "author": { "name": "Rain Open Source Software Ltd" }, diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index e8c7ee3..6778263 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "adversarial-mutation-test", "displayName": "Adversarial Mutation Testing", - "version": "0.33.0", + "version": "0.34.0", "description": "Find BUGS and harden the test suite for a whole repository. Two co-equal halves: ADVERSARIAL — treat the spec as the oracle and the code as suspect, hunt for inputs where the code is wrong, and surface candidates for triage (never self-adjudicate); and MUTATION — break each line and prove a test catches it. Whole-repo, resumable, language-agnostic.", "author": { "name": "Rain Open Source Software Ltd", diff --git a/.github/scripts/check-scan-record-schema.sh b/.github/scripts/check-scan-record-schema.sh new file mode 100644 index 0000000..1ab218e --- /dev/null +++ b/.github/scripts/check-scan-record-schema.sh @@ -0,0 +1,89 @@ +#!/bin/sh +# The scan-record schema lives in two places — the README's "Scan record +# template" (canonical) and SKILL.md's "Committed scan record" (what a closing +# run has in front of it) — and it is documentation, so nothing else in this +# repo executes it. That is how the schema shipped for 33 versions with no field +# naming the tree its after-campaign counts hold at: rain.sol.codegen committed +# "testsAfter": 102, a count occurring at no commit in the range its record +# covered, and no reader or tool had anything to check it against. +# +# This pins the parts of the schema a reader needs in order to falsify a record: +# the template is real JSON, both trees are named, both are full SHAs, and both +# documents state it. Run it from anywhere: sh .github/scripts/check-scan-record-schema.sh +set -eu + +root=$(CDPATH='' cd -- "$(dirname -- "$0")/../.." && pwd) +readme="$root/README.md" +skill="$root/skills/adversarial-mutation-test/SKILL.md" + +fail() { + echo "FAIL $*" >&2 + exit 1 +} + +# The fenced JSON block under the README's "## Scan record template" heading. +template=$(awk ' + /^## Scan record template$/ { in_section = 1; next } + in_section && /^## / { exit } + in_section && /^```json$/ { in_block = 1; next } + in_block && /^```$/ { exit } + in_block { print } +' "$readme") + +[ -n "$template" ] || fail "README.md has no fenced json block under '## Scan record template'" + +echo "$template" | jq -e . >/dev/null 2>&1 || + fail "the README scan record template is not valid JSON — it is the schema consumers copy" + +keys=$(echo "$template" | jq -r 'keys_unsorted | join(" ")') +commit=$(echo "$template" | jq -r '.commit // ""') +tests_after_commit=$(echo "$template" | jq -r '.testsAfterCommit // ""') +echo "template keys: $keys" +echo "template commit=$commit testsAfterCommit=$tests_after_commit" + +# The must-haves. A record missing any of these cannot be checked against the +# repo it describes. +for key in timestamp commit testsAfterCommit publishedTag commitsAheadOfTag; do + echo "$template" | jq -e --arg key "$key" 'has($key)' >/dev/null || + fail "the template is missing must-have '$key'" +done +echo "OK must-haves present: timestamp commit testsAfterCommit publishedTag commitsAheadOfTag" + +# Adjacency is what makes the pair readable as a pair: the before tree and the +# after tree sit together, ahead of everything measured at either. +echo "$template" | jq -e 'keys_unsorted | index("testsAfterCommit") == (index("commit") + 1)' >/dev/null || + fail "testsAfterCommit must come immediately after commit; key order is: $keys" +echo "OK testsAfterCommit is immediately after commit" + +# Full SHAs. A 7-char prefix is a weaker anchor than a full SHA and grows +# ambiguous as history grows — which is the same class of defect the after-tree +# field exists to close, and rain.solmem's record already carries one. +for key in commit testsAfterCommit; do + echo "$template" | jq -e --arg key "$key" '.[$key] | type == "string" and test("^[0-9a-f]{40}$")' >/dev/null || + fail "template '$key' must be a full 40-character lowercase hex SHA" +done +echo "OK commit and testsAfterCommit are full 40-character SHAs" + +# The template alone only exemplifies the field; the prose around it is what +# states the rule (which numbers hold at which tree, and that a run landing +# nothing sets them equal), so the field has to appear outside the fences too. +prose=$(awk '/^```/ { fenced = !fenced; next } !fenced' "$readme") +echo "$prose" | grep -q 'testsAfterCommit' || + fail "README.md names testsAfterCommit only inside a code fence — the rule it obeys is unstated" +echo "OK README prose states the testsAfterCommit rule" + +# SKILL.md is what a run closing its record actually has in front of it. A field +# documented only in the README is a field campaigns will not write. +section=$(awk ' + /^## Committed scan record$/ { in_section = 1; next } + in_section && /^## / { exit } + in_section { print } +' "$skill") + +[ -n "$section" ] || fail "SKILL.md has no '## Committed scan record' section" + +echo "$section" | grep -q 'testsAfterCommit' || + fail "SKILL.md's '## Committed scan record' does not name testsAfterCommit" +echo "OK SKILL.md's committed scan record names testsAfterCommit" + +echo "scan record schema OK" diff --git a/.github/workflows/schema-hygiene.yaml b/.github/workflows/schema-hygiene.yaml new file mode 100644 index 0000000..befafcc --- /dev/null +++ b/.github/workflows/schema-hygiene.yaml @@ -0,0 +1,17 @@ +name: schema hygiene +on: [push, pull_request] +permissions: + contents: read +jobs: + # The scan-record schema is documentation, so nothing else in this repo runs + # it. Kept separate from version hygiene so a red here means "the schema + # consumers copy is broken", not "a version pointer drifted". + scan-record-schema: + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + persist-credentials: false + - name: scan record template conforms to the schema it documents + run: sh .github/scripts/check-scan-record-schema.sh diff --git a/README.md b/README.md index 831805c..0ca806f 100644 --- a/README.md +++ b/README.md @@ -128,24 +128,44 @@ JSON, no comments: ```json { "timestamp": "2026-08-12T19:40:00Z", - "commit": "08d547fdeadbeef", + "commit": "08d547fdeadbeefc0ffee1122334455667788990", + "testsAfterCommit": "1f9be22cafebabe0ddf00d998877665544332211", "publishedTag": "v1.2.3", "commitsAheadOfTag": 0, "scope": "whole repo", "tool": "adversarial-mutation-test", - "skillVersion": "0.33.0", + "skillVersion": "0.34.0", "summary": { "behaviours": 600, "candidates": 89, "confirmed": 30, + "testsBefore": 41, + "testsAfter": 84, "filed": ["#2651", "#2660"] } } ``` -`timestamp` is UTC at run end; `commit` the exact SHA scanned; `publishedTag` -the release at that commit (null if unreleased) with `commitsAheadOfTag` its -distance. Those three are the must-haves; `summary` is nice-to-have. +`timestamp` is UTC at run end; `commit` the exact SHA scanned; `testsAfterCommit` +the exact SHA the run's own output landed at; `publishedTag` the release at +`commit` (null if unreleased) with `commitsAheadOfTag` its distance. Those four +are the must-haves; `summary` is nice-to-have. + +A record spans two trees, and every number in it is measured at one of them: +`commit` is the tree the scan ran against, which every *before* number +(`testsBefore`, baseline counts) holds at; `testsAfterCommit` is the tree with +the run's coverage PRs merged, which every *after* number (`testsAfter`, and +anything else measured post-landing) holds at. Both are full 40-character SHAs — +a short prefix is a weaker anchor and grows ambiguous as history grows. + +`testsAfterCommit` is never null and never omitted: a run that landed nothing +sets it **equal to `commit`**. "Nothing landed" and "nobody recorded where it +landed" have to stay distinguishable, so an absent field means the record is +malformed rather than that the run was clean. This is the field that makes an +after-state count checkable at all — `rain.sol.codegen` committed +`testsAfter: 102`, a count that occurs at no commit in the range the record +covers, and nothing could catch it because the record named no tree to check it +against. ## License diff --git a/flake.nix b/flake.nix index 6fd363f..414ab2b 100644 --- a/flake.nix +++ b/flake.nix @@ -54,6 +54,9 @@ pkgs.rustc pkgs.clippy pkgs.rustfmt + # `.github/scripts/` runs on it, so a hygiene check is runnable + # locally the same way CI runs it rather than checked by eye. + pkgs.jq ]; }; } diff --git a/skills/adversarial-mutation-test/SKILL.md b/skills/adversarial-mutation-test/SKILL.md index c2460b4..baf72b0 100644 --- a/skills/adversarial-mutation-test/SKILL.md +++ b/skills/adversarial-mutation-test/SKILL.md @@ -1,6 +1,6 @@ --- name: adversarial-mutation-test -version: 0.33.0 +version: 0.34.0 description: Use to systematically find BUGS in and harden the test suite for a WHOLE repository (or a whole module of it). Two co-equal goals the name carries: ADVERSARIAL (treat spec/intent as the oracle and the code as suspect — derive expected behavior independently and hunt for inputs where the code is wrong) and MUTATION (prove tests cover the code). Mutation-drives a behavior-centric coverage ledger — for each behavior, break the line and check the whole suite: existing tests that kill the mutant are validated and logged (so existing coverage is audited and in scope), and only surviving mutants (real gaps) get a new discriminating test. An existing test that already kills mutants is left as-is; one meant to cover a behavior but that a mutant survives is strengthened in place (not duplicated); one broken on the unmutated baseline is fixed or its underlying code bug surfaced; a test is never edited to swallow a mutation. Designed for long campaigns that outlive the context window: progress lives in a durable gitignored scratch file so it survives compaction. A single change/PR/function is just a narrowed scope. Triggers on "test the whole repo", "harden the test suite", "mutation test the codebase", "audit the tests", "adversarial tests", "prove these tests cover the code", "exhaust the eventualities". --- @@ -196,10 +196,17 @@ One mutation, one behavior — the failing-test set stays diagnostic. Close every run — including a clean one — by appending an entry to a committed `audit/mutation-test-scans.json` and landing it on the default branch: -timestamp, scanned commit, published tag (+ commits ahead), scope, tool + skill -version, summary with filed issue numbers. The org health check reads the newest -entry for "which release was last audited"; the JSON template lives in this -repo's README. +timestamp, scanned commit, `testsAfterCommit`, published tag (+ commits ahead), +scope, tool + skill version, summary with filed issue numbers. The org health +check reads the newest entry for "which release was last audited"; the JSON +template lives in this repo's README. + +Two trees, both full 40-char SHAs: the scanned `commit` is what every *before* +number holds at; `testsAfterCommit` — the tree with this run's coverage PRs +merged — is what every *after* number in `summary` holds at. An after-count with +no tree named is unfalsifiable, so `testsAfterCommit` is a must-have alongside +the scanned commit: a run that landed nothing sets it equal to `commit`, never +null and never omitted. ## Principles From d5213c2ae472f9abe79d8f53151d6a51955b1dee Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 16 Aug 2026 19:14:52 +0000 Subject: [PATCH 2/3] Apply the rainix denofmt pass, and drop an unverified count from a comment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `denofmt` reflows the must-haves paragraph and writes emphasis as `_x_`, not `*x*`; the diff here is exactly what `pre-commit run --all-files` produced in `static / rs-static`. The check script's header claimed the schema "shipped for 33 versions" without the after-tree field. The README template was added at 0.31.0 and the record format predates it, so the number was never derived — the sentence now says only what is true, that nothing in this repo executed the schema. Co-Authored-By: Claude Opus 5 (1M context) --- .github/scripts/check-scan-record-schema.sh | 8 ++++---- README.md | 12 ++++++------ skills/adversarial-mutation-test/SKILL.md | 4 ++-- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/scripts/check-scan-record-schema.sh b/.github/scripts/check-scan-record-schema.sh index 1ab218e..b7fd3a4 100644 --- a/.github/scripts/check-scan-record-schema.sh +++ b/.github/scripts/check-scan-record-schema.sh @@ -2,10 +2,10 @@ # The scan-record schema lives in two places — the README's "Scan record # template" (canonical) and SKILL.md's "Committed scan record" (what a closing # run has in front of it) — and it is documentation, so nothing else in this -# repo executes it. That is how the schema shipped for 33 versions with no field -# naming the tree its after-campaign counts hold at: rain.sol.codegen committed -# "testsAfter": 102, a count occurring at no commit in the range its record -# covered, and no reader or tool had anything to check it against. +# repo executes it. That is how it shipped with no field naming the tree its +# after-campaign counts hold at: rain.sol.codegen committed "testsAfter": 102, a +# count occurring at no commit in the range its record covered, and no reader or +# tool had anything to check it against. # # This pins the parts of the schema a reader needs in order to falsify a record: # the template is real JSON, both trees are named, both are full SHAs, and both diff --git a/README.md b/README.md index 0ca806f..dab1114 100644 --- a/README.md +++ b/README.md @@ -146,15 +146,15 @@ JSON, no comments: } ``` -`timestamp` is UTC at run end; `commit` the exact SHA scanned; `testsAfterCommit` -the exact SHA the run's own output landed at; `publishedTag` the release at -`commit` (null if unreleased) with `commitsAheadOfTag` its distance. Those four -are the must-haves; `summary` is nice-to-have. +`timestamp` is UTC at run end; `commit` the exact SHA scanned; +`testsAfterCommit` the exact SHA the run's own output landed at; `publishedTag` +the release at `commit` (null if unreleased) with `commitsAheadOfTag` its +distance. Those four are the must-haves; `summary` is nice-to-have. A record spans two trees, and every number in it is measured at one of them: -`commit` is the tree the scan ran against, which every *before* number +`commit` is the tree the scan ran against, which every _before_ number (`testsBefore`, baseline counts) holds at; `testsAfterCommit` is the tree with -the run's coverage PRs merged, which every *after* number (`testsAfter`, and +the run's coverage PRs merged, which every _after_ number (`testsAfter`, and anything else measured post-landing) holds at. Both are full 40-character SHAs — a short prefix is a weaker anchor and grows ambiguous as history grows. diff --git a/skills/adversarial-mutation-test/SKILL.md b/skills/adversarial-mutation-test/SKILL.md index baf72b0..e550b59 100644 --- a/skills/adversarial-mutation-test/SKILL.md +++ b/skills/adversarial-mutation-test/SKILL.md @@ -201,9 +201,9 @@ scope, tool + skill version, summary with filed issue numbers. The org health check reads the newest entry for "which release was last audited"; the JSON template lives in this repo's README. -Two trees, both full 40-char SHAs: the scanned `commit` is what every *before* +Two trees, both full 40-char SHAs: the scanned `commit` is what every _before_ number holds at; `testsAfterCommit` — the tree with this run's coverage PRs -merged — is what every *after* number in `summary` holds at. An after-count with +merged — is what every _after_ number in `summary` holds at. An after-count with no tree named is unfalsifiable, so `testsAfterCommit` is a must-have alongside the scanned commit: a run that landed nothing sets it equal to `commit`, never null and never omitted. From afe795d928a01e9da3182c85079b400982054d6d Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 16 Aug 2026 19:19:07 +0000 Subject: [PATCH 3/3] Pin the rules, not just the field name (CodeRabbit #24) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two findings, both real. The check accepted a README that named `testsAfterCommit` anywhere outside a fence, so it would have passed a document that kept the field in one sentence and dropped every rule attached to it — which numbers hold at which tree, that both trees are full SHAs, and that a run landing nothing sets the field equal to `commit` rather than null. Each rule is now pinned by the shortest phrase that carries it, scoped to the scan-record section rather than the whole README, and the failure names which rule went missing. SKILL.md's closing section is pinned the same way for the two rules a run applies when it writes its record. Newlines are collapsed before matching. `deno fmt` reflows both documents, so a phrase split across lines is not a rule going missing — `never null` was already wrapped that way in SKILL.md. "Those four are the must-haves" listed five fields, which the validator also requires. The clause counted semicolons, not fields; it now says five. Co-Authored-By: Claude Opus 5 (1M context) --- .github/scripts/check-scan-record-schema.sh | 52 ++++++++++++++++----- README.md | 2 +- 2 files changed, 42 insertions(+), 12 deletions(-) diff --git a/.github/scripts/check-scan-record-schema.sh b/.github/scripts/check-scan-record-schema.sh index b7fd3a4..7b3b1bf 100644 --- a/.github/scripts/check-scan-record-schema.sh +++ b/.github/scripts/check-scan-record-schema.sh @@ -64,16 +64,43 @@ for key in commit testsAfterCommit; do done echo "OK commit and testsAfterCommit are full 40-character SHAs" -# The template alone only exemplifies the field; the prose around it is what -# states the rule (which numbers hold at which tree, and that a run landing -# nothing sets them equal), so the field has to appear outside the fences too. -prose=$(awk '/^```/ { fenced = !fenced; next } !fenced' "$readme") -echo "$prose" | grep -q 'testsAfterCommit' || - fail "README.md names testsAfterCommit only inside a code fence — the rule it obeys is unstated" -echo "OK README prose states the testsAfterCommit rule" +# The template exemplifies the field; only the prose states the RULES it obeys, +# and a record is falsifiable because of the rules, not because of the field. A +# check that accepted the name alone would pass a README that had kept +# `testsAfterCommit` in one sentence and dropped every rule attached to it, so +# each rule is pinned by the shortest phrase that carries it. Reword freely — the +# failure names which rule went missing, and re-pinning it is a one-line edit. +readme_prose=$(awk ' + /^## Scan record template$/ { in_section = 1; next } + in_section && /^## / { exit } + in_section && /^```/ { fenced = !fenced; next } + in_section && !fenced { print } +' "$readme") + +[ -n "$readme_prose" ] || fail "README.md has no prose under '## Scan record template' — only the template" + +# Newlines are collapsed first: these documents are reflowed by `deno fmt`, so a +# phrase may be split across lines at any time and that is not a rule going +# missing. +states() { + printf '%s' "$2" | tr '\n' ' ' | tr -s ' ' | grep -qF -- "$1" || + fail "$3 no longer states $4 (looked for '$1')" +} + +readme_states() { states "$1" "$readme_prose" "the README scan record prose" "$2"; } + +readme_states 'testsAfterCommit' "which field names the tree after-state counts hold at" +readme_states '_before_' "that before-numbers hold at the scanned commit" +readme_states '_after_' "that after-numbers hold at testsAfterCommit" +readme_states '40-character' "that both trees are named by full-length SHAs" +readme_states 'equal to' "that a run landing nothing sets testsAfterCommit equal to commit" +readme_states 'never null' "that testsAfterCommit is never null" +readme_states 'never omitted' "that testsAfterCommit is never omitted" +echo "OK README prose states every testsAfterCommit rule" # SKILL.md is what a run closing its record actually has in front of it. A field -# documented only in the README is a field campaigns will not write. +# documented only in the README is a field campaigns will not write, and a rule +# only the README states is a rule the closing run does not apply. section=$(awk ' /^## Committed scan record$/ { in_section = 1; next } in_section && /^## / { exit } @@ -82,8 +109,11 @@ section=$(awk ' [ -n "$section" ] || fail "SKILL.md has no '## Committed scan record' section" -echo "$section" | grep -q 'testsAfterCommit' || - fail "SKILL.md's '## Committed scan record' does not name testsAfterCommit" -echo "OK SKILL.md's committed scan record names testsAfterCommit" +skill_states() { states "$1" "$section" "SKILL.md's '## Committed scan record'" "$2"; } + +skill_states 'testsAfterCommit' "which field names the tree after-state counts hold at" +skill_states 'equal to' "what a run that landed nothing writes" +skill_states 'never null' "that testsAfterCommit is never null and never omitted" +echo "OK SKILL.md's committed scan record states the field and its rules" echo "scan record schema OK" diff --git a/README.md b/README.md index dab1114..7c74539 100644 --- a/README.md +++ b/README.md @@ -149,7 +149,7 @@ JSON, no comments: `timestamp` is UTC at run end; `commit` the exact SHA scanned; `testsAfterCommit` the exact SHA the run's own output landed at; `publishedTag` the release at `commit` (null if unreleased) with `commitsAheadOfTag` its -distance. Those four are the must-haves; `summary` is nice-to-have. +distance. All five are must-haves; `summary` is nice-to-have. A record spans two trees, and every number in it is measured at one of them: `commit` is the tree the scan ran against, which every _before_ number