diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 5590e75..24535ed 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -10,7 +10,7 @@ "name": "adversarial-mutation-test", "source": "./", "description": "Find BUGS and harden the test suite for a whole repository — adversarial (spec as oracle, code as suspect; surface candidates for triage) + mutation (break each line, prove a test catches it). Whole-repo, resumable, language-agnostic.", - "version": "0.33.0", + "version": "0.34.0", "author": { "name": "Rain Open Source Software Ltd" }, diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index e8c7ee3..6778263 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "adversarial-mutation-test", "displayName": "Adversarial Mutation Testing", - "version": "0.33.0", + "version": "0.34.0", "description": "Find BUGS and harden the test suite for a whole repository. Two co-equal halves: ADVERSARIAL — treat the spec as the oracle and the code as suspect, hunt for inputs where the code is wrong, and surface candidates for triage (never self-adjudicate); and MUTATION — break each line and prove a test catches it. Whole-repo, resumable, language-agnostic.", "author": { "name": "Rain Open Source Software Ltd", diff --git a/.github/scripts/check-scan-record-schema.sh b/.github/scripts/check-scan-record-schema.sh new file mode 100644 index 0000000..7b3b1bf --- /dev/null +++ b/.github/scripts/check-scan-record-schema.sh @@ -0,0 +1,119 @@ +#!/bin/sh +# The scan-record schema lives in two places — the README's "Scan record +# template" (canonical) and SKILL.md's "Committed scan record" (what a closing +# run has in front of it) — and it is documentation, so nothing else in this +# repo executes it. That is how it shipped with no field naming the tree its +# after-campaign counts hold at: rain.sol.codegen committed "testsAfter": 102, a +# count occurring at no commit in the range its record covered, and no reader or +# tool had anything to check it against. +# +# This pins the parts of the schema a reader needs in order to falsify a record: +# the template is real JSON, both trees are named, both are full SHAs, and both +# documents state it. Run it from anywhere: sh .github/scripts/check-scan-record-schema.sh +set -eu + +root=$(CDPATH='' cd -- "$(dirname -- "$0")/../.." && pwd) +readme="$root/README.md" +skill="$root/skills/adversarial-mutation-test/SKILL.md" + +fail() { + echo "FAIL $*" >&2 + exit 1 +} + +# The fenced JSON block under the README's "## Scan record template" heading. +template=$(awk ' + /^## Scan record template$/ { in_section = 1; next } + in_section && /^## / { exit } + in_section && /^```json$/ { in_block = 1; next } + in_block && /^```$/ { exit } + in_block { print } +' "$readme") + +[ -n "$template" ] || fail "README.md has no fenced json block under '## Scan record template'" + +echo "$template" | jq -e . >/dev/null 2>&1 || + fail "the README scan record template is not valid JSON — it is the schema consumers copy" + +keys=$(echo "$template" | jq -r 'keys_unsorted | join(" ")') +commit=$(echo "$template" | jq -r '.commit // ""') +tests_after_commit=$(echo "$template" | jq -r '.testsAfterCommit // ""') +echo "template keys: $keys" +echo "template commit=$commit testsAfterCommit=$tests_after_commit" + +# The must-haves. A record missing any of these cannot be checked against the +# repo it describes. +for key in timestamp commit testsAfterCommit publishedTag commitsAheadOfTag; do + echo "$template" | jq -e --arg key "$key" 'has($key)' >/dev/null || + fail "the template is missing must-have '$key'" +done +echo "OK must-haves present: timestamp commit testsAfterCommit publishedTag commitsAheadOfTag" + +# Adjacency is what makes the pair readable as a pair: the before tree and the +# after tree sit together, ahead of everything measured at either. +echo "$template" | jq -e 'keys_unsorted | index("testsAfterCommit") == (index("commit") + 1)' >/dev/null || + fail "testsAfterCommit must come immediately after commit; key order is: $keys" +echo "OK testsAfterCommit is immediately after commit" + +# Full SHAs. A 7-char prefix is a weaker anchor than a full SHA and grows +# ambiguous as history grows — which is the same class of defect the after-tree +# field exists to close, and rain.solmem's record already carries one. +for key in commit testsAfterCommit; do + echo "$template" | jq -e --arg key "$key" '.[$key] | type == "string" and test("^[0-9a-f]{40}$")' >/dev/null || + fail "template '$key' must be a full 40-character lowercase hex SHA" +done +echo "OK commit and testsAfterCommit are full 40-character SHAs" + +# The template exemplifies the field; only the prose states the RULES it obeys, +# and a record is falsifiable because of the rules, not because of the field. A +# check that accepted the name alone would pass a README that had kept +# `testsAfterCommit` in one sentence and dropped every rule attached to it, so +# each rule is pinned by the shortest phrase that carries it. Reword freely — the +# failure names which rule went missing, and re-pinning it is a one-line edit. +readme_prose=$(awk ' + /^## Scan record template$/ { in_section = 1; next } + in_section && /^## / { exit } + in_section && /^```/ { fenced = !fenced; next } + in_section && !fenced { print } +' "$readme") + +[ -n "$readme_prose" ] || fail "README.md has no prose under '## Scan record template' — only the template" + +# Newlines are collapsed first: these documents are reflowed by `deno fmt`, so a +# phrase may be split across lines at any time and that is not a rule going +# missing. +states() { + printf '%s' "$2" | tr '\n' ' ' | tr -s ' ' | grep -qF -- "$1" || + fail "$3 no longer states $4 (looked for '$1')" +} + +readme_states() { states "$1" "$readme_prose" "the README scan record prose" "$2"; } + +readme_states 'testsAfterCommit' "which field names the tree after-state counts hold at" +readme_states '_before_' "that before-numbers hold at the scanned commit" +readme_states '_after_' "that after-numbers hold at testsAfterCommit" +readme_states '40-character' "that both trees are named by full-length SHAs" +readme_states 'equal to' "that a run landing nothing sets testsAfterCommit equal to commit" +readme_states 'never null' "that testsAfterCommit is never null" +readme_states 'never omitted' "that testsAfterCommit is never omitted" +echo "OK README prose states every testsAfterCommit rule" + +# SKILL.md is what a run closing its record actually has in front of it. A field +# documented only in the README is a field campaigns will not write, and a rule +# only the README states is a rule the closing run does not apply. +section=$(awk ' + /^## Committed scan record$/ { in_section = 1; next } + in_section && /^## / { exit } + in_section { print } +' "$skill") + +[ -n "$section" ] || fail "SKILL.md has no '## Committed scan record' section" + +skill_states() { states "$1" "$section" "SKILL.md's '## Committed scan record'" "$2"; } + +skill_states 'testsAfterCommit' "which field names the tree after-state counts hold at" +skill_states 'equal to' "what a run that landed nothing writes" +skill_states 'never null' "that testsAfterCommit is never null and never omitted" +echo "OK SKILL.md's committed scan record states the field and its rules" + +echo "scan record schema OK" diff --git a/.github/workflows/schema-hygiene.yaml b/.github/workflows/schema-hygiene.yaml new file mode 100644 index 0000000..befafcc --- /dev/null +++ b/.github/workflows/schema-hygiene.yaml @@ -0,0 +1,17 @@ +name: schema hygiene +on: [push, pull_request] +permissions: + contents: read +jobs: + # The scan-record schema is documentation, so nothing else in this repo runs + # it. Kept separate from version hygiene so a red here means "the schema + # consumers copy is broken", not "a version pointer drifted". + scan-record-schema: + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + persist-credentials: false + - name: scan record template conforms to the schema it documents + run: sh .github/scripts/check-scan-record-schema.sh diff --git a/README.md b/README.md index 831805c..7c74539 100644 --- a/README.md +++ b/README.md @@ -128,24 +128,44 @@ JSON, no comments: ```json { "timestamp": "2026-08-12T19:40:00Z", - "commit": "08d547fdeadbeef", + "commit": "08d547fdeadbeefc0ffee1122334455667788990", + "testsAfterCommit": "1f9be22cafebabe0ddf00d998877665544332211", "publishedTag": "v1.2.3", "commitsAheadOfTag": 0, "scope": "whole repo", "tool": "adversarial-mutation-test", - "skillVersion": "0.33.0", + "skillVersion": "0.34.0", "summary": { "behaviours": 600, "candidates": 89, "confirmed": 30, + "testsBefore": 41, + "testsAfter": 84, "filed": ["#2651", "#2660"] } } ``` -`timestamp` is UTC at run end; `commit` the exact SHA scanned; `publishedTag` -the release at that commit (null if unreleased) with `commitsAheadOfTag` its -distance. Those three are the must-haves; `summary` is nice-to-have. +`timestamp` is UTC at run end; `commit` the exact SHA scanned; +`testsAfterCommit` the exact SHA the run's own output landed at; `publishedTag` +the release at `commit` (null if unreleased) with `commitsAheadOfTag` its +distance. All five are must-haves; `summary` is nice-to-have. + +A record spans two trees, and every number in it is measured at one of them: +`commit` is the tree the scan ran against, which every _before_ number +(`testsBefore`, baseline counts) holds at; `testsAfterCommit` is the tree with +the run's coverage PRs merged, which every _after_ number (`testsAfter`, and +anything else measured post-landing) holds at. Both are full 40-character SHAs — +a short prefix is a weaker anchor and grows ambiguous as history grows. + +`testsAfterCommit` is never null and never omitted: a run that landed nothing +sets it **equal to `commit`**. "Nothing landed" and "nobody recorded where it +landed" have to stay distinguishable, so an absent field means the record is +malformed rather than that the run was clean. This is the field that makes an +after-state count checkable at all — `rain.sol.codegen` committed +`testsAfter: 102`, a count that occurs at no commit in the range the record +covers, and nothing could catch it because the record named no tree to check it +against. ## License diff --git a/flake.nix b/flake.nix index 6fd363f..414ab2b 100644 --- a/flake.nix +++ b/flake.nix @@ -54,6 +54,9 @@ pkgs.rustc pkgs.clippy pkgs.rustfmt + # `.github/scripts/` runs on it, so a hygiene check is runnable + # locally the same way CI runs it rather than checked by eye. + pkgs.jq ]; }; } diff --git a/skills/adversarial-mutation-test/SKILL.md b/skills/adversarial-mutation-test/SKILL.md index c2460b4..e550b59 100644 --- a/skills/adversarial-mutation-test/SKILL.md +++ b/skills/adversarial-mutation-test/SKILL.md @@ -1,6 +1,6 @@ --- name: adversarial-mutation-test -version: 0.33.0 +version: 0.34.0 description: Use to systematically find BUGS in and harden the test suite for a WHOLE repository (or a whole module of it). Two co-equal goals the name carries: ADVERSARIAL (treat spec/intent as the oracle and the code as suspect — derive expected behavior independently and hunt for inputs where the code is wrong) and MUTATION (prove tests cover the code). Mutation-drives a behavior-centric coverage ledger — for each behavior, break the line and check the whole suite: existing tests that kill the mutant are validated and logged (so existing coverage is audited and in scope), and only surviving mutants (real gaps) get a new discriminating test. An existing test that already kills mutants is left as-is; one meant to cover a behavior but that a mutant survives is strengthened in place (not duplicated); one broken on the unmutated baseline is fixed or its underlying code bug surfaced; a test is never edited to swallow a mutation. Designed for long campaigns that outlive the context window: progress lives in a durable gitignored scratch file so it survives compaction. A single change/PR/function is just a narrowed scope. Triggers on "test the whole repo", "harden the test suite", "mutation test the codebase", "audit the tests", "adversarial tests", "prove these tests cover the code", "exhaust the eventualities". --- @@ -196,10 +196,17 @@ One mutation, one behavior — the failing-test set stays diagnostic. Close every run — including a clean one — by appending an entry to a committed `audit/mutation-test-scans.json` and landing it on the default branch: -timestamp, scanned commit, published tag (+ commits ahead), scope, tool + skill -version, summary with filed issue numbers. The org health check reads the newest -entry for "which release was last audited"; the JSON template lives in this -repo's README. +timestamp, scanned commit, `testsAfterCommit`, published tag (+ commits ahead), +scope, tool + skill version, summary with filed issue numbers. The org health +check reads the newest entry for "which release was last audited"; the JSON +template lives in this repo's README. + +Two trees, both full 40-char SHAs: the scanned `commit` is what every _before_ +number holds at; `testsAfterCommit` — the tree with this run's coverage PRs +merged — is what every _after_ number in `summary` holds at. An after-count with +no tree named is unfalsifiable, so `testsAfterCommit` is a must-have alongside +the scanned commit: a run that landed nothing sets it equal to `commit`, never +null and never omitted. ## Principles