diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 5c916db..5590e75 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -10,7 +10,7 @@ "name": "adversarial-mutation-test", "source": "./", "description": "Find BUGS and harden the test suite for a whole repository — adversarial (spec as oracle, code as suspect; surface candidates for triage) + mutation (break each line, prove a test catches it). Whole-repo, resumable, language-agnostic.", - "version": "0.32.0", + "version": "0.33.0", "author": { "name": "Rain Open Source Software Ltd" }, diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index d1cfa41..e8c7ee3 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "adversarial-mutation-test", "displayName": "Adversarial Mutation Testing", - "version": "0.32.0", + "version": "0.33.0", "description": "Find BUGS and harden the test suite for a whole repository. Two co-equal halves: ADVERSARIAL — treat the spec as the oracle and the code as suspect, hunt for inputs where the code is wrong, and surface candidates for triage (never self-adjudicate); and MUTATION — break each line and prove a test catches it. Whole-repo, resumable, language-agnostic.", "author": { "name": "Rain Open Source Software Ltd", diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml new file mode 100644 index 0000000..7a6f1c5 --- /dev/null +++ b/.github/workflows/rust.yml @@ -0,0 +1,25 @@ +name: rust +on: [push, pull_request] +permissions: + contents: read +jobs: + test: + uses: rainlanguage/rainix/.github/workflows/rainix-rs-test.yaml@main + secrets: inherit + static: + uses: rainlanguage/rainix/.github/workflows/rainix-rs-static.yaml@main + secrets: inherit + # The rainix reusables above build via cargo. Consumers run the flake PACKAGE + # (`nix run …#mutation-probe`), which cargo CI does not exercise — a + # workspace/lockfile drift can leave cargo green while the package fails to + # build. This job guards the path consumers actually take. + nix-build: + name: nix-build + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + persist-credentials: false + - uses: DeterminateSystems/nix-installer-action@a7ad9c4f0c65208097f4d34f3cfa1913b80cce5c # main + - run: nix build .#mutation-probe --print-build-logs diff --git a/.github/workflows/version-hygiene.yaml b/.github/workflows/version-hygiene.yaml index 606d94f..3b59942 100644 --- a/.github/workflows/version-hygiene.yaml +++ b/.github/workflows/version-hygiene.yaml @@ -1,10 +1,8 @@ name: version hygiene on: pull_request: - permissions: contents: read - jobs: version-hygiene: runs-on: ubuntu-latest @@ -13,7 +11,6 @@ jobs: with: # Full history so the PR base is available to diff and read against. fetch-depth: 0 - # The marketplace listing is the only published version pointer (this repo # tags no releases), so it must name the same version as the plugin it # serves. Skew here silently publishes the wrong version to installers — @@ -28,7 +25,6 @@ jobs: exit 1 fi echo "versions agree: $plugin" - # A skill's content IS its release. Editing skills/ without bumping the # version is invisible to version-based update detection ('/plugin' compares # version strings), so every install silently keeps running the stale diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b5ee936 --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +target/ +/result +# Generated into the nix store and symlinked in by git-hooks.nix when you enter +# the rainix dev shell — it hard-codes absolute /nix/store paths from whichever +# machine generated it, so a committed copy is unusable on any other checkout. +# Every rainix consumer ignores it; CI regenerates it inside the shell. +.pre-commit-config.yaml diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..f15c7e0 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,235 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "mutation-probe" +version = "0.1.0" +dependencies = [ + "libc", + "regex", + "serde", + "serde_json", + "toml", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_spanned" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" +dependencies = [ + "serde", +] + +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "toml" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "toml_write", + "winnow", +] + +[[package]] +name = "toml_write" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "winnow" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" +dependencies = [ + "memchr", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..cf7f57f --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,8 @@ +# Workspace root so the rainix rust reusables (cargo test / rainix-rs-static at +# the repo root) pick up the crate under mutation-probe-rs/. +[workspace] +resolver = "2" +members = ["mutation-probe-rs"] + +[profile.release] +opt-level = 2 diff --git a/README.md b/README.md index 4ec528e..831805c 100644 --- a/README.md +++ b/README.md @@ -5,8 +5,8 @@ repository's test suite so every test **provably** covers code. The core idea: every test makes an assertion that differs between correct and buggy code, validated by deliberately breaking the exact line it claims to cover and confirming the test fails. Surviving mutants (lines a mutation can break with no -test noticing) are real coverage gaps and get a new discriminating test; existing -tests that already kill their mutant are credited and logged. +test noticing) are real coverage gaps and get a new discriminating test; +existing tests that already kill their mutant are credited and logged. Language- and harness-agnostic. Designed for long, whole-repo campaigns that outlive the conversation context window: progress lives in a durable gitignored @@ -34,17 +34,17 @@ You can pass a scope as an argument, e.g.: /adversarial-mutation-test:adversarial-mutation-test this PR ``` -The skill also auto-triggers on requests like "harden the test suite", -"mutation test the codebase", "prove these tests cover the code", or "exhaust -the eventualities". +The skill also auto-triggers on requests like "harden the test suite", "mutation +test the codebase", "prove these tests cover the code", or "exhaust the +eventualities". ## What it does - **Surveys** the repo, inventories testable units and existing tests, and finds the gaps (coverage tooling + mutation probing). -- **Groups** the work by the behaviours each group contains — not by module — and - ships each group as its own branch + PR. -- Runs a per-unit loop: enumerate behaviors → baseline green → break *every* +- **Groups** the work by the behaviours each group contains — not by module — + and ships each group as its own branch + PR. +- Runs a per-unit loop: enumerate behaviors → baseline green → break _every_ behavior with one targeted mutation against the **pre-existing** suite, before writing anything → credit by name each existing test that catches one → then work the survivors, which are the worklist, with new or strengthened tests. @@ -59,9 +59,9 @@ the eventualities". 1. Enumerate behaviors (each guard, comparison, computation, side-effect, early-return, error path). 2. Baseline the existing suite green. -3. Probe *every* enumerated behavior with one targeted mutation, against the - **pre-existing** suite and before writing any test of your own — each mutation - made live in whatever the tests actually execute (regenerate any +3. Probe _every_ enumerated behavior with one targeted mutation, against the + **pre-existing** suite and before writing any test of your own — each + mutation made live in whatever the tests actually execute (regenerate any built/cached/generated/etched artifact first — stale artifacts are the #1 way mutation testing lies to you), restoring the source after each probe. 4. A test fails → behavior covered, credited to that named test. No test fails → @@ -72,9 +72,81 @@ the eventualities". clone at the base commit and a second full mutation pass. 6. Record the result. -See [`skills/adversarial-mutation-test/SKILL.md`](skills/adversarial-mutation-test/SKILL.md) +See +[`skills/adversarial-mutation-test/SKILL.md`](skills/adversarial-mutation-test/SKILL.md) for the full method. +## The probe harness (`mutation-probe`) + +The mutate → run → score → restore machinery is a tested Rust bin shipped by +this repo's nix flake — campaigns author mutants declaratively and never +hand-roll the harness (hand-rolls kept faking matrices: zero-match mutants +scored as survived, crashed suites scored at all, imperfect restores poisoning +later probes). + +```sh +nix run github:rainlanguage/adversarial-mutation-test#mutation-probe -- mutants.toml +``` + +`mutation-probe --help` is the complete manual. The short form: the mutants file +names the suite command as argv (artifact regeneration included — the probe runs +exactly that per verdict), a proof-of-run regex reading the suite's own +pass/fail tally, and the mutants as exact-string `(file, target, replacement)` +triples that must match exactly once. + +```toml +[suite] +root = "." +command = ["nix", "develop", "-c", "cargo", "test"] +proof = '(\d+) passed; (\d+) failed' +fail-pattern = 'test (\S+) \.\.\. FAILED' # optional: names the killer +timeout-secs = 1800 # optional + +[[mutants]] +name = "M01 guard inverted" +file = "src/lib.rs" +target = "if !ok {" +replacement = "if ok {" +``` + +Verdicts: `KILLED` (failing tally, or non-zero exit with proof of a run) / +`SURVIVED` (ran green: a real gap) / `NO-RUN` (no proof the suite ran — crash, +compile error, timeout — never scored as survived) / `HARNESS-ERROR` (target not +matched exactly once). A red, silent, or zero-test baseline aborts before any +probe; writes are atomic and every restore is verified byte-exact; a hung +suite's whole process group is killed at `timeout-secs`. Exit 0 only when every +probed mutant is killed; 1 on any non-kill; 2 when the pass cannot be trusted. +`--only ` re-runs a subset while strengthening a killer; +`--json ` writes the machine-readable report. + +## Scan record template + +Campaigns close by appending one entry per run to a committed +`audit/mutation-test-scans.json` on the default branch (see SKILL.md). Valid +JSON, no comments: + +```json +{ + "timestamp": "2026-08-12T19:40:00Z", + "commit": "08d547fdeadbeef", + "publishedTag": "v1.2.3", + "commitsAheadOfTag": 0, + "scope": "whole repo", + "tool": "adversarial-mutation-test", + "skillVersion": "0.33.0", + "summary": { + "behaviours": 600, + "candidates": 89, + "confirmed": 30, + "filed": ["#2651", "#2660"] + } +} +``` + +`timestamp` is UTC at run end; `commit` the exact SHA scanned; `publishedTag` +the release at that commit (null if unreleased) with `commitsAheadOfTag` its +distance. Those three are the must-haves; `summary` is nice-to-have. + ## License [DecentraLicense 1.0](LICENSE) (`LicenseRef-DCL-1.0`). diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000..ef10519 --- /dev/null +++ b/flake.lock @@ -0,0 +1,61 @@ +{ + "nodes": { + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1783187857, + "narHash": "sha256-CoKGv2FwkvFwzsVLB/N89eFjr40puk/p51IMvIp+ddo=", + "owner": "nixos", + "repo": "nixpkgs", + "rev": "19a8a1e6d8b7315b6fd84e5a51977ce6f69d5a5b", + "type": "github" + }, + "original": { + "owner": "nixos", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000..6fd363f --- /dev/null +++ b/flake.nix @@ -0,0 +1,61 @@ +{ + description = "adversarial-mutation-test — the skill, plus mutation-probe, its probe harness as a tested tool."; + + inputs = { + nixpkgs.url = "github:nixos/nixpkgs/nixpkgs-unstable"; + flake-utils.url = "github:numtide/flake-utils"; + }; + + outputs = + { nixpkgs, flake-utils, ... }: + flake-utils.lib.eachDefaultSystem ( + system: + let + pkgs = import nixpkgs { inherit system; }; + inherit (pkgs) lib; + # The crate is a workspace member, so the Cargo.lock lives at the repo + # root. buildRustPackage needs the lock inside src, so src is the + # workspace root — filtered to just the manifests + crate. Without the + # filter, skill/doc churn would rebuild the bin, and a consumer's + # `nix run` would pay a rebuild for a SKILL.md edit; `target/` is + # excluded explicitly because a `path:` ref copies the working + # directory as-is, gitignore not consulted. + src = lib.fileset.toSource { + root = ./.; + fileset = lib.fileset.unions [ + ./Cargo.toml + ./Cargo.lock + ./LICENSE + # Subtract rather than whitelist src/: a whitelist silently drops + # anything the crate gains later (tests/, benches/, build.rs). + (lib.fileset.difference ./mutation-probe-rs (lib.fileset.maybeMissing ./mutation-probe-rs/target)) + ]; + }; + # The probe harness the skill's mutation passes run. Tests run in-build + # via doCheck; invoked directly as `mutation-probe ` — no + # wrapper. Consumers: `nix run github:rainlanguage/adversarial-mutation-test#mutation-probe -- mutants.toml`. + mutation-probe = pkgs.rustPlatform.buildRustPackage { + pname = "mutation-probe"; + # From the manifest, so the two cannot drift. + inherit ((lib.importTOML ./mutation-probe-rs/Cargo.toml).package) version; + meta.mainProgram = "mutation-probe"; + inherit src; + cargoLock.lockFile = ./Cargo.lock; + }; + in + { + packages = { + inherit mutation-probe; + default = mutation-probe; + }; + devShells.default = pkgs.mkShell { + packages = [ + pkgs.cargo + pkgs.rustc + pkgs.clippy + pkgs.rustfmt + ]; + }; + } + ); +} diff --git a/mutation-probe-rs/Cargo.toml b/mutation-probe-rs/Cargo.toml new file mode 100644 index 0000000..c4d6160 --- /dev/null +++ b/mutation-probe-rs/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "mutation-probe" +version = "0.1.0" +edition = "2021" +rust-version = "1.82" +license-file = "../LICENSE" +description = "The skill's probe harness as a tested tool: applies exact-string mutants, proves the suite ran, and scores KILLED/SURVIVED/NO-RUN/HARNESS-ERROR." + +[[bin]] +name = "mutation-probe" +path = "src/main.rs" + +[dependencies] +regex = "1" +serde = { version = "1", features = ["derive"] } +serde_json = "1" +toml = "0.8" + +# Timeout enforcement must kill the suite's whole PROCESS GROUP: killing only the +# direct child (sh) leaves descendants holding the output pipes, and the reader +# joins would block forever. +[target.'cfg(unix)'.dependencies] +libc = "0.2" diff --git a/mutation-probe-rs/src/main.rs b/mutation-probe-rs/src/main.rs new file mode 100644 index 0000000..c69648f --- /dev/null +++ b/mutation-probe-rs/src/main.rs @@ -0,0 +1,851 @@ +// mutation-probe — the skill's probe harness as a tested tool. +// +// The adversarial-mutation-test skill needs, per PR/campaign, a harness that applies +// exact-string mutants and scores the suite's reaction. Hand-rolling that harness each +// time re-risks the same integrity bugs every time: a zero-match "mutation" that mutates +// nothing scoring as "survived", a suite that never ran (crash, E2BIG, wrong dir) scoring +// as anything at all, a red baseline silently probing garbage, a restore that leaves the +// tree mutated. Each of those has faked a matrix in a real incident. This bin owns the +// machinery once, tested; the adversarial half — deriving WHICH mutants would prove the +// suite discriminates — stays with the agent, in the mutants file. +// +// Verdicts: +// KILLED — the suite ran (proof line matched) and failed. +// SURVIVED — the suite ran and passed: a real coverage gap. +// NO-RUN — the suite produced no proof of running (crash / compile error / +// timeout). Unscorable, never "survived". +// HARNESS-ERROR — the mutant itself is invalid (target not found exactly once, or the +// file changed under us). The harness is wrong, not the suite. +// +// Exit codes: 0 = baseline green and every probed mutant KILLED; 1 = the pass ran and +// something was not killed (survivor / no-run / harness-error); 2 = the pass could not +// run or could not be trusted (config unreadable, baseline not green, restore failed). + +use std::collections::BTreeMap; +use std::io::Read; +use std::process::Stdio; +use std::time::{Duration, Instant}; + +use serde::{Deserialize, Serialize}; + +// ---------------------------------------------------------------- config ---- + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct Config { + suite: SuiteConfig, + #[serde(default)] + mutants: Vec, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct SuiteConfig { + /// Target repo root the suite runs in, resolved relative to the mutants file. + root: String, + /// The suite as argv — no shell. Artifact regeneration (build.sh, codegen) must be + /// part of this command: the probe runs exactly one command per verdict, and a suite + /// that tests stale artifacts is the #1 way a mutation matrix lies. + command: Vec, + /// Proof-of-run regex over the suite's combined stdout+stderr. Needs two capture + /// groups: passed count, failed count. Multiple matches sum (cargo prints one result + /// line per test binary). No match anywhere = the suite did not provably run. + proof: String, + /// Optional: one capture group extracting a failing test's name, for `killedBy`. + #[serde(rename = "fail-pattern")] + fail_pattern: Option, + /// Per-run wall clock limit. A hung suite is NO-RUN, not a hung campaign. + #[serde(rename = "timeout-secs", default = "default_timeout")] + timeout_secs: u64, +} + +fn default_timeout() -> u64 { + 1800 +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct MutantConfig { + name: String, + /// File the mutant applies to, relative to `suite.root`. + file: String, + /// Must occur EXACTLY once in the file, or the mutant is a HARNESS-ERROR. + target: String, + replacement: String, +} + +// --------------------------------------------------------------- verdicts ---- + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(tag = "verdict", rename_all = "SCREAMING-KEBAB-CASE")] +enum Verdict { + Killed { + #[serde(skip_serializing_if = "Vec::is_empty")] + killed_by: Vec, + }, + Survived, + NoRun { + detail: String, + }, + HarnessError { + detail: String, + }, +} + +impl Verdict { + fn label(&self) -> &'static str { + match self { + Verdict::Killed { .. } => "KILLED", + Verdict::Survived => "SURVIVED", + Verdict::NoRun { .. } => "NO-RUN", + Verdict::HarnessError { .. } => "HARNESS-ERROR", + } + } +} + +/// What one suite invocation reported, before it means anything for a mutant. +#[derive(Debug, PartialEq, Eq)] +enum SuiteOutcome { + /// Proof line(s) matched: the suite ran and this is its own tally. + Ran { + passed: u64, + failed: u64, + exit_ok: bool, + output: String, + }, + /// No proof anywhere in the output: crash, compile error, wrong dir. + NoProof { output: String }, + /// Wall-clock limit hit; the child was killed. + TimedOut { secs: u64 }, +} + +/// PURE: score one suite run against the proof regex. +/// +/// Summing across matches is what makes one `proof` work for multi-binary harnesses +/// (cargo prints a result line per test binary and per doctest run); proof-of-run is +/// "at least one match", so a partial crash after one binary still counts as ran — +/// its failures are in the tally. +fn classify_suite(output: &str, exit_ok: bool, proof: ®ex::Regex) -> SuiteOutcome { + let mut passed: u64 = 0; + let mut failed: u64 = 0; + let mut matched = false; + for cap in proof.captures_iter(output) { + let p = cap.get(1).and_then(|m| m.as_str().parse::().ok()); + let f = cap.get(2).and_then(|m| m.as_str().parse::().ok()); + if let (Some(p), Some(f)) = (p, f) { + matched = true; + passed += p; + failed += f; + } + } + if !matched { + return SuiteOutcome::NoProof { + output: output.to_string(), + }; + } + SuiteOutcome::Ran { + passed, + failed, + exit_ok, + output: output.to_string(), + } +} + +/// PURE: a mutant's verdict from its suite outcome. +/// +/// KILLED on failed > 0 OR a non-zero exit with proof present: a harness that proves it +/// ran and then exits non-zero is declaring failure even when its tally line predates +/// the failure (deno prints the tally, then exits 1). SURVIVED requires the suite to +/// have both passed its own tally and exited zero. +fn mutant_verdict(outcome: SuiteOutcome, fail_pattern: Option<®ex::Regex>) -> Verdict { + match outcome { + SuiteOutcome::TimedOut { secs } => Verdict::NoRun { + detail: format!("suite timed out after {secs}s"), + }, + SuiteOutcome::NoProof { output } => Verdict::NoRun { + detail: format!( + "no proof-of-run in suite output; tail: {}", + tail(&output, 400) + ), + }, + SuiteOutcome::Ran { + failed, + exit_ok, + output, + .. + } => { + if failed > 0 || !exit_ok { + let killed_by = fail_pattern + .map(|re| { + re.captures_iter(&output) + .filter_map(|c| c.get(1)) + .map(|m| m.as_str().to_string()) + .take(5) + .collect() + }) + .unwrap_or_default(); + Verdict::Killed { killed_by } + } else { + Verdict::Survived + } + } + } +} + +/// PURE: why a baseline run blocks the pass, or None if it is sound. +/// +/// A red baseline probes garbage; a zero-test baseline is the "0 tests ran" incident — +/// every later probe would run testless and report universal survival. Both abort. +fn baseline_defect(outcome: &SuiteOutcome) -> Option { + match outcome { + SuiteOutcome::TimedOut { secs } => Some(format!("baseline suite timed out after {secs}s")), + SuiteOutcome::NoProof { output } => Some(format!( + "baseline produced no proof-of-run; tail: {}", + tail(output, 400) + )), + SuiteOutcome::Ran { + passed, + failed, + exit_ok, + .. + } => { + if *failed > 0 { + Some(format!( + "baseline is RED ({failed} failed) — fix the suite before probing" + )) + } else if !*exit_ok { + Some( + "baseline is RED (green tally but non-zero exit) — fix the suite before probing" + .to_string(), + ) + } else if *passed == 0 { + Some("baseline ran 0 tests — nothing can kill anything".to_string()) + } else { + None + } + } + } +} + +/// PURE: exit code from the pass's verdicts (baseline defects exit earlier, as 2). +fn exit_code(verdicts: &[Verdict]) -> i32 { + if verdicts.iter().all(|v| matches!(v, Verdict::Killed { .. })) { + 0 + } else { + 1 + } +} + +fn tail(s: &str, n: usize) -> String { + let cleaned = s.replace('\n', " "); + let chars: Vec = cleaned.chars().collect(); + if chars.len() <= n { + cleaned + } else { + chars[chars.len() - n..].iter().collect() + } +} + +// ---------------------------------------------------------------- running ---- + +/// Per-stream capture cap. The TAIL is kept: tallies and failure lists live at the +/// end of suite output, and a mutant that makes the suite log in a loop must cost +/// memory O(cap), not O(output). +const CAPTURE_CAP: usize = 4 * 1024 * 1024; + +/// Write via temp-file + rename in the target's own directory: rename is atomic on a +/// same-filesystem move, so the target is always either its old or its new content — +/// never truncated by a failed write. +fn write_atomic(path: &std::path::Path, content: &str) -> Result<(), String> { + let tmp = path.with_extension("mutation-probe.tmp"); + std::fs::write(&tmp, content).map_err(|e| format!("writing {}: {e}", tmp.display()))?; + std::fs::rename(&tmp, path).map_err(|e| format!("renaming over {}: {e}", path.display())) +} + +/// Drain a pipe keeping at most the last `cap` bytes. +fn drain_capped(mut r: impl Read, cap: usize) -> Vec { + let mut buf = Vec::new(); + let mut chunk = [0u8; 65536]; + loop { + match r.read(&mut chunk) { + Ok(0) => break, + Ok(n) => { + buf.extend_from_slice(&chunk[..n]); + if buf.len() > cap.saturating_mul(2) { + buf.drain(..buf.len() - cap); + } + } + Err(_) => break, + } + } + if buf.len() > cap { + buf.drain(..buf.len() - cap); + } + buf +} + +/// Run the suite once: piped output drained on threads (a full pipe would deadlock a +/// chatty suite), wall clock enforced by poll + kill. +/// +/// The suite runs in its OWN PROCESS GROUP, and timeout kills the group: killing only +/// the direct child (`sh`, `nix develop`) leaves descendants holding the output pipes, +/// and the reader joins below would block forever on a suite that hung past its wrapper. +fn run_suite( + cfg: &SuiteConfig, + root: &std::path::Path, + proof: ®ex::Regex, +) -> Result { + let (program, args) = cfg.command.split_first().ok_or("suite.command is empty")?; + let mut command = std::process::Command::new(program); + command + .args(args) + .current_dir(root) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + #[cfg(unix)] + { + use std::os::unix::process::CommandExt; + command.process_group(0); + } + let mut child = command + .spawn() + .map_err(|e| format!("cannot spawn suite {program:?}: {e}"))?; + + let stdout = child.stdout.take().expect("stdout was piped"); + let stderr = child.stderr.take().expect("stderr was piped"); + let out_thread = std::thread::spawn(move || drain_capped(stdout, CAPTURE_CAP)); + let err_thread = std::thread::spawn(move || drain_capped(stderr, CAPTURE_CAP)); + + let deadline = Instant::now() + Duration::from_secs(cfg.timeout_secs); + let status = loop { + match child.try_wait() { + Ok(Some(status)) => break Some(status), + Ok(None) => { + if Instant::now() >= deadline { + #[cfg(unix)] + // SAFETY: plain syscall on the pgid this process created above. + unsafe { + libc::killpg(child.id() as libc::pid_t, libc::SIGKILL); + } + #[cfg(not(unix))] + let _ = child.kill(); + let _ = child.wait(); + break None; + } + std::thread::sleep(Duration::from_millis(100)); + } + Err(e) => return Err(format!("waiting on suite: {e}")), + } + }; + let out = out_thread.join().unwrap_or_default(); + let err = err_thread.join().unwrap_or_default(); + + let Some(status) = status else { + return Ok(SuiteOutcome::TimedOut { + secs: cfg.timeout_secs, + }); + }; + let combined = format!( + "{}\n{}", + String::from_utf8_lossy(&out), + String::from_utf8_lossy(&err) + ); + Ok(classify_suite(&combined, status.success(), proof)) +} + +// ----------------------------------------------------------------- report ---- + +#[derive(Serialize)] +struct Report { + baseline: BaselineReport, + mutants: Vec, + summary: Summary, +} + +#[derive(Serialize)] +struct BaselineReport { + passed: u64, + failed: u64, +} + +#[derive(Serialize)] +struct MutantReport { + name: String, + file: String, + #[serde(flatten)] + verdict: Verdict, +} + +#[derive(Serialize, Default)] +struct Summary { + killed: usize, + survived: usize, + no_run: usize, + harness_error: usize, +} + +// ------------------------------------------------------------------- main ---- + +fn fail(msg: &str) -> ! { + eprintln!("error: {msg}"); + std::process::exit(2); +} + +/// The manual lives here (and in the repo README), NOT in the skill text: skill prose +/// is a recurring per-invocation context cost, while --help is read on demand. +const HELP: &str = r#"mutation-probe — apply exact-string mutants, prove the suite ran, score honestly. + +USAGE + mutation-probe [--json ] [--only ] + + --json also write the machine-readable report + --only probe EVERY mutant whose name contains the substring, so a + short value ("M07") also selects longer names ("M070"). + Deliberately not exact matching: names carry prose, and + over-selection is fail-safe — the extra mutants are probed + and scored, exit 0 still demands all of them KILLED, and a + value matching nothing is an error, never a silent no-op. + --help, -h this manual + +MUTANTS FILE (TOML) + [suite] + root = "." # repo the suite runs in, relative to this file + command = ["sh", "check.sh"] # argv, no shell. Include any artifact regeneration + # here (wrapper script is fine): the probe runs + # exactly this per verdict, and a suite that tests + # stale artifacts is the #1 way a matrix lies. + proof = '(\d+) passed; (\d+) failed' + # 2 capture groups: passed, failed — read from the + # suite's own tally. Multiple matches SUM (cargo + # prints one line per test binary). No match = + # the suite did not provably run. + fail-pattern = 'test (\S+) \.\.\. FAILED' # optional: 1 group naming a killer + timeout-secs = 1800 # optional; the suite's process group is killed + + [[mutants]] + name = "M01 guard inverted" + file = "src/lib.rs" # relative to root + target = "if !ok {" # must occur EXACTLY once in the file + replacement = "if ok {" + +VERDICTS + KILLED suite ran and failed (failing tally, or non-zero exit with proof + present — the tally is trusted over a lying wrapper exit code, + and vice versa) + SURVIVED suite ran green: a real coverage gap + NO-RUN no proof the suite ran (crash / compile error / timeout) — + unscorable, never "survived" + HARNESS-ERROR the mutant is invalid: target not found exactly once + +INTEGRITY (enforced) + A red, silent, or zero-test baseline aborts before any probe. Writes are + atomic (temp + rename): no failure mode leaves a file truncated. Every + restore is verified byte-exact, and each file is re-checked pristine before + the next mutant. Suite output is capped per stream (oldest bytes dropped). + +EXIT CODES + 0 baseline green and every probed mutant KILLED + 1 the pass ran; something SURVIVED, was NO-RUN, or was a HARNESS-ERROR + 2 the pass could not run or be trusted (config error, red baseline, + restore failure) +"#; + +fn main() { + let mut args = std::env::args().skip(1); + let mut config_path: Option = None; + let mut json_path: Option = None; + let mut only: Option = None; + while let Some(a) = args.next() { + match a.as_str() { + "--help" | "-h" => { + print!("{HELP}"); + std::process::exit(0); + } + "--json" => { + json_path = Some(args.next().unwrap_or_else(|| fail("--json needs a path"))) + } + "--only" => { + only = Some( + args.next() + .unwrap_or_else(|| fail("--only needs a substring")), + ) + } + // A misspelled flag must not silently become the config path. + other if other.starts_with('-') => fail(&format!("unknown flag {other:?} (--help)")), + _ if config_path.is_none() => config_path = Some(a), + other => fail(&format!("unexpected argument {other:?}")), + } + } + let config_path = config_path.unwrap_or_else(|| { + fail("usage: mutation-probe [--json ] [--only ] (--help for the manual)") + }); + + let raw = std::fs::read_to_string(&config_path) + .unwrap_or_else(|e| fail(&format!("cannot read {config_path}: {e}"))); + let cfg: Config = toml::from_str(&raw).unwrap_or_else(|e| fail(&format!("{config_path}: {e}"))); + + // Validate regexes at load, loudly: a proof with fewer than two capture groups can + // never prove a run, which would score every mutant NO-RUN and look like a broken + // suite instead of a broken config. + let proof = regex::Regex::new(&cfg.suite.proof) + .unwrap_or_else(|e| fail(&format!("suite.proof is not a valid regex: {e}"))); + if proof.captures_len() < 3 { + fail("suite.proof needs two capture groups: (passed) and (failed)"); + } + let fail_pattern = cfg.suite.fail_pattern.as_deref().map(|p| { + let re = regex::Regex::new(p) + .unwrap_or_else(|e| fail(&format!("suite.fail-pattern is not a valid regex: {e}"))); + if re.captures_len() < 2 { + fail("suite.fail-pattern needs one capture group: the failing test's name"); + } + re + }); + + let config_dir = std::path::Path::new(&config_path) + .parent() + .map(|p| p.to_path_buf()) + .unwrap_or_default(); + let root = config_dir.join(&cfg.suite.root); + if !root.is_dir() { + fail(&format!("suite.root {} is not a directory", root.display())); + } + + let selected: Vec<&MutantConfig> = cfg + .mutants + .iter() + .filter(|m| only.as_deref().is_none_or(|o| m.name.contains(o))) + .collect(); + if selected.is_empty() { + fail(match only { + Some(_) => "--only matched no mutant", + None => "no [[mutants]] in the config", + }); + } + + // Original bytes of every file the pass touches, read once up front. Also the + // restore oracle: after every probe the file must byte-match this. + let mut originals: BTreeMap<&str, String> = BTreeMap::new(); + for m in &selected { + if !originals.contains_key(m.file.as_str()) { + let content = std::fs::read_to_string(root.join(&m.file)) + .unwrap_or_else(|e| fail(&format!("cannot read {}: {e}", m.file))); + originals.insert(m.file.as_str(), content); + } + } + + // Baseline: the suite must prove itself green on the unmutated tree first. + println!("baseline: running suite ..."); + let baseline = run_suite(&cfg.suite, &root, &proof).unwrap_or_else(|e| fail(&e)); + if let Some(defect) = baseline_defect(&baseline) { + fail(&defect); + } + let (base_passed, base_failed) = match &baseline { + SuiteOutcome::Ran { passed, failed, .. } => (*passed, *failed), + _ => unreachable!("baseline_defect rejects non-Ran outcomes"), + }; + println!("baseline: green ({base_passed} passed)"); + + let mut reports: Vec = Vec::new(); + for m in &selected { + let original = &originals[m.file.as_str()]; + let path = root.join(&m.file); + let verdict = 'v: { + // The tree must still be pristine — a suite that mutates the tree, or a + // failed earlier restore, invalidates every occurrence count taken at load. + let on_disk = std::fs::read_to_string(&path) + .unwrap_or_else(|e| fail(&format!("cannot re-read {}: {e}", m.file))); + if on_disk != *original { + fail(&format!( + "{} changed on disk mid-pass — tree is not pristine, aborting", + m.file + )); + } + let occurrences = original.matches(m.target.as_str()).count(); + if occurrences != 1 { + break 'v Verdict::HarnessError { + detail: format!( + "target occurs {occurrences}x (need exactly 1) — mutates nothing" + ), + }; + } + let mutated = original.replacen(m.target.as_str(), &m.replacement, 1); + // Atomic (temp + rename) in both directions: a plain fs::write truncates + // first, so a failure mid-write (ENOSPC, EIO) would leave the file + // truncated with nothing to restore from on disk. + write_atomic(&path, &mutated) + .unwrap_or_else(|e| fail(&format!("cannot write mutant to {}: {e}", m.file))); + let outcome = run_suite(&cfg.suite, &root, &proof); + // Restore before anything can early-return, then verify byte-exact: a tree + // left mutated poisons every later probe and the working copy itself. + write_atomic(&path, original).unwrap_or_else(|e| { + fail(&format!( + "RESTORE FAILED for {}: {e} — tree is dirty", + m.file + )) + }); + let restored = std::fs::read_to_string(&path) + .unwrap_or_else(|e| fail(&format!("cannot verify restore of {}: {e}", m.file))); + if restored != *original { + fail(&format!( + "restore of {} is not byte-exact — tree is dirty", + m.file + )); + } + let outcome = outcome.unwrap_or_else(|e| fail(&e)); + mutant_verdict(outcome, fail_pattern.as_ref()) + }; + let line = match &verdict { + Verdict::Killed { killed_by } if !killed_by.is_empty() => { + format!("{} — killed by: {}", verdict.label(), killed_by.join(", ")) + } + Verdict::NoRun { detail } | Verdict::HarnessError { detail } => { + format!("{} — {}", verdict.label(), detail) + } + _ => verdict.label().to_string(), + }; + println!("{}: {line}", m.name); + reports.push(MutantReport { + name: m.name.clone(), + file: m.file.clone(), + verdict, + }); + } + + let mut summary = Summary::default(); + for r in &reports { + match r.verdict { + Verdict::Killed { .. } => summary.killed += 1, + Verdict::Survived => summary.survived += 1, + Verdict::NoRun { .. } => summary.no_run += 1, + Verdict::HarnessError { .. } => summary.harness_error += 1, + } + } + println!( + "\n== {}/{} killed; survived: {}; no-run: {}; harness errors: {}", + summary.killed, + reports.len(), + summary.survived, + summary.no_run, + summary.harness_error + ); + + let verdicts: Vec = reports.iter().map(|r| r.verdict.clone()).collect(); + let report = Report { + baseline: BaselineReport { + passed: base_passed, + failed: base_failed, + }, + mutants: reports, + summary, + }; + if let Some(p) = json_path { + let json = serde_json::to_string_pretty(&report).expect("report serializes"); + std::fs::write(&p, json).unwrap_or_else(|e| fail(&format!("cannot write {p}: {e}"))); + } + std::process::exit(exit_code(&verdicts)); +} + +// ------------------------------------------------------------------ tests ---- + +#[cfg(test)] +mod tests { + use super::*; + + fn proof() -> regex::Regex { + regex::Regex::new(r"(\d+) passed \| (\d+) failed").unwrap() + } + + #[test] + fn no_proof_line_is_never_scorable() { + let out = classify_suite("panicked before any tests ran", true, &proof()); + assert!(matches!(out, SuiteOutcome::NoProof { .. })); + // ...and becomes NO-RUN, not SURVIVED, whatever the exit code claimed. + assert!(matches!(mutant_verdict(out, None), Verdict::NoRun { .. })); + } + + #[test] + fn proof_matches_sum_across_binaries() { + let out = classify_suite( + "test result: 3 passed | 0 failed\nlater: 4 passed | 2 failed", + false, + &proof(), + ); + match out { + SuiteOutcome::Ran { passed, failed, .. } => { + assert_eq!(passed, 7); + assert_eq!(failed, 2); + } + other => panic!("expected Ran, got {other:?}"), + } + } + + #[test] + fn failed_tests_kill() { + let out = classify_suite("5 passed | 1 failed", false, &proof()); + assert!(matches!(mutant_verdict(out, None), Verdict::Killed { .. })); + } + + #[test] + fn failing_tally_kills_even_when_the_exit_code_lies() { + // A wrapper that swallows the suite's exit code must not launder a failure: + // the tally is the suite's own word, and it says something failed. + let out = classify_suite("5 passed | 1 failed", true, &proof()); + assert!(matches!(mutant_verdict(out, None), Verdict::Killed { .. })); + } + + #[test] + fn nonzero_exit_with_proof_kills_even_at_zero_failed_tally() { + // deno prints its tally and then exits 1 on a failing permission/sanitizer step; + // the suite declared failure, so the mutant is caught. + let out = classify_suite("5 passed | 0 failed", false, &proof()); + assert!(matches!(mutant_verdict(out, None), Verdict::Killed { .. })); + } + + #[test] + fn clean_pass_survives() { + let out = classify_suite("5 passed | 0 failed", true, &proof()); + assert_eq!(mutant_verdict(out, None), Verdict::Survived); + } + + #[test] + fn timeout_is_no_run() { + assert!(matches!( + mutant_verdict(SuiteOutcome::TimedOut { secs: 9 }, None), + Verdict::NoRun { .. } + )); + } + + #[test] + fn killed_by_extracts_failing_test_names() { + let fp = regex::Regex::new(r"(?m)^(\S+) \.\.\. FAILED$").unwrap(); + let out = classify_suite( + "guard_test ... FAILED\n1 passed | 1 failed", + false, + &proof(), + ); + match mutant_verdict(out, Some(&fp)) { + Verdict::Killed { killed_by } => assert_eq!(killed_by, vec!["guard_test"]), + other => panic!("expected Killed, got {other:?}"), + } + } + + #[test] + fn baseline_red_and_empty_and_silent_all_block() { + let red = classify_suite("3 passed | 1 failed", false, &proof()); + assert!(baseline_defect(&red).unwrap().contains("RED")); + let empty = classify_suite("0 passed | 0 failed", true, &proof()); + assert!(baseline_defect(&empty).unwrap().contains("0 tests")); + let silent = classify_suite("", true, &proof()); + assert!(baseline_defect(&silent).unwrap().contains("no proof")); + let nonzero_exit = classify_suite("3 passed | 0 failed", false, &proof()); + assert!(baseline_defect(&nonzero_exit).is_some()); + let green = classify_suite("3 passed | 0 failed", true, &proof()); + assert!(baseline_defect(&green).is_none()); + } + + #[test] + fn exit_zero_only_when_everything_killed() { + let killed = Verdict::Killed { killed_by: vec![] }; + assert_eq!(exit_code(&[killed.clone(), killed.clone()]), 0); + assert_eq!(exit_code(&[killed.clone(), Verdict::Survived]), 1); + assert_eq!( + exit_code(&[ + killed.clone(), + Verdict::NoRun { + detail: String::new() + } + ]), + 1 + ); + assert_eq!( + exit_code(&[ + killed, + Verdict::HarnessError { + detail: String::new() + } + ]), + 1 + ); + } + + #[test] + fn config_parses_with_defaults_and_rejects_unknown_keys() { + let cfg: Config = toml::from_str( + r#" + [suite] + root = "." + command = ["sh", "check.sh"] + proof = '(\d+) passed \| (\d+) failed' + + [[mutants]] + name = "M01" + file = "code.txt" + target = "a" + replacement = "b" + "#, + ) + .unwrap(); + assert_eq!(cfg.suite.timeout_secs, 1800); + assert_eq!(cfg.mutants.len(), 1); + + let unknown: Result = toml::from_str( + r#" + [suite] + root = "." + command = ["sh"] + proof = "x" + typo-field = 1 + "#, + ); + assert!( + unknown.is_err(), + "an unknown key must be a loud config error" + ); + } + + #[test] + fn red_baseline_names_the_actual_defect() { + let tally_red = classify_suite("3 passed | 1 failed", false, &proof()); + assert!(baseline_defect(&tally_red).unwrap().contains("1 failed")); + let exit_red = classify_suite("3 passed | 0 failed", false, &proof()); + assert!(baseline_defect(&exit_red) + .unwrap() + .contains("green tally but non-zero exit")); + } + + #[test] + fn capped_drain_keeps_the_tail() { + let data: Vec = (0..100_000u32).flat_map(|i| i.to_le_bytes()).collect(); + let out = drain_capped(std::io::Cursor::new(data.clone()), 1024); + assert_eq!(out.len(), 1024); + assert_eq!( + out[..], + data[data.len() - 1024..], + "the TAIL survives the cap" + ); + let small = drain_capped(std::io::Cursor::new(b"abc".to_vec()), 1024); + assert_eq!(small, b"abc"); + } + + #[test] + fn atomic_write_replaces_content_and_leaves_no_temp() { + let dir = std::env::temp_dir().join(format!("mp-atomic-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let target = dir.join("f.txt"); + std::fs::write(&target, "old").unwrap(); + write_atomic(&target, "new").unwrap(); + assert_eq!(std::fs::read_to_string(&target).unwrap(), "new"); + assert!( + !target.with_extension("mutation-probe.tmp").exists(), + "the temp file is consumed by the rename" + ); + let _ = std::fs::remove_dir_all(&dir); + } + + #[test] + fn tail_keeps_the_end_and_flattens_newlines() { + assert_eq!(tail("abc\ndef", 4), " def"); + assert_eq!(tail("ab", 4), "ab"); + } +} diff --git a/mutation-probe-rs/tests/toy.rs b/mutation-probe-rs/tests/toy.rs new file mode 100644 index 0000000..8be525f --- /dev/null +++ b/mutation-probe-rs/tests/toy.rs @@ -0,0 +1,324 @@ +// End-to-end probe runs against a toy repo with a deliberately weak suite: one behavior +// covered (killed), one uncovered (survived), one mutant that crashes the suite before +// its summary (no-run), one whose target does not exist (harness-error). The toy suite +// is plain `sh`, so the whole matrix runs hermetically inside `cargo test`. + +use std::path::{Path, PathBuf}; +use std::process::Command; + +/// The suite: crashes pre-summary on BOOM, checks GUARD, never checks CAP. +const CHECK_SH: &str = r#" +p=0; f=0 +if grep -q "BOOM" code.txt; then exit 7; fi +if grep -q "GUARD on" code.txt; then echo "guard_test ... ok"; p=$((p+1)); else echo "guard_test ... FAILED"; f=$((f+1)); fi +echo "cap_test ... ok"; p=$((p+1)) +echo "$p passed | $f failed" +[ "$f" -eq 0 ] || exit 1 +"#; + +fn toy(dir: &Path, code: &str) { + std::fs::create_dir_all(dir).unwrap(); + std::fs::write(dir.join("code.txt"), code).unwrap(); + std::fs::write(dir.join("check.sh"), CHECK_SH).unwrap(); +} + +fn unique_dir(tag: &str) -> PathBuf { + std::env::temp_dir().join(format!("mutation-probe-toy-{tag}-{}", std::process::id())) +} + +fn write_config(dir: &Path, mutants_toml: &str) -> PathBuf { + let config = format!( + r#" +[suite] +root = "." +command = ["sh", "check.sh"] +proof = '(\d+) passed \| (\d+) failed' +fail-pattern = '(\S+) \.\.\. FAILED' +timeout-secs = 60 +{mutants_toml} +"# + ); + let path = dir.join("mutants.toml"); + std::fs::write(&path, config).unwrap(); + path +} + +fn run(config: &Path, extra: &[&str]) -> (i32, String, serde_json::Value) { + let json_path = config.with_extension("report.json"); + let output = Command::new(env!("CARGO_BIN_EXE_mutation-probe")) + .arg(config) + .arg("--json") + .arg(&json_path) + .args(extra) + .output() + .unwrap(); + let report = std::fs::read_to_string(&json_path) + .ok() + .and_then(|s| serde_json::from_str(&s).ok()) + .unwrap_or(serde_json::Value::Null); + ( + output.status.code().unwrap_or(-1), + format!( + "{}\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ), + report, + ) +} + +const ALL_FOUR: &str = r#" +[[mutants]] +name = "M-kill guard off" +file = "code.txt" +target = "GUARD on" +replacement = "GUARD off" + +[[mutants]] +name = "M-survive cap unchecked" +file = "code.txt" +target = "CAP 10" +replacement = "CAP 99" + +[[mutants]] +name = "M-norun crash the suite" +file = "code.txt" +target = "MODE strict" +replacement = "BOOM" + +[[mutants]] +name = "M-zero no such target" +file = "code.txt" +target = "ABSENT TEXT" +replacement = "whatever" +"#; + +#[test] +fn weak_suite_scores_all_four_verdicts_and_restores() { + let dir = unique_dir("four"); + let code = "GUARD on\nCAP 10\nMODE strict\n"; + toy(&dir, code); + let config = write_config(&dir, ALL_FOUR); + + let (exit, out, report) = run(&config, &[]); + assert_eq!(exit, 1, "survivors must exit 1; output:\n{out}"); + + let verdicts: Vec<(&str, &str)> = report["mutants"] + .as_array() + .unwrap() + .iter() + .map(|m| (m["name"].as_str().unwrap(), m["verdict"].as_str().unwrap())) + .collect(); + assert_eq!( + verdicts, + vec![ + ("M-kill guard off", "KILLED"), + ("M-survive cap unchecked", "SURVIVED"), + ("M-norun crash the suite", "NO-RUN"), + ("M-zero no such target", "HARNESS-ERROR"), + ] + ); + assert_eq!( + report["mutants"][0]["killed_by"][0].as_str(), + Some("guard_test"), + "fail-pattern names the killer" + ); + assert_eq!(report["baseline"]["passed"].as_u64(), Some(2)); + assert_eq!( + std::fs::read_to_string(dir.join("code.txt")).unwrap(), + code, + "the tree must be restored byte-exact after the pass" + ); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn all_killed_exits_zero() { + let dir = unique_dir("killed"); + toy(&dir, "GUARD on\nCAP 10\nMODE strict\n"); + let config = write_config( + &dir, + r#" +[[mutants]] +name = "M-kill guard off" +file = "code.txt" +target = "GUARD on" +replacement = "GUARD off" +"#, + ); + let (exit, out, report) = run(&config, &[]); + assert_eq!(exit, 0, "an all-killed pass exits 0; output:\n{out}"); + assert_eq!(report["summary"]["killed"].as_u64(), Some(1)); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn red_baseline_aborts_without_probing() { + let dir = unique_dir("red"); + toy(&dir, "GUARD off\nCAP 10\nMODE strict\n"); + let config = write_config(&dir, ALL_FOUR); + let (exit, out, report) = run(&config, &[]); + assert_eq!(exit, 2, "a red baseline is an abort; output:\n{out}"); + assert!( + out.contains("RED"), + "the abort names the red baseline:\n{out}" + ); + assert_eq!( + report, + serde_json::Value::Null, + "no report on an aborted pass" + ); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn crashed_baseline_aborts_as_no_proof() { + let dir = unique_dir("crash"); + toy(&dir, "GUARD on\nCAP 10\nBOOM\n"); + let config = write_config(&dir, ALL_FOUR); + let (exit, out, _) = run(&config, &[]); + assert_eq!(exit, 2); + assert!( + out.contains("no proof-of-run"), + "a baseline that cannot prove it ran must say so:\n{out}" + ); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn hung_suite_times_out_as_no_run_and_restores() { + // The mutant makes the suite sleep far past timeout-secs; sh's CHILD (sleep) + // holds the output pipes, so this also proves the process-group kill — with a + // child-only kill the probe would hang on the pipe readers, not finish. + let dir = unique_dir("timeout"); + let code = "GUARD on\nCAP 10\nMODE strict\n"; + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(dir.join("code.txt"), code).unwrap(); + std::fs::write( + dir.join("check.sh"), + "if grep -q SLOW code.txt; then sleep 600; fi\n".to_string() + CHECK_SH, + ) + .unwrap(); + let config = r#" +[suite] +root = "." +command = ["sh", "check.sh"] +proof = '(\d+) passed \| (\d+) failed' +timeout-secs = 2 + +[[mutants]] +name = "M-hang the suite sleeps forever" +file = "code.txt" +target = "MODE strict" +replacement = "SLOW" +"#; + let config_path = dir.join("mutants.toml"); + std::fs::write(&config_path, config).unwrap(); + let started = std::time::Instant::now(); + let (exit, out, report) = run(&config_path, &[]); + assert!( + started.elapsed() < std::time::Duration::from_secs(60), + "the probe must not hang on the hung suite's pipes" + ); + assert_eq!(exit, 1, "a NO-RUN is a non-kill; output:\n{out}"); + assert_eq!(report["mutants"][0]["verdict"].as_str(), Some("NO-RUN")); + assert!( + report["mutants"][0]["detail"] + .as_str() + .unwrap() + .contains("timed out"), + "the detail names the timeout" + ); + assert_eq!( + std::fs::read_to_string(dir.join("code.txt")).unwrap(), + code, + "restored despite the timeout" + ); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn failing_tally_with_zero_exit_still_kills() { + // A wrapper that swallows the suite's exit code must not launder a failure the + // suite's own tally reports — end-to-end twin of the unit test. + let dir = unique_dir("liar"); + let code = "GUARD on\nCAP 10\nMODE strict\n"; + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(dir.join("code.txt"), code).unwrap(); + // Same suite, but the final failure-propagating line is replaced with exit 0. + let swallowing = CHECK_SH.replace("[ \"$f\" -eq 0 ] || exit 1", "exit 0"); + assert_ne!( + swallowing, CHECK_SH, + "the exit-propagation line must exist to be swallowed" + ); + std::fs::write(dir.join("check.sh"), swallowing).unwrap(); + let config = write_config( + &dir, + r#" +[[mutants]] +name = "M-kill guard off" +file = "code.txt" +target = "GUARD on" +replacement = "GUARD off" +"#, + ); + let (exit, out, report) = run(&config, &[]); + assert_eq!(exit, 0, "killed via the tally alone; output:\n{out}"); + assert_eq!(report["mutants"][0]["verdict"].as_str(), Some("KILLED")); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn only_filter_narrows_the_pass() { + let dir = unique_dir("only"); + let code = "GUARD on\nCAP 10\nMODE strict\n"; + toy(&dir, code); + let config = write_config(&dir, ALL_FOUR); + let (exit, out, report) = run(&config, &["--only", "M-kill"]); + assert_eq!(exit, 0, "the killed mutant alone exits 0; output:\n{out}"); + assert_eq!(report["mutants"].as_array().unwrap().len(), 1); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn only_is_a_substring_filter_so_a_prefix_selects_every_match() { + // `--only` matches by SUBSTRING, deliberately: names carry prose, and typing a + // whole one exactly is hostile, while over-selection is fail-safe — a wider + // match only adds verdicts, and exit 0 still demands that every one of them be + // KILLED. This pins both halves against a silent switch to exact matching (which + // would select nothing here and abort) or to first-match-wins (which would + // report one mutant and exit 0, laundering the survivor out of the pass). + let dir = unique_dir("substring"); + let code = "GUARD on\nCAP 10\nMODE strict\n"; + toy(&dir, code); + let config = write_config( + &dir, + r#" +[[mutants]] +name = "M07 guard off" +file = "code.txt" +target = "GUARD on" +replacement = "GUARD off" + +[[mutants]] +name = "M070 cap unchecked" +file = "code.txt" +target = "CAP 10" +replacement = "CAP 99" +"#, + ); + let (exit, out, report) = run(&config, &["--only", "M07"]); + let mutants = report["mutants"].as_array().unwrap(); + assert_eq!( + mutants.len(), + 2, + "M07 is a substring of both names, so both are probed; output:\n{out}" + ); + assert_eq!(mutants[0]["verdict"].as_str(), Some("KILLED")); + assert_eq!(mutants[1]["verdict"].as_str(), Some("SURVIVED")); + assert_eq!( + exit, 1, + "the extra match's survival still fails the pass; output:\n{out}" + ); + let _ = std::fs::remove_dir_all(&dir); +} diff --git a/skills/adversarial-mutation-test/SKILL.md b/skills/adversarial-mutation-test/SKILL.md index 1a2dfc9..c2460b4 100644 --- a/skills/adversarial-mutation-test/SKILL.md +++ b/skills/adversarial-mutation-test/SKILL.md @@ -1,213 +1,220 @@ --- name: adversarial-mutation-test +version: 0.33.0 description: Use to systematically find BUGS in and harden the test suite for a WHOLE repository (or a whole module of it). Two co-equal goals the name carries: ADVERSARIAL (treat spec/intent as the oracle and the code as suspect — derive expected behavior independently and hunt for inputs where the code is wrong) and MUTATION (prove tests cover the code). Mutation-drives a behavior-centric coverage ledger — for each behavior, break the line and check the whole suite: existing tests that kill the mutant are validated and logged (so existing coverage is audited and in scope), and only surviving mutants (real gaps) get a new discriminating test. An existing test that already kills mutants is left as-is; one meant to cover a behavior but that a mutant survives is strengthened in place (not duplicated); one broken on the unmutated baseline is fixed or its underlying code bug surfaced; a test is never edited to swallow a mutation. Designed for long campaigns that outlive the context window: progress lives in a durable gitignored scratch file so it survives compaction. A single change/PR/function is just a narrowed scope. Triggers on "test the whole repo", "harden the test suite", "mutation test the codebase", "audit the tests", "adversarial tests", "prove these tests cover the code", "exhaust the eventualities". -version: 0.32.0 --- # Adversarial Mutation Testing (whole-repo, resumable) -This has TWO co-equal goals, and the name carries both: **adversarial** = hunt for actual BUGS (places the code does the wrong thing), and **mutation** = prove the tests cover the code. Default scope is the **whole repo** (or a whole module of it); a single change / PR / function is just a narrowed target. Any language / harness. - -- **Mutation** treats the current code as the oracle: break a line, check that a test notices → finds **test gaps**. Every test makes an assertion that differs between correct and buggy code, validated by breaking the exact line it claims to cover and confirming the test fails. -- **Adversarial** treats the *spec/intent* as the oracle and the **code as suspect**: independently derive what the code *should* do, then actively try to make it do something wrong → finds **bugs**. - -Doing only the mutation half is the common failure: it pins whatever the code currently does and structurally CANNOT find a bug, then rationalizes "no bugs found" as fine. A run that adds tests but never tries to break the code has done half the job. Every unit gets BOTH passes (see the adversarial pass below). - -A whole-repo run is long and **will outlive the conversation's context window**. Treat conversation memory as unreliable; committed git history (tests, filed issues) is the durable record of the work itself. - -**Run this in ultracode — native Workflow orchestration.** This skill is written for ultracode: the whole-repo campaign is a fan-out driven by the native Workflow tool — `agent()` / `parallel()` / `pipeline()`, schema-forced structured returns, the run journal (`resumeFromRunId`), and `budget`. Let those primitives own concurrency, resume, and convergence — and do NOT hand-roll them — while each worker takes its own **fresh clone** for isolation (see Parallelizing; worktrees are wrong for this skill). The **orchestrator** (the agent authoring the Workflow) owns the survey-slice, the loop-until-dry convergence, and the final synthesis; subagents do the per-unit work and return validated findings. A narrowed scope (one change / PR / function) can run inline and serial without a Workflow; everywhere below, "orchestrator" means the Workflow script when fanned out, and the serial driver when not. - -## 0. Durable progress — two roles, kept separate - -A long run has two distinct durability needs; don't collapse them into one mechanism: - -- **Run-time resume + convergence (fan-out) is owned NATIVELY.** Re-invoke the Workflow with `resumeFromRunId` and the unchanged prefix of `agent()` calls replays from cache — completed units/batches are never re-run. The orchestrator's own loop state (the sliced worklist, round counter, seen-set) is the convergence signal. Do NOT recover run state by having agents read/write a shared progress file — that is the agents-edit-shared-state bug (see Parallelizing). -- **Human/audit trail + serial-mode resume is a gitignored scratch file** `.mutation-test/PROGRESS.md` at repo root. It is the human-readable narrative and, in the non-Workflow serial mode (a narrowed scope run inline), the resume source. It is **never** the run-time convergence signal for a fan-out. - -Set up the scratch file first: -- Make git ignore it **without a tracked change**: append `.mutation-test/` to `.git/info/exclude` (preferred — no repo diff). Use `.gitignore` only if the team wants it shared. -- It holds: **scope**; the **harness commands** (build / test / regenerate-artifacts / coverage); the full **worklist** with per-unit status; and the per-unit **mutation matrices**. - -Protocol: -- **Serial mode (no Workflow):** on re-entry after compaction, READ PROGRESS.md to recover; **mark exactly one unit `IN PROGRESS`** with a one-line note of the precise sub-step so a mid-unit interruption resumes exactly; after each unit update it (`DONE`, matrix, commit hash). Keep it small and skimmable. -- **Fan-out mode:** the orchestrator writes PROGRESS.md (and the committed audit record) ONCE, from the aggregated structured returns of its agents — subagents never edit it; resume is `resumeFromRunId`, not parsing the file. **So every "record … in PROGRESS.md" / "resume there next iteration" instruction in the steps below is the *serial-mode* action**; under a Workflow the executing agent instead RETURNS that (its matrix, bug candidates, branch/PR, unprobed list) as its schema-validated result, and the orchestrator records it once. - -Suggested format: -``` -# Adversarial Mutation Test — PROGRESS -Scope: -Harness: build= | test= | regen= | coverage= - -## Groups & worklist (one branch + PR per group) -### math-libs — branch — PR #123 (merged) -- [DONE] LibFoo — validated; 2 existing confirmed, 1 gap filled -- [DONE] LibBar — validated; 1 gap filled -### arb — branch — PR not opened yet -- [WIP] GenericPoolArb.exchange — probing branch 2 -- [TODO] RouteProcessorArb -... - -## Coverage ledger (unit.behavior -> mutation -> killer; SURVIVED = gap) -LibFoo.guard: negate -> existing:testGuardRejects ✓ (existing test validated) -LibFoo.cap: off-by-one -> SURVIVED -> added:testCapBoundary ✓ -LibFoo.emit: drop emit -> SURVIVED -> GAP (todo) -``` - -The ledger is **behavior-centric**: each behavior gets a row recording the mutation and the test that killed it — *existing or new*. So existing tests enter scope and the tracker by being **mutation-validated** (they earn a row only by actually killing a mutant), and weak existing tests are exposed: a line `coverage` calls "covered" whose mutation **survives** had only incidental coverage and is still a gap. - -**Coverage is a separate axis from correctness.** A killed mutant proves the behavior is PINNED (a test and the code agree), never that it is CORRECT — the test may simply mirror the implementation and enshrine a bug. This ledger answers "is it tested?"; the adversarial pass answers "is it right?", and it judges EVERY behavior including the ones marked covered here. Never read a `covered ✓` row as validation. +Two co-equal goals, and the name carries both: **adversarial** = spec/intent is +the oracle and the code is suspect — hunt for actual BUGS; **mutation** = code +is the oracle — break a line, prove a test notices. Mutation alone is the common +failure: it pins whatever the code does and structurally CANNOT find a bug. +Every unit gets BOTH passes. Default scope is the whole repo; a single change / +PR / function is just a narrowed target. + +Whole-repo runs are ultracode-shaped: fan out with the native Workflow tool +(`agent()` / `parallel()` / `pipeline()`, schema-forced returns, +`resumeFromRunId`, `budget`); a narrowed scope runs inline and serial. +"Orchestrator" below means the Workflow author when fanned out, the serial +driver when not. + +## Durable progress — two roles, kept separate + +- **Fan-out resume + convergence is owned natively**: re-invoke with + `resumeFromRunId`; the orchestrator's own loop state (sliced worklist, + seen-set) is the convergence signal. Never recover run state from agents + editing a shared file. +- **Human/audit trail + serial-mode resume** is a gitignored + `.mutation-test/PROGRESS.md` (add to `.git/info/exclude`): scope, harness + commands, worklist with per-unit status, per-unit matrices. Serial mode marks + exactly one unit `IN PROGRESS` and updates after each; fan-out mode has the + orchestrator write it once from aggregated structured returns. + +The coverage ledger is **behavior-centric**: one row per behavior — the mutation +and the test that killed it, existing or new — so existing tests earn credit +only by actually killing a mutant, and "covered" lines whose mutants survive are +exposed as gaps. A killed mutant proves the behavior is PINNED, never CORRECT: a +test can mirror the implementation and enshrine its bug, which is why the +adversarial pass judges covered behaviors too. ## Repo-wide campaign -1. **Survey & inventory.** Enumerate testable units (modules / contracts / files / public functions) and existing tests. Find the gaps with coverage tooling — `forge coverage`, `cargo llvm-cov`, `pytest --cov`, `go test -cover`, `nyc`, etc. Zero-coverage and weakly-covered units are highest value. Write the worklist into PROGRESS.md. -2. **Prioritize, chunk, and group.** Order by risk × coverage gap (security-critical / complex / untested first). Process **one unit at a time**, but organize the worklist into groups that **each ship as their own branch + PR** — this keeps PRs small and reviewable and ships coverage incrementally instead of one giant branch. **A group is sized by the behaviours it contains, NOT by "a module / package / coverage area".** - - **Size by enumerated behaviours; a file is not indivisible.** A unit count carries no size signal, so a unit whose behaviours exceed one agent's budget is sliced into several groups rather than treated as one. On `rain.sol.codegen`, three libraries were three groups under the module rule: `LibHexString` (28 lines, 1 function) and `LibFs` (42 lines, 2 functions) each finished clean in one agent; `LibCodeGen` (314 lines, 11 functions, 95 semantic + 30 structural mutants, its own suite grown to 51 tests) **died on context twice and needed two successor agents** — the same shape by unit count, six times the work. Its eleven functions were separable (five pointer wrappers, four constant emitters, two hash emitters) and nothing in it needed to be probed together. Shards of one file are ordinary groups — own branch, own PR; they append to the same test file, so expect to resolve a textual merge conflict between their PRs, which is far cheaper than the group dying. - - **An over-sized group does not degrade gracefully — it dies.** Cost is superlinear, not proportional: behaviours × mutants, a full probe cycle each, plus the adversarial pass, the authored tests and the PR — and every test the group adds lengthens the suite that each LATER probe in that same group re-runs, so late probes cost several times what early ones did. The overrun is then not the excess: it is a handoff written by an agent that can no longer run tools, a successor that re-reads the state and re-derives context, and finished-but-unverified work the successor must either trust or repeat. `LibCodeGen` paid that cycle twice, and the second agent's job was almost entirely reconstruction. - - **When behaviours cannot be estimated cheaply, slice smaller.** The asymmetry is not close — a group that finishes early costs a clone; a group that dies costs an agent — so over-slicing is the safe error. -3. **Learn the harness once.** Discover build/test commands and any **artifact-regeneration** step (compiled output, generated bindings, etched/deployed bytecode, golden files, snapshots) BEFORE mutating. Record them in PROGRESS.md. Re-running tests after editing source but without regenerating the artifact they execute tests **nothing**. -4. **Run the per-unit loop**, **committing each unit's tests** as you finish it (durable record; the scratch file only tracks meta-progress). Work each group on its own branch off the default. When a group's units are done and the suite is green, **push and open a PR for that group**, then start the next group on a fresh branch. Added tests are additive (no source/bytecode change), so per-group test PRs are independent and CI-safe — they review and merge on their own; don't accumulate everything on one branch. Record each group's branch + PR in PROGRESS.md. -5. **Resume.** Serial mode: on re-entry, read PROGRESS.md and pick up the current group's `[WIP]` (or next `[TODO]`) unit. Fan-out mode: re-invoke the Workflow with `resumeFromRunId` — the journal replays completed agents from cache and the orchestrator's sliced-list state resumes convergence; the audit record is regenerated from the aggregated returns, not parsed to decide what to redo. Either way: never re-do a completed unit; if a group is finished but unshipped, push + open its PR first; stop only when the worklist (or agreed scope) is exhausted, then report repo-wide coverage proven, the PRs opened, and gaps remaining. +1. **Survey**: enumerate testable units and existing tests; find gaps with + coverage tooling. Zero- and weakly-covered units are highest value. Price + each unit in BEHAVIOURS — a unit list without a size term cannot be grouped. +2. **Prioritize and group** by risk × coverage gap; each group ships as its own + branch + PR (additive test-only PRs are independent and CI-safe). **A group + is sized by the behaviours it holds, never by "a module / package / coverage + area"** — a file is not indivisible, so a unit over one agent's budget is + split across groups. Shards of one file are ordinary groups; their PRs append + to the same test file, and that textual conflict is far cheaper than a group + dying. An over-sized group does not degrade, it DIES: cost is behaviours × + mutants × a probe cycle each, and every test the group adds lengthens the + suite its own later probes re-run, so the overrun is not the excess but a + handoff plus a successor's re-read. When a unit cannot be priced cheaply, + slice smaller — finishing early costs a clone, dying costs an agent. +3. **Learn the harness once** — build, test, and any artifact-regeneration step + — BEFORE probing. Regeneration belongs INSIDE the probe's suite command: + tests that execute a stale artifact test nothing. +4. **Run the per-unit loop**, committing each unit's tests as you finish; push + and open each group's PR before starting the next group. +5. **Resume**: serial via PROGRESS.md; fan-out via `resumeFromRunId`. Never + re-do a completed unit; stop only when the worklist is exhausted. ## The per-unit loop -Drive it by mutation so existing tests are **credited** and you only add tests for genuine gaps. **Never edit or delete existing tests — only add.** Every behavior is judged against the *whole* suite (existing + anything you add). - -1. **Enumerate behaviors.** Each conditional, comparison, computation, side-effect, filter, early-return, and error/skip path is a separate thing a test can claim to cover. Include happy path, every branch, boundaries, interactions, and "should NOT happen" cases. -2. **Baseline — a green check AND a full probe of the PRE-EXISTING suite. Both complete before step 4.** - - **Green check.** Run the existing suite on the *unmutated* code. If green, proceed. If a test fails here (not under a mutation), it's legitimately broken — stop and diagnose: fix a genuinely-wrong/outdated/flaky test, or if the failure reveals a real code bug, surface it. Never mask a baseline failure by editing the assertion to match buggy behavior, and don't start mutating on top of a red baseline. - - **Then probe the whole enumerated list (step 3) against that suite, with none of your own tests written yet.** Finish the pass, *then* start writing. This pass is what produces the ledger's attribution: every kill credits a **named pre-existing** test, and **the survivors ARE step 4's worklist**. (Probing once your tests exist is not itself wrong — step 4's confirmation and step 6's later rounds both do it, recording the killer by name. What costs is the pre-existing suite never having been probed on its own.) - - **A test written before the unit's probe pass completes forfeits that attribution, and it cannot be recovered in place.** Once your tests are in the suite, what you are probing is the *combined* suite, so a kill no longer says whether the original tests already covered the behavior. Recovering it costs a **second clone at the base commit** (pre-existing tests only), **a second full mutation pass against it**, and **a diff of the two matrices** to attribute each mutant. Measured on `rain.sol.codegen`: the whole 95-mutant pass, paid a second time, for numbers the first pass would have produced for free — 14 killed (the existing tests, credited), 81 survived (the worklist). -3. **Probe each behavior with a mutation.** Apply ONE targeted mutation that breaks exactly that behavior (catalog below); **make it live in what the tests execute** (regenerate any built/cached/generated/etched artifact, and sanity-check the mutation actually changed behavior — stale artifacts are the #1 failure mode); run the **whole** suite. - - **A test fails → behavior already covered.** Note which test (often a pre-existing one). No new test needed. - - **No test fails → the mutant *survived* → a real coverage gap.** - - **Restore** source via VCS before the next probe. Never leave or commit a mutation. -4. **Kill each surviving mutant** — step 2's survivor set is the worklist. Make a test catch it: - - If an existing test **purports** to cover that behavior (by name/intent/setup) but the mutant survived, the test is inadequate → **strengthen it in place** — tighten its assertions until it fails under the mutation. (Making a test *fail* under a mutation it should catch is the goal — the opposite of editing one to *pass* under a mutation.) - - If **no** existing test targets the behavior → **add a new** test. - Either way the test must be **discriminating** — its assertion yields a *different observable value* under correct vs. wrong code (not "it runs" or a bare "it reverts"; prefer exact values / events / amounts) — and must **pass** on the unmutated baseline and **fail** under the mutation. Re-apply, regenerate, confirm both, then restore. -5. **Record the matrix** in PROGRESS.md: each behavior → covered by (existing test / new test) / still-gap. -6. **Loop until dry — a single pass is NOT done.** One prioritized pass over a handful of high-value behaviors leaves the long tail (and the subtle bugs) uncovered. Re-survey the unit for behaviors you have not yet probed — large units have dozens: every public/external entrypoint, each branch and boundary, every revert/skip/early-return path, each accounting step, event, and cross-feature interaction — and keep probing until a **full pass adds no new gap**. Unit size and per-probe cost (e.g. regenerating etched artifacts) are NEVER reasons to stop early or to sample; they only change *pacing*. "Already heavily hardened" is a hypothesis to disprove by mutation, not a reason to skip. If a unit is genuinely too large to finish in one sitting, do NOT declare it done — record in PROGRESS.md the exact list of behaviors still UNPROBED and resume there next iteration. - -## Adversarial correctness pass (find BUGS, not just gaps) - -Run this PER UNIT alongside the mutation loop — it is the half that finds bugs. The mutation loop asks "is this behavior tested?"; this asks "is this behavior CORRECT?". - -0. **Ingest the authoritative intent oracle FIRST.** The spec, NatSpec, interface contracts, and domain invariants are the oracle for what the code SHOULD do — start there and treat the code as suspect. Do NOT substitute a convenient intent the code happens to satisfy (the classic self-own: observing "dust is retained by the contract" and calling it "conservative, safe" when it's really an orphaned-funds bug). Widen the bar beyond "exploitable": accounting/UX/correctness divergences (orphaned funds, reverts-that-should-succeed, wrong-but-not-stolen values) are findings too. -1. **Derive the intended behavior independently** of the code: from the authoritative oracle above, plus type/unit constraints and domain invariants. Write the expected value/property from intent — do NOT read it off the code's current output. -2. **Enumerate invariants and properties** the unit must uphold: conservation (no value created/destroyed; balances reconcile), monotonicity, bounds (no overflow/underflow/precision loss), rounding DIRECTION (does rounding ever favor the wrong party?), idempotence, access control (only authorized callers), isolation (no cross-owner/namespace/account leakage), ordering independence, reentrancy safety, and "this must NEVER happen" cases. -3. **Try to FALSIFY each** — adversarially, against REAL dependencies. Construct the inputs, sequences, boundaries, and hostile counterparties most likely to violate the property: extreme/zero/max values, non-default decimals, dust/rounding edges, repeated/reordered/interleaved operations, reentrant callbacks, unexpected token behaviors, self-referential parties. Exercise the **real** tokens/contracts/dependencies, not the suite's always-succeed mocks — a mock that returns true for everything structurally hides conservation/decimal/solvency divergences. A single falsifying case that FAILS on the unmutated baseline against a real oracle is a **candidate bug**. -4. **When a mutant SURVIVES, ask the adversarial question first.** A surviving mutant means no test pins the behavior — before reflexively pinning the *current* output, check the current output is actually CORRECT per step 1. If it is, add the test (mutation gap). If it is NOT, you've found a bug: do not enshrine it in a test. -4b. **A KILLED mutant means PINNED, not CORRECT — covered behaviors are still in adversarial scope.** A test killing a mutant only proves the test and the code agree; they can be co-wrong. Tests routinely **mirror the implementation** — assert the code's own output, hardcode the value the code happens to produce, or recompute the "expected" with the SAME formula the code uses — so a green, mutant-killing test faithfully **enshrines whatever the code does, bug included**. Therefore run the correctness check (steps 1–3) on well-covered behaviors too, NOT only on surviving mutants. Treat as a red flag any existing test whose expected value looks **derived from the code** (same magic constants, same formula, "assert x == "): re-derive the expected value INDEPENDENTLY from the spec; if it differs from what the test asserts, the test is enshrining a bug — surface it for triage, do not trust the green check. "An existing test covers it" is never a reason to skip correctness review. -5. **Surface every candidate for TRIAGE — do NOT adjudicate it yourself.** You are not the arbiter of bug-vs-intended; the code owner is. For each observation write a neutral triage item — the behavior, where, why it might or might not be intended, and a cheap repro if available — and hand it to the owner / a triage list. **Ambiguous → flag it, never drop it.** Refutation reasoning is attached as a NOTE, never used as a GATE: auto-refute pipelines (refute-by-default skeptics, majority-vote, "not exploitable so discard") silently bury real-but-subtle and non-exploitable findings (orphaned dust, reverts-that-should-succeed) — that is the failure mode this whole pass exists to avoid. The only thing you may discard is a provably-broken repro you wrote yourself (the test asserted the wrong thing); even then keep the underlying behavior question on the list. Never weaken an assertion to make the code look correct. "I judged it fine" is not a disposition you get to make. - -Record bug candidates separately from coverage in PROGRESS.md (e.g. a `## SUSPECTED BUGS` section): unit, the violated property, the repro, verify status (real / refuted / needs-input). "No bugs found" is only credible after this pass actually ran. - -The output of this pass is a **triaged finding with a verified repro**, not a passing test merged into the suite: a bug-repro merged green either enshrines the bug or rots red, so it belongs ON the finding (the mutation pass produces the green coverage PRs; the adversarial pass produces filed issues + their repros). And **re-verify a candidate yourself before filing it** — re-run the repro against current code, because a sub-agent's scratch repro is often gone or asserted the wrong thing; file only what you reproduce. File it per **Findings → issues** below — including the label gate, which is what makes the finding visible to anything outside the issue itself. - -## Findings → issues (the adversarial pass's output) - -Findings are tracked as **GitHub issues** — the durable product record, and the half of the run that is not a PR. The orchestrator files them after synthesis and after re-verifying each repro, not per-agent mid-run. - -- **Every filed finding carries the `audit` label. It is not optional.** `audit` is the org-wide handle for "this repo has an outstanding finding": the `rain-org-health` scan counts a repo's backlog with `gh search issues --owner --label audit --state open`. A finding filed without it is **invisible** — the graph reports the repo as having zero outstanding findings, so as far as every consumer downstream of the issue is concerned the finding does not exist. This is not hypothetical: `rain.solmem`'s adversarial pass filed three real findings (#50, #54, #55) with no label, and the dashboard read `openAuditIssues: 0` until they were hand-labelled days later. -- **Create the label set FIRST (mandatory, before the first `gh issue create`).** `gh issue create --label ` **hard-fails when the label does not exist in that repo**, and a repo being scanned for the first time usually has neither label. So, once per repo, list what exists and create every missing label before filing anything: - ```sh - gh label list -R / --limit 100 - # for each missing name in: audit adversarial - gh label create audit -R / --color 5319E7 --description "Audit finding" - gh label create adversarial -R / --color A371F7 --description "Adversarial mutation-test finding" - ``` -- **Never recover from a label error by filing the issue unlabelled.** Dropping the label turns a loud failure into a silent one: the issue exists, reads fine, and is counted by nothing. If a label genuinely cannot be created (no permission), STOP and tell the user rather than filing unlabelled. -- **Issue shape:** Title = the violated behavior in one line (what is wrong and where — not "investigate X"); Labels = **`audit`** (required — the countable one) **plus `adversarial`** (provenance, so this skill's findings stay distinguishable from the audit skill's while both stay countable); Body = the unit, the intent oracle the expected behavior was derived from, the violated property, the verified repro, and the neutral triage framing of step 5 above — why it might or might not be intended. You are surfacing a candidate, not adjudicating it. -- **Verify the labels landed.** After filing, re-list (`gh issue list -R / --label audit --state open`) and confirm every issue you just created is returned. An issue created while its label was missing is silently label-less, so a `gh issue create` that printed a URL is not proof; if any are missing, add the labels now (`gh issue edit --add-label audit,adversarial`). -- **Record the filed issue numbers** in `summary.filed` of the committed scan record (see below) and in PROGRESS.md, so the run's own record and the org health graph tell the same story. +1. **Enumerate behaviors**: every conditional, comparison, computation, + side-effect, filter, early-return, and error/skip path — happy path, each + branch, boundaries, interactions, "must NOT happen" cases. +2. **Baseline green, then probe the pre-existing suite in full.** A test failing + on unmutated code is legitimately broken: fix a wrong/outdated/flaky test, or + surface the real code bug its failure reveals. Never mask a baseline failure + by matching the assertion to buggy behavior. Then, with **none of your own + tests written**, probe the whole enumerated list against that suite and + finish the pass before writing anything: every kill credits a **named + pre-existing** test, and the survivors are step 4's worklist. Writing early + forfeits that attribution and cannot be recovered in place — recovery costs a + second clone at the base commit, a second full pass, and a diff of the two + matrices, which `rain.sol.codegen` paid across 95 mutants to recover 14 + killed / 81 survived. +3. **Probe with the bundled tool.** Author one targeted mutation per behavior + (catalog below) in a mutants file, then: + + ```sh + nix run github:rainlanguage/adversarial-mutation-test#mutation-probe -- mutants.toml + ``` + + `mutation-probe --help` is the manual (file format, verdicts, exit codes). + The bin enforces probe integrity — green non-empty baseline, proof the suite + actually ran, exactly-once targets, byte-exact restore — so a crashed suite + or a no-op mutant can never fake a result. Yours to uphold: **commit before + the first probe** (the auditable recovery point), and **keep targets out of + test code** — a target in the oracle co-mutates the expectation and voids the + probe. +4. **Act on verdicts — step 2's survivor set is the worklist.** KILLED = + covered: credit the killing test in the ledger. SURVIVED = a real gap: if an + existing test purports to cover the behavior, strengthen it in place until it + kills the mutant; otherwise add a new test. Either way the test must be + **discriminating** — a different observable value under correct vs wrong code + (exact values over bare reverts) — passing on baseline and failing under the + mutation; re-probe (`--only`) until killed. Never delete a test, never weaken + one, and never edit one to pass under a mutation — that encodes the injected + bug. +5. **Loop until dry.** Record the matrix, then re-survey the unit for unprobed + behaviors and keep probing until a full pass adds no new gap. Size and + per-probe cost change pacing, never scope; an unfinished unit records its + exact unprobed list rather than being declared done. + +## Adversarial correctness pass (per unit — the half that finds bugs) + +0. **Ingest the intent oracle first**: spec, NatSpec, interface contracts, + domain invariants. Do not substitute an intent the code happens to satisfy + ("dust retained = conservative" is the classic self-own). Findings are wider + than exploits: orphaned funds, reverts-that-should-succeed, wrong-but-not- + stolen values. +1. **Derive expected behavior independently** of the code, from the oracle. +2. **Enumerate invariants**: conservation, monotonicity, bounds, rounding + DIRECTION, idempotence, access control, isolation, ordering independence, + reentrancy, "must never happen". +3. **Try to falsify each** against REAL dependencies — extreme/zero/max values, + odd decimals, dust edges, reordered/interleaved/reentrant sequences, hostile + counterparties — never the suite's always-succeed mocks. A falsifying case + failing on unmutated code is a candidate bug. +4. **A surviving mutant gets the adversarial question first**: is the current + output even correct? If yes, add the test; if no, that's a bug — do not + enshrine it. **A killed mutant is still in scope**: a test whose expected + value looks derived from the code (same formula, same magic constant) is a + red flag — re-derive from spec; a mismatch means the test enshrines a bug. +5. **Surface every candidate for TRIAGE — never adjudicate yourself.** Neutral + framing: behavior, where, why it might or might not be intended, cheap repro. + Ambiguous → flag, never drop. Refutation reasoning attaches as a NOTE, never + gates a finding; the only discardable item is a provably-broken repro you + wrote yourself, and its behavior question stays on the list. + +Re-verify each candidate's repro against current code before filing; the repro +belongs on the finding, never merged green into the suite. + +## Findings → issues + +- Every finding is filed as a GitHub issue carrying the **`audit` label plus + `adversarial`** — `audit` is what the org health scan counts; an unlabelled + finding is invisible and effectively does not exist. +- **Create missing labels before the first `gh issue create`** (it hard-fails on + absent labels), and never recover from a label error by filing unlabelled — + STOP and report instead. +- **Verify after filing**: re-list by `--label audit` and confirm every filed + issue appears; add labels to any that slipped through. +- Title = the violated behavior; body = unit, intent oracle, violated property, + verified repro, neutral triage framing. Record filed numbers in the scan + record and PROGRESS.md. ## Mutation catalog (break ONE behavior) -- **Conditionals:** negate (`x`→`!x`), force `true`/`false`, swap branches. +- **Conditionals:** negate, force true/false, swap branches. - **Comparisons:** `<`↔`<=`, `>`↔`>=`, `==`↔`!=`, swap operands. - **Arithmetic / off-by-one:** `+1`→`-1`/`+0`, `*`↔`/`, drop a term. -- **Returns / outputs:** return early, empty/zero/default, a constant. -- **Side-effects:** delete a write/emit/update; or move it across a guard so it runs in the wrong cases. -- **Constants / identifiers:** change a literal, swap an error/event, use the wrong variable. -- **Filters / scopes:** remove a predicate (ownership / namespace / key) — validates isolation tests. +- **Returns / outputs:** early return, empty/zero/default, a constant. +- **Side-effects:** delete a write/emit/update, or move it across a guard. +- **Constants / identifiers:** change a literal, swap an error/event/variable. +- **Filters / scopes:** remove a predicate (ownership / namespace / key). -Pick the mutation that maps to exactly one behavior so the failing-test set is diagnostic. +One mutation, one behavior — the failing-test set stays diagnostic. ## Parallelizing across groups (the native fan-out) -Groups are independent (separate branch, additive test-only PR), so fan them out with the native Workflow: one `agent()` per group, collected with `parallel(thunks)` (a barrier — you want every group's ledger together to aggregate) or `pipeline(groups, …)` for per-item flow. Each worker runs its group's per-unit loop, then commits, pushes, and opens its own PR. Let the runtime own the mechanics: - -- **Isolation is a fresh CLONE per worker — not a worktree, not a shared checkout.** Each worker mutates source AND commits / restores / pushes concurrently, so it needs TOTAL isolation of two things. (a) **Build state:** `forge` writes bytecode to `out/` + an incremental `cache/`, soldeer writes `dependencies/`, and any regen step (`build.sh`) rewrites generated sources — if that untracked build state is shared, one worker's mutated/stale artifact becomes what ANOTHER worker's test executes, so a mutant reads as "killed"/"survived" for the wrong reason and the campaign **silently lies** (the #1 failure mode). (b) **Git state:** refs, hooks, config, index. A `git worktree` happens to dir-isolate (a) but **shares one `.git`**, so concurrent ref-updates / commits / checkouts — and the repo's pre-commit hooks firing on every worker's commit — contend, and a bad op's blast radius reaches the real source checkout. A **`git clone` per worker isolates BOTH**, with nothing shared except the immutable global fork/compiler cache under `~/.foundry` (which you WANT shared). So each worker `git clone`s the repo, provisions it independently (install deps + build), runs its loop, commits, pushes, opens its PR. **Do NOT use `isolation:'worktree'` for this skill — its shared `.git` is exactly the problem.** (The org-wide sweep clones each DIFFERENT repo — same primitive, uniformly.) -- **Each worker builds/regenerates inside its own clone** — keep the install / build / regenerate-artifact step so the mutated source is what the tests actually execute (stale artifacts are the #1 way mutation lies). Parallelism is **across** groups; **within** a group the `mutate → regenerate → test → restore` cycle stays serial. -- **Don't hand-manage concurrency.** The runtime auto-caps at `min(16, cpu-2)` and backstops total agents at 1000 (a runaway guard, NEVER a coverage cap). Emit one agent per group/batch and let the scheduler throttle. -- **Effort per stage.** Probing (mutate → regenerate → test → restore, observe pass/fail) is mechanical — run probe agents at `effort:'low'`. Killing a survivor with a discriminating test, and the whole adversarial correctness pass + the final synthesis, are hard reasoning — `effort:'high'`/`'xhigh'`. Don't run the bug-finding half at probe altitude. -- **Failed vs empty is the native `null` contract — don't re-implement it.** An agent that dies after the runtime's retries returns `null`; `.filter(Boolean)` drops those, and a `null` is your re-dispatch signal. An agent that ran and returned a schema-valid empty result genuinely found nothing. So "failed to provision" vs "found nothing" is distinguished for free — no sentinel bookkeeping. `agent()` retries only terminal **API** errors, so a clone / install / build failure INSIDE a worker is the worker's OWN bash to retry (have its prompt retry a few times, cleaning the partial checkout between attempts). A group whose agent came back `null` is a GAP — re-dispatch it **promptly, in parallel with the rest**, never parked "until the end"; the run isn't complete until every group either produced results or is verifiably empty. -- **Clean up the per-worker clones when the run ends.** Each clone is throwaway scratch infrastructure — a full provisioned checkout (deps + build artifacts, often hundreds of MB). When the run finishes or is abandoned, delete the clones it created — but FIRST check each for unpushed commits / uncommitted work and preserve anything of value (push it, or capture it in the issue/PR). Reusing a clone by resetting it to a new branch is NOT cleanup; never delete a checkout you didn't create for this run. - -### Survey → slice → loop-until-dry (orchestrator-owned) - -For a unit too big for one agent's context, the **orchestrator** slices the worklist — do NOT have agents self-select from a shared file. - -- **Survey returns a validated list, not a count.** Run the survey as `agent(prompt, {schema})` forcing an array of behaviour/unit items; the Workflow validates at the tool layer, so the orchestrator gets a real array to `.slice()` with zero parsing. Force every downstream probe / kill / adversarial agent to a schema too, so the coverage ledger and triage list assemble from validated returns, never parsed prose. -- **Make the survey schema carry a per-unit behaviour estimate — a list of units is size-blind.** "Three libraries" reads identically whether one of them is six times the others, so an orchestrator slicing on unit identity alone has no term for size and cannot apply the sizing rule above. Force each item to return its enumerated (or estimated) behaviour count alongside its name, and treat a cheap estimate as a FLOOR: where the survey cannot price a unit, slice it smaller. -- **Partition in orchestrator code** into disjoint batches (~5–8 items) and dispatch one agent per batch with its EXPLICIT items in the prompt — partitioning is on the behaviour axis, so a 95-behaviour unit is a dozen batches, not one. Every behaviour is assigned exactly once; coverage is deterministic and exhaustive by construction. -- **Converge with the native loop-until-dry, in orchestrator code.** Collect each round's structured returns (via `parallel()`, so you have the whole round), add newly-surfaced behaviours to a **seen-set**, re-slice the new ones, and fan out again — repeat until a round adds nothing new. Dedup against the seen-set, NOT against confirmed gaps. Convergence depends on orchestrator-controlled state (the sliced list is exhausted), **never** on agents faithfully editing shared state. - - **The failure mode this avoids (learned the hard way):** do NOT have chunk-agents read a shared `[TODO]` checklist, pick "the next few", probe them, and mark them `[HUNTED]` across iterations with the loop terminating on a `dry` flag. Sub-agents reliably skip the bookkeeping — so the checklist never updates, every chunk re-reads the same `[TODO]`s and re-does the same work, the `dry` flag never flips, and the loop burns its whole budget on duplicate work (≈180 agents, zero net progress, large wasted compute, in one real run). -- **Cross-restart resume is `resumeFromRunId`** — re-invoking the Workflow replays the journaled `agent()` calls from cache, so completed batches aren't re-run and the orchestrator's loop state reconstructs natively; you don't hand-persist the sliced list. -- **Never cap coverage with a fixed agent count** — run as many batches as the (possibly growing) list requires; the 1000-agent cap is only the runaway backstop. Scale rounds to `budget.remaining()` when the user set a token target. - -### Dispatcher duties (YOU, the orchestrator authoring the Workflow — not the sub-agents) - -Fanning work out does NOT delegate the review; you own the synthesis, at high effort, AFTER the Workflow returns. The sub-agents wrote the rules above; these are for you. - -- **A worker's conclusion is an INPUT, not a verdict.** "No candidates", "no bugs", "all green" from a worker is a claim to audit, never to relay. You have not found "no bugs" until you have reviewed what the worker actually did and saw. -- **Put the prose where you can act on it — make NOTES a schema field.** The buried findings live in the prose, so force each worker's structured return to carry it (e.g. `{candidates, notes, suppressionFlags}`). Then every "discarded", "not exploitable", "by design", "benign", "safe", "conservative", "expected", "no recovery path", "only a UX issue", "refuted", "out of scope" lands in a validated field you iterate **deterministically** — surfacing the buried behavior is orchestrator code, not a hope that you read free prose. -- **Union, never vote.** Collect verify agents with `parallel(thunks).filter(Boolean)` and UNION every surfaced candidate into the triage list; a skeptic's refutation attaches as a NOTE, never gates or out-votes the finding. Your report inherits the weakest worker — one surfaced item survives even if three others "refuted" it. Do not average findings away. -- **Cross-check a worker's "clean" claim against the oracle yourself.** Re-derive the intended behavior for an area a worker reports clean and confirm it actually exercised the risky cases (conservation, rounding direction, decimals, isolation, access) against the spec — "clean" is a claim to verify, not accept (it is how subtle accounting findings get buried under "conservative, safe"). -- **Re-read suspicious reasoning for errors.** Workers analyze things backwards (e.g. claiming "pull before push" for code that pushes before pulling). If a worker's safety argument hinges on an ordering/sign/rounding claim, verify the claim against the code before accepting it. - -## Leave a committed scan record (org-wide health tracking) - -At the END of a run, commit a **minimal, machine-readable** record so an org-wide health-check can tell which repos were scanned recently from which are stale — and against **which release**. This is the inverse of `PROGRESS.md`: that is gitignored local working state; this is a small **committed** file that travels with the repo. - -- **Predictable path, same in every repo** so a health-check can fetch it uniformly (`gh api` / raw URL): **`audit/mutation-test-scans.json`**. (Create the `audit/` directory if the repo has none — the scan record is an audit artifact and belongs with audit outputs. It is committed, unlike the gitignored `.mutation-test/` scratch dir.) -- **Append one entry per run** (keep history; the health-check reads the newest `timestamp` for recency): - ```json - { - "timestamp": "2026-06-06T19:40:00Z", // UTC, when the run finished - "commit": "08d547f…", // the exact SHA scanned - "publishedTag": "v1.2.3", // the published/release version AT that commit, or null if unreleased - "commitsAheadOfTag": 0, // how far the scanned commit is past that tag - "scope": "whole repo", // or the module scoped - "tool": "adversarial-mutation-test", "skillVersion": "0.28.0", - "summary": { "behaviours": 600, "candidates": 89, "confirmed": 30, "filed": ["#2651","#2660"] } - } - ``` -- **Record what was CHECKED — including the published tag.** Staleness is "which *release* was last audited," not just "when." Resolve the published version at the scanned commit: the release tag (`git describe --tags --abbrev=0`), and/or the version in the package manifest (`soldeer.toml` / `Cargo.toml` / `package.json`). If the scanned commit is ahead of the last release, record both the tag and `commitsAheadOfTag`. -- **Land it on the default branch** — include the record commit in the findings PR, or a tiny dedicated PR; a record that never leaves a local branch is invisible to the org health-check. (Commit it even if the run found nothing — "scanned, clean, on date X" is exactly the signal a health-check needs.) -- Minimal is fine: `timestamp` + `commit` + `publishedTag` are the must-haves; the `summary` is nice-to-have. +- **One fresh CLONE per worker — never a worktree, never a shared checkout.** + Workers mutate source and build state concurrently; shared untracked build + output makes one worker's stale artifact another worker's test subject (the + matrix silently lies), and a worktree's shared `.git` contends on every + commit/restore. A clone isolates both; each worker provisions, probes, + commits, pushes, and opens its own PR. +- **Orchestrator slices; agents never self-select.** Survey returns a + schema-validated list carrying each item's behaviour count — an identity-only + list is size-blind, and slicing it cannot honour the sizing rule above. + Partition on the BEHAVIOUR axis into explicit disjoint batches in orchestrator + code (a 95-behaviour unit is a dozen batches, not one), treating an estimated + count as a floor; converge loop-until-dry against a seen-set. Agents editing a + shared TODO list is the known failure mode (duplicate work, no convergence). +- A `null` agent return is a re-dispatch signal, promptly and in parallel — a + group is either productive or verifiably empty before the run is complete. + Probe agents run at low effort; kill/adversarial/synthesis at high. +- **Clean up clones at run end** — after checking each for unpushed work worth + preserving. Never delete a checkout you didn't create. + +## Dispatcher duties (the orchestrator's own review) + +- A worker's "no bugs / all green / discarded / by design / not exploitable" is + an INPUT to audit, never a verdict to relay. Force NOTES into the structured + return schema and iterate the suppressions deterministically. +- **Union, never vote**: every surfaced candidate enters triage; a skeptic's + refutation is a note. Cross-check "clean" claims against the oracle yourself, + and verify any safety argument hinging on an ordering/sign/rounding claim + against the actual code — workers analyze these backwards. + +## Committed scan record + +Close every run — including a clean one — by appending an entry to a committed +`audit/mutation-test-scans.json` and landing it on the default branch: +timestamp, scanned commit, published tag (+ commits ahead), scope, tool + skill +version, summary with filed issue numbers. The org health check reads the newest +entry for "which release was last audited"; the JSON template lives in this +repo's README. ## Principles -- **Never edit a test to pass under a MUTATION.** A test failing under a mutation is SUCCESS — it caught the injected bug; reverting the mutation restores green. Changing a test's assertion to swallow a mutation encodes the bug. The only test you adjust *mid-mutation* is a *new* one you just wrote that failed to discriminate (didn't fail under its own target mutation) — strengthen it. -- **Strengthen weak tests; don't duplicate them.** If a mutant survives and an existing test *purports* to cover that behavior, it's inadequate — tighten it in place until it kills the mutant, rather than leaving it and adding a redundant parallel test. Add a *new* test only when no existing test targets the behavior. Don't gratuitously rewrite tests that already do their job. -- **Fix legitimately broken tests; surface real bugs.** A test failing on the *unmutated* baseline is broken: fix it if its assertion is wrong/outdated/flaky, or if the failure exposes a real code bug, report the bug — never mask a baseline failure by editing the assertion to match buggy code. Distinguish **"the test is wrong"** (fix it) from **"the code is wrong"** (report it). -- **Discriminating assertions** — "got 3, expected 1" beats "it reverted". -- **One mutation, one behavior** — isolation makes the failing set identify the covered line. -- **Confirm the mutation is live** — stale artifacts are the #1 way this lies to you. -- **Probe the pre-existing suite BEFORE writing a test** — attribution exists only while the suite is untouched. Write first and every probe measures your own tests too; recovering which mutants the ORIGINAL tests killed then costs a second clone at the base commit, a second full mutation pass, and a diff of the two matrices. -- **Harden the probe harness itself — a lying harness fakes the whole campaign.** Four integrity rules, each from a real incident where the matrix was silently wrong: - - **Commit (or pin) the suite BEFORE the first probe.** Restore-via-VCS restores committed state; when the new tests share a file with the mutated source (e.g. a Rust in-file `#[cfg(test)]` module) and are uncommitted, the first restore WIPES them — every later probe runs testless and reports universal survival. - - **Assert the baseline count before probing.** The harness must run the clean tree first and abort unless it sees the expected `N passed`; "0 tests ran" must be a loud failure, never a silent "everything survived". - - **Classify probe outcomes from the test harness's own result line, not by grepping for "error".** `cargo test` prints `error: test failed` on every KILL — a naive error-grep reclassifies kills as compile failures. Harness-ran (result line present) → killed/survived from the failing-test list; result line absent → the mutation was invalid. - - **Scope automated mutations away from the oracle.** A whole-file sed can rewrite the test module's expected values in lockstep with the code (same literal in both) — the mutant then "survives" against a co-mutated oracle. Restrict the mutation to the code region (address range above `#[cfg(test)]`, exclude test dirs), and treat a survival whose diff touched test code as void. -- **Durable state, not conversation memory** — committed tests/issues are the durable record of WORK; PROGRESS.md is the authoritative HUMAN/audit narrative; under a Workflow, run-time resume and convergence are owned natively (the run journal / `resumeFromRunId` + cached agent results + orchestrator loop state), not by PROGRESS.md. -- **Always restore** mutations via VCS; verify a clean tree before committing tests. -- **Comments describe behavior, not the mutation process.** -- **Scale to scope, and size groups by behaviours** — a fix → a handful of mutations; a whole repo → a chunked, tracked, resumable campaign with a warm toolchain, and for very large repos a parallel fan-out whose groups are sized by their enumerated behaviours, never by "a module" (one 314-line library was one group by that rule and died on context twice while its 28- and 42-line siblings finished clean). Split an over-budget unit across groups; an over-sized group does not degrade gracefully, it dies and pays for a handoff plus a re-read, and a group that finishes early only costs a clone. -- **The dispatcher owns the synthesis — review sub-agent work, don't relay it.** When you fan out to sub-agents, their conclusions are inputs you must audit, not verdicts you forward. Read their NOTES (not just the structured result) for buried suppressions — "discarded / by design / not exploitable / benign / safe" are items to pull up, not accept. Verify any safety argument that hinges on an ordering/sign/rounding claim against the actual code. Relaying a worker's "no bugs found" without reviewing what it observed is the same rubber-stamp as a bare "Reviewed" on a PR. -- **A passing test is not a correct behavior — coverage ≠ correctness.** A test killing a mutant only proves the test and the code agree, and tests routinely mirror the implementation (assert the code's own output / recompute with the same formula), so a green test can faithfully enshrine a bug. Run the correctness check on covered behaviors too; treat a test whose expected values look derived from the code as a red flag and re-derive them independently from the spec. "An existing test covers it" is never validation, and never a reason to skip adversarial scrutiny. -- **Question the oracle — adversarial, not just mutation.** Mutation testing makes the code the oracle and can only find test gaps; it CANNOT find a bug, because it pins whatever the code does. The "adversarial" half makes the *spec/intent* the oracle and the code suspect: derive the expected value/property independently and hunt for inputs where the code violates it. A run that only adds passing tests and reports "no bugs" did half the job — say so honestly rather than calling the absence "expected". Every exact-value assertion is a chance to check the value against intent, not just against the code's output. -- **Exhaust, don't sample — loop until dry.** Probe *every* behavior of a unit, then re-survey for the ones you missed; stop only when a full pass surfaces no new gap. A single pass over the high-value subset is a coverage *sample*, not coverage. A large or expensive-to-probe unit (e.g. one whose tests run etched/regenerated bytecode) gets paced and resumed — never truncated, and never declared done on the strength of "it looked well-tested already". -- **Every filed finding is labelled `audit` — create the labels before filing, never file unlabelled.** The org health graph counts a repo's outstanding findings by `--label audit`, so an unlabelled finding is invisible and effectively does not exist (`rain.solmem` filed three and the graph read zero). `gh issue create --label` hard-fails on a label the repo lacks, so create `audit` (+ `adversarial` for provenance) up front, and re-list after filing to confirm they stuck. A label that cannot be created is a STOP-and-report, never a reason to file bare. -- **End with a committed scan record.** Every run closes by appending an entry to a committed `audit/mutation-test-scans.json` (timestamp, scanned commit, published tag, scope, summary) and landing it on the default branch — even a clean run — so org-wide health tracking can distinguish recently-scanned repos from stale ones and know which release was audited. +- Strengthen weak tests in place; add only where nothing targets the behavior; + never gratuitously rewrite tests that do their job. +- Discriminating assertions: "got 3, expected 1" beats "it reverted". +- Confirm the mutation is live — regeneration inside the suite command; stale + artifacts are the #1 way a matrix lies. +- Probe the pre-existing suite before writing a test — attribution exists only + while the suite is untouched. +- Coverage ≠ correctness: green, mutant-killing tests can enshrine a bug; + re-derive expected values from spec. +- Exhaust, don't sample — loop until a full pass adds nothing; pace, never + truncate. +- Durable state over conversation memory: committed tests and issues are the + record of work; PROGRESS.md is the narrative; the Workflow journal owns + fan-out resume. +- Comments describe behavior, not the mutation process.