From 8b4a93592d3a7a50b47b97feedcbf0a241faa8f2 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 16 Aug 2026 16:48:16 +0000 Subject: [PATCH 1/3] fail-pattern: abort when it matches the baseline, and ship known-good patterns MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #13. Two independent defects hit `suite.fail-pattern` in one campaign, neither a property of the repo it happened in. TOO WIDE. `'\] (test\w+)\('` matches forge's `[PASS] testFoo(` as readily as `[FAIL: ...] testFoo(`, so mutants were credited to tests that pass under them — a pure-constant assertion named as the killer of a guard mutant. The green baseline is a free oracle for this: nothing failed there, so anything the pattern captures out of baseline output is a passing test. `mutation-probe` now aborts on that, with the offending captures in the message, the way a red baseline already aborts. TOO NARROW, AND SILENT. `'\[FAIL.*?\] (test\w+)\('` fixes the first and then names nobody, because `.` does not cross the newlines forge puts inside a multi-line assertion message. The verdict is unaffected (it comes from the tally), so the only symptom is an empty killer column. The baseline check cannot see this one — a pattern that matches nothing matches nothing at baseline either — so each such kill now prints "killer NOT NAMED". And `harness = "forge" | "cargo"` supplies both patterns, so a campaign stops authoring the field that carries both mistakes. Each shipped pattern is pinned by a test to real captured output of that harness under fixtures/, green and red, across two forge versions. Co-Authored-By: Claude Opus 5 (1M context) --- README.md | 36 +- .../fixtures/cargo-1.95.0-green.txt | 8 + .../fixtures/cargo-1.95.0-red.txt | 59 +++ .../fixtures/forge-1.0.0-nightly-red.txt | 74 +++ .../fixtures/forge-1.7.1-green.txt | 8 + .../fixtures/forge-1.7.1-red.txt | 83 +++ mutation-probe-rs/src/main.rs | 473 +++++++++++++++++- mutation-probe-rs/tests/toy.rs | 147 ++++++ 8 files changed, 857 insertions(+), 31 deletions(-) create mode 100644 mutation-probe-rs/fixtures/cargo-1.95.0-green.txt create mode 100644 mutation-probe-rs/fixtures/cargo-1.95.0-red.txt create mode 100644 mutation-probe-rs/fixtures/forge-1.0.0-nightly-red.txt create mode 100644 mutation-probe-rs/fixtures/forge-1.7.1-green.txt create mode 100644 mutation-probe-rs/fixtures/forge-1.7.1-red.txt diff --git a/README.md b/README.md index b2af637..2bcabec 100644 --- a/README.md +++ b/README.md @@ -82,17 +82,16 @@ nix run github:rainlanguage/adversarial-mutation-test#mutation-probe -- mutants. `mutation-probe --help` is the complete manual. The short form: the mutants file names the suite command as argv (artifact regeneration included — the probe runs -exactly that per verdict), a proof-of-run regex reading the suite's own -pass/fail tally, and the mutants as exact-string `(file, target, replacement)` -triples that must match exactly once. +exactly that per verdict), the harness whose output it should read, and the +mutants as exact-string `(file, target, replacement)` triples that must match +exactly once. ```toml [suite] root = "." command = ["nix", "develop", "-c", "cargo", "test"] -proof = '(\d+) passed; (\d+) failed' -fail-pattern = 'test (\S+) \.\.\. FAILED' # optional: names the killer -timeout-secs = 1800 # optional +harness = "cargo" # or "forge" — supplies proof + fail-pattern +timeout-secs = 1800 # optional [[mutants]] name = "M01 guard inverted" @@ -101,15 +100,28 @@ target = "if !ok {" replacement = "if ok {" ``` +`harness` exists because the two regexes it stands in for — a proof-of-run over +the suite's own pass/fail tally, and a `fail-pattern` naming the test that +killed a mutant — describe a harness's OUTPUT FORMAT, not a repo. Written per +campaign they get written wrong the same two ways: too wide, matching passing +result lines so that mutants are credited to tests which cannot kill them; or +too narrow, matching nothing and emptying the killer column without ever saying +so. Both are still available (`proof`, `fail-pattern`) and override the +harness's, and the probe now aborts on the first class — a fail-pattern that +captures anything from the GREEN baseline is matching passing lines by +construction — and prints `killer NOT NAMED` per kill for the second. + Verdicts: `KILLED` (failing tally, or non-zero exit with proof of a run) / `SURVIVED` (ran green: a real gap) / `NO-RUN` (no proof the suite ran — crash, compile error, timeout — never scored as survived) / `HARNESS-ERROR` (target not -matched exactly once). A red, silent, or zero-test baseline aborts before any -probe; writes are atomic and every restore is verified byte-exact; a hung -suite's whole process group is killed at `timeout-secs`. Exit 0 only when every -probed mutant is killed; 1 on any non-kill; 2 when the pass cannot be trusted. -`--only ` re-runs a subset while strengthening a killer; -`--json ` writes the machine-readable report. +matched exactly once). None of them is read from `fail-pattern`: a broken +pattern degrades attribution and never the verdict. A red, silent, or zero-test +baseline aborts before any probe; writes are atomic and every restore is +verified byte-exact; a hung suite's whole process group is killed at +`timeout-secs`. Exit 0 only when every probed mutant is killed; 1 on any +non-kill; 2 when the pass cannot be trusted. `--only ` re-runs a +subset while strengthening a killer; `--json ` writes the machine-readable +report. ## Scan record template diff --git a/mutation-probe-rs/fixtures/cargo-1.95.0-green.txt b/mutation-probe-rs/fixtures/cargo-1.95.0-green.txt new file mode 100644 index 0000000..48d1540 --- /dev/null +++ b/mutation-probe-rs/fixtures/cargo-1.95.0-green.txt @@ -0,0 +1,8 @@ + Finished `test` profile [unoptimized + debuginfo] target(s) in 0.00s + Running unittests src/lib.rs (target/debug/deps/cargo_toy-9851a04534560ea2) + +running 1 test +test tests::unit_passes ... ok + +test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 1 filtered out; finished in 0.00s + diff --git a/mutation-probe-rs/fixtures/cargo-1.95.0-red.txt b/mutation-probe-rs/fixtures/cargo-1.95.0-red.txt new file mode 100644 index 0000000..9ff842e --- /dev/null +++ b/mutation-probe-rs/fixtures/cargo-1.95.0-red.txt @@ -0,0 +1,59 @@ + Finished `test` profile [unoptimized + debuginfo] target(s) in 0.02s + Running unittests src/lib.rs (target/debug/deps/cargo_toy-9851a04534560ea2) + +running 2 tests +test tests::unit_passes ... ok +test tests::unit_fails_multiline ... FAILED + +failures: + +---- tests::unit_fails_multiline stdout ---- + +thread 'tests::unit_fails_multiline' (228965) panicked at src/lib.rs:18:9: +assertion `left == right` failed: generated source differs: +line one +line two + left: "a\nb\nc" + right: "a\nb\nd" +note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace + + +failures: + tests::unit_fails_multiline + +test result: FAILED. 1 passed; 1 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.02s + +error: test failed, to rerun pass `--lib` + Running tests/integration.rs (target/debug/deps/integration-4de883e90258a294) + +running 2 tests +test integration_fails ... FAILED +test integration_passes ... ok + +failures: + +---- integration_fails stdout ---- + +thread 'integration_fails' (228968) panicked at tests/integration.rs:8:5: +assertion `left == right` failed + left: 1 + right: 2 +note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace + + +failures: + integration_fails + +test result: FAILED. 1 passed; 1 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.01s + +error: test failed, to rerun pass `--test integration` + Doc-tests cargo_toy + +running 1 test +test src/lib.rs - two (line 2) ... ok + +test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.47s + +error: 2 targets failed: + `--lib` + `--test integration` diff --git a/mutation-probe-rs/fixtures/forge-1.0.0-nightly-red.txt b/mutation-probe-rs/fixtures/forge-1.0.0-nightly-red.txt new file mode 100644 index 0000000..cfb6a64 --- /dev/null +++ b/mutation-probe-rs/fixtures/forge-1.0.0-nightly-red.txt @@ -0,0 +1,74 @@ +Compiling 22 files with Solc 0.8.25 +Solc 0.8.25 finished in 638.67ms +Compiler run successful! +proptest: Saving this and future failures in cache/fuzz/failures +proptest: If this test was run on a CI system, you may wish to add the following line to your copy of the file. (You may need to create it.) +cc 8b5dede949389349e6efb74f7812838e6847e5ecccff479e0cb60fa9bbb572bc + +Ran 4 tests for test/Evidence.t.sol:EvidenceTest +[PASS] testAppliedIsIdempotent() (gas: 266) +[FAIL: assertion failed: // SPDX-License-Identifier: MIT +pragma solidity =0.8.25; + +library LibGenerated { + bytes32 constant HEAD = bytes32(uint256(1)); +} + != // SPDX-License-Identifier: MIT +pragma solidity =0.8.25; + +library LibGenerated { + bytes32 constant HEAD = bytes32(uint256(2)); +} +] testGeneratedSourceMatchesSnapshot() (gas: 5529) +[PASS] testHeadGenesisIsNotZero() (gas: 212) +[FAIL: assertion failed: 1 != 2] testSingleLineFailure() (gas: 3333) +Suite result: FAILED. 2 passed; 2 failed; 0 skipped; finished in 404.84µs (269.75µs CPU time) + +Ran 5 tests for test/Shapes.t.sol:ShapesTest +[FAIL: Custom(1, 2)] testCustomErrorRevertIsUncaught() (gas: 5877) +[FAIL: assertion failed: 958 >= 10; counterexample: calldata=0x8054777000000000000000000000000000000000000000000000000000000000000007a6 args=[1958]] testFuzz_BoundedIsAlwaysSmall(uint256) (runs: 1, μ: 747, ~: 747) +[FAIL: revert: plain string reason] testPlainRevert() (gas: 495) +[PASS] testThisOnePasses() (gas: 255) +[FAIL: assertion failed: 3 != 4] test_snake_case_name_fails() (gas: 3355) +Suite result: FAILED. 1 passed; 4 failed; 0 skipped; finished in 817.47µs (966.39µs CPU time) + +Ran 2 tests for test/Invariant.t.sol:InvariantTest +[FAIL: assertion failed: 1 != 0] + [Sequence] (original: 1, shrunk: 1) + sender=0x0000000000000000000000000000000000000020 addr=[test/Invariant.t.sol:Counter]0x5615dEB798BB3E4dFa0139dFa1b3D433Cc23b72f calldata=inc() args=[] + invariant_NeverIncrements() (runs: 0, calls: 0, reverts: 0) +[PASS] testCounterStartsAtZero() (gas: 7803) +Suite result: FAILED. 1 passed; 1 failed; 0 skipped; finished in 1.47ms (786.05µs CPU time) + +Ran 3 test suites in 4.24ms (2.69ms CPU time): 4 tests passed, 7 failed, 0 skipped (11 total tests) + +Failing tests: +Encountered 2 failing tests in test/Evidence.t.sol:EvidenceTest +[FAIL: assertion failed: // SPDX-License-Identifier: MIT +pragma solidity =0.8.25; + +library LibGenerated { + bytes32 constant HEAD = bytes32(uint256(1)); +} + != // SPDX-License-Identifier: MIT +pragma solidity =0.8.25; + +library LibGenerated { + bytes32 constant HEAD = bytes32(uint256(2)); +} +] testGeneratedSourceMatchesSnapshot() (gas: 5529) +[FAIL: assertion failed: 1 != 2] testSingleLineFailure() (gas: 3333) + +Encountered 1 failing test in test/Invariant.t.sol:InvariantTest +[FAIL: assertion failed: 1 != 0] + [Sequence] (original: 1, shrunk: 1) + sender=0x0000000000000000000000000000000000000020 addr=[test/Invariant.t.sol:Counter]0x5615dEB798BB3E4dFa0139dFa1b3D433Cc23b72f calldata=inc() args=[] + invariant_NeverIncrements() (runs: 0, calls: 0, reverts: 0) + +Encountered 4 failing tests in test/Shapes.t.sol:ShapesTest +[FAIL: Custom(1, 2)] testCustomErrorRevertIsUncaught() (gas: 5877) +[FAIL: assertion failed: 958 >= 10; counterexample: calldata=0x8054777000000000000000000000000000000000000000000000000000000000000007a6 args=[1958]] testFuzz_BoundedIsAlwaysSmall(uint256) (runs: 1, μ: 747, ~: 747) +[FAIL: revert: plain string reason] testPlainRevert() (gas: 495) +[FAIL: assertion failed: 3 != 4] test_snake_case_name_fails() (gas: 3355) + +Encountered a total of 7 failing tests, 4 tests succeeded diff --git a/mutation-probe-rs/fixtures/forge-1.7.1-green.txt b/mutation-probe-rs/fixtures/forge-1.7.1-green.txt new file mode 100644 index 0000000..bfe10d4 --- /dev/null +++ b/mutation-probe-rs/fixtures/forge-1.7.1-green.txt @@ -0,0 +1,8 @@ +No files changed, compilation skipped + +Ran 2 tests for test/Evidence.t.sol:EvidenceTest +[PASS] testAppliedIsIdempotent() (gas: 266) +[PASS] testHeadGenesisIsNotZero() (gas: 212) +Suite result: ok. 2 passed; 0 failed; 0 skipped; finished in 325.01µs (152.22µs CPU time) + +Ran 1 test suite in 4.38ms (325.01µs CPU time): 2 tests passed, 0 failed, 0 skipped (2 total tests) diff --git a/mutation-probe-rs/fixtures/forge-1.7.1-red.txt b/mutation-probe-rs/fixtures/forge-1.7.1-red.txt new file mode 100644 index 0000000..c062e25 --- /dev/null +++ b/mutation-probe-rs/fixtures/forge-1.7.1-red.txt @@ -0,0 +1,83 @@ +Compiling 22 files with Solc 0.8.25 +Solc 0.8.25 finished in 642.30ms +Compiler run successful! +{"timestamp":1786898783,"event":"failure","invariant":"invariant_NeverIncrements","target":"test/Invariant.t.sol:InvariantTest","reason":"assertion failed: 1 != 0"} + +Ran 4 tests for test/Evidence.t.sol:EvidenceTest +[PASS] testAppliedIsIdempotent() (gas: 266) +[FAIL: assertion failed: // SPDX-License-Identifier: MIT +pragma solidity =0.8.25; + +library LibGenerated { + bytes32 constant HEAD = bytes32(uint256(1)); +} + != // SPDX-License-Identifier: MIT +pragma solidity =0.8.25; + +library LibGenerated { + bytes32 constant HEAD = bytes32(uint256(2)); +} +] testGeneratedSourceMatchesSnapshot() (gas: 5529) +[PASS] testHeadGenesisIsNotZero() (gas: 212) +[FAIL: assertion failed: 1 != 2] testSingleLineFailure() (gas: 3333) +Suite result: FAILED. 2 passed; 2 failed; 0 skipped; finished in 2.36ms (421.32µs CPU time) + +Ran 5 tests for test/Shapes.t.sol:ShapesTest +[FAIL: Custom(1, 2)] testCustomErrorRevertIsUncaught() (gas: 5877) +[FAIL: assertion failed: 659 >= 10; counterexample: calldata=0x8054777000000000000000000000000000214feb2caf420de2e7d1a5d26675eab6a0e2bb args=[742886487892369547698386939959150855759913659 [7.428e44]]] testFuzz_BoundedIsAlwaysSmall(uint256) (runs: 1, μ: 747, ~: 747) +[FAIL: plain string reason] testPlainRevert() (gas: 495) +[PASS] testThisOnePasses() (gas: 255) +[FAIL: assertion failed: 3 != 4] test_snake_case_name_fails() (gas: 3355) +Suite result: FAILED. 1 passed; 4 failed; 0 skipped; finished in 2.37ms (2.43ms CPU time) + +Ran 2 tests for test/Invariant.t.sol:InvariantTest +[FAIL: assertion failed: 1 != 0] + [Sequence] (original: 1, shrunk: 1) + sender=0x00000000000000000000000000000000000000E6 addr=[test/Invariant.t.sol:Counter]0x5615dEB798BB3E4dFa0139dFa1b3D433Cc23b72f calldata=inc() args=[] + invariant_NeverIncrements() (runs: 0, calls: 0, reverts: 0) + +╭----------+----------+-------+---------+----------╮ +| Contract | Selector | Calls | Reverts | Discards | ++==================================================+ +| Counter | inc | 1 | 0 | 0 | +╰----------+----------+-------+---------+----------╯ + +[PASS] testCounterStartsAtZero() (gas: 7803) +Suite result: FAILED. 1 passed; 1 failed; 0 skipped; finished in 66.86ms (66.08ms CPU time) + +Ran 3 test suites in 67.83ms (71.59ms CPU time): 4 tests passed, 7 failed, 0 skipped (11 total tests) + +Failing tests: +Encountered 2 failing tests in test/Evidence.t.sol:EvidenceTest +[FAIL: assertion failed: // SPDX-License-Identifier: MIT +pragma solidity =0.8.25; + +library LibGenerated { + bytes32 constant HEAD = bytes32(uint256(1)); +} + != // SPDX-License-Identifier: MIT +pragma solidity =0.8.25; + +library LibGenerated { + bytes32 constant HEAD = bytes32(uint256(2)); +} +] testGeneratedSourceMatchesSnapshot() (gas: 5529) +[FAIL: assertion failed: 1 != 2] testSingleLineFailure() (gas: 3333) + +Encountered 1 failing test in test/Invariant.t.sol:InvariantTest +[FAIL: assertion failed: 1 != 0] + [Sequence] (original: 1, shrunk: 1) + sender=0x00000000000000000000000000000000000000E6 addr=[test/Invariant.t.sol:Counter]0x5615dEB798BB3E4dFa0139dFa1b3D433Cc23b72f calldata=inc() args=[] + invariant_NeverIncrements() (runs: 0, calls: 0, reverts: 0) + +Encountered 4 failing tests in test/Shapes.t.sol:ShapesTest +[FAIL: Custom(1, 2)] testCustomErrorRevertIsUncaught() (gas: 5877) +[FAIL: assertion failed: 659 >= 10; counterexample: calldata=0x8054777000000000000000000000000000214feb2caf420de2e7d1a5d26675eab6a0e2bb args=[742886487892369547698386939959150855759913659 [7.428e44]]] testFuzz_BoundedIsAlwaysSmall(uint256) (runs: 1, μ: 747, ~: 747) +[FAIL: plain string reason] testPlainRevert() (gas: 495) +[FAIL: assertion failed: 3 != 4] test_snake_case_name_fails() (gas: 3355) + +Encountered a total of 7 failing tests, 4 tests succeeded + +Tip: Run `forge test --rerun` to retry only the 7 failed tests + +Fuzz seed: 0x1 (use `--fuzz-seed` to reproduce) diff --git a/mutation-probe-rs/src/main.rs b/mutation-probe-rs/src/main.rs index 1802f84..6ef63b8 100644 --- a/mutation-probe-rs/src/main.rs +++ b/mutation-probe-rs/src/main.rs @@ -47,12 +47,21 @@ struct SuiteConfig { /// part of this command: the probe runs exactly one command per verdict, and a suite /// that tests stale artifacts is the #1 way a mutation matrix lies. command: Vec, + /// Optional: name a shipped harness (`forge`, `cargo`) and its known-good `proof` + /// and `fail-pattern` are used. Both scrape a harness's OUTPUT FORMAT, which is a + /// property of the harness and not of any repo — so authoring them per campaign + /// re-derives the same two mistakes every time (see HARNESSES). + #[serde(default)] + harness: Option, /// Proof-of-run regex over the suite's combined stdout+stderr. Needs two capture /// groups: passed count, failed count. Multiple matches sum (cargo prints one result /// line per test binary). No match anywhere = the suite did not provably run. - proof: String, + /// Required unless `harness` supplies it; given here it overrides the harness. + #[serde(default)] + proof: Option, /// Optional: one capture group extracting a failing test's name, for `killedBy`. - #[serde(rename = "fail-pattern")] + /// Given here it overrides the harness's. + #[serde(rename = "fail-pattern", default)] fail_pattern: Option, /// Per-run wall clock limit. A hung suite is NO-RUN, not a hung campaign. #[serde(rename = "timeout-secs", default = "default_timeout")] @@ -63,6 +72,95 @@ fn default_timeout() -> u64 { 1800 } +// -------------------------------------------------------------- harnesses ---- + +/// A harness's own output format, scraped once here instead of per campaign. +struct Harness { + name: &'static str, + proof: &'static str, + fail_pattern: &'static str, +} + +/// Known-good patterns, each pinned by a test to REAL captured output of that harness +/// (`fixtures/`), green and red. +/// +/// `fail-pattern` is the field campaigns get wrong, in two ways that look nothing alike: +/// +/// 1. TOO WIDE. `'\] (test\w+)\('` against forge matches `[PASS] testFoo(` exactly as +/// readily as `[FAIL: …] testFoo(`, so every mutant is "killed by" whichever passing +/// tests happen to be printed — a killer column that is not evidence of anything. +/// `fail_pattern_defect` now catches this class at baseline. +/// 2. TOO NARROW, AND SILENT. `'\[FAIL.*?\] (test\w+)\('` fixes (1) and then names +/// nobody, because `.` does not match `\n` and forge puts multi-line assertion +/// messages inside the brackets. The verdict stays correct (it is read from the +/// tally, never from this pattern) while the killer column empties out. +/// +/// `(?s)` is NOT the fix for (2): it lets a FAIL entry with no `] name(` shape of its +/// own — forge's invariant failures print the name on a later line, after a `[Sequence]` +/// block — run on into the NEXT entry and name that test instead. Trading a blank cell +/// for a wrong one is the worse half of the trade, so these patterns stay line-anchored. +const HARNESSES: &[Harness] = &[ + Harness { + name: "forge", + // Per test CONTRACT, summed; the trailing per-run line ("3 tests passed, 6 + // failed") is comma-shaped and deliberately not matched twice. + proof: r"(?m)^Suite result: \w+\. (\d+) passed; (\d+) failed;", + // A failing entry's name is preceded by `] ` (single-line message), by `] ` at + // the start of a continuation line (multi-line message), by the tail of a + // continuation line, or by one space (invariant). A `[PASS]`/`[SKIP]` line + // reaches none of those: the alternation admits a leading `[` only for `[FAIL`. + fail_pattern: r"(?m)^(?:(?:(?:\[FAIL|[^\[\n])[^\n]*?)?\] | )(\w+)\([^\n]*\) \((?:gas|runs):", + }, + Harness { + name: "cargo", + // One line per test binary and per doctest run; they sum. + proof: r"(?m)^test result: \w+\. (\d+) passed; (\d+) failed;", + // `(.+)` not `(\S+)`: a doctest's name has spaces in it + // ("src/lib.rs - two (line 2)"). + fail_pattern: r"(?m)^test (.+) \.\.\. FAILED$", + }, +]; + +fn harness_named(name: &str) -> Option<&'static Harness> { + HARNESSES.iter().find(|h| h.name == name) +} + +fn harness_names() -> String { + HARNESSES + .iter() + .map(|h| h.name) + .collect::>() + .join(", ") +} + +/// PURE: the proof and fail patterns a suite config resolves to. +/// +/// An explicit pattern wins over the harness's: a repo whose suite wraps the harness +/// (a build.sh that reformats output) must still be able to say so, and silently +/// ignoring what the config asked for would be its own lie. +fn resolve_patterns(cfg: &SuiteConfig) -> Result<(String, Option), String> { + let harness = match cfg.harness.as_deref() { + None => None, + Some(name) => Some(harness_named(name).ok_or_else(|| { + format!( + "suite.harness {name:?} is not one this build ships (have: {}) — \ + drop it and write suite.proof yourself, or add the harness", + harness_names() + ) + })?), + }; + let proof = cfg + .proof + .clone() + .or_else(|| harness.map(|h| h.proof.to_string())) + .ok_or("suite.proof is required unless suite.harness supplies it")?; + let fail_pattern = cfg + .fail_pattern + .clone() + .or_else(|| harness.map(|h| h.fail_pattern.to_string())); + Ok((proof, fail_pattern)) +} + #[derive(Deserialize)] #[serde(deny_unknown_fields)] struct MutantConfig { @@ -151,6 +249,27 @@ fn classify_suite(output: &str, exit_ok: bool, proof: ®ex::Regex) -> SuiteOut } } +/// How many killers `killedBy` carries. The matrix wants "which test caught this", +/// not a transcript of the whole failing suite. +const KILLED_BY_CAP: usize = 5; + +/// PURE: the distinct names a fail-pattern captures from suite output, first seen first. +/// +/// DISTINCT because harnesses repeat themselves: forge prints every failing test twice, +/// once inline and again under `Failing tests:`, so an undeduped list spends its cap on +/// two names printed twice and drops the rest. +fn captured_names(output: &str, fail_pattern: ®ex::Regex) -> Vec { + let mut names: Vec = Vec::new(); + for cap in fail_pattern.captures_iter(output) { + if let Some(m) = cap.get(1) { + if !names.iter().any(|n| n == m.as_str()) { + names.push(m.as_str().to_string()); + } + } + } + names +} + /// PURE: a mutant's verdict from its suite outcome. /// /// KILLED on failed > 0 OR a non-zero exit with proof present: a harness that proves it @@ -177,11 +296,9 @@ fn mutant_verdict(outcome: SuiteOutcome, fail_pattern: Option<®ex::Regex>) -> if failed > 0 || !exit_ok { let killed_by = fail_pattern .map(|re| { - re.captures_iter(&output) - .filter_map(|c| c.get(1)) - .map(|m| m.as_str().to_string()) - .take(5) - .collect() + let mut names = captured_names(&output, re); + names.truncate(KILLED_BY_CAP); + names }) .unwrap_or_default(); Verdict::Killed { killed_by } @@ -227,6 +344,34 @@ fn baseline_defect(outcome: &SuiteOutcome) -> Option { } } +/// PURE: why a fail-pattern cannot be trusted, or None if it is sound. +/// +/// The baseline already runs, and by the time this is asked `baseline_defect` has +/// established that ZERO tests failed there. So anything the fail-pattern captures out +/// of the baseline's own output is the name of a PASSING test — the pattern matches +/// result lines it must not, and every `killedBy` it goes on to produce credits tests +/// that cannot possibly have killed anything. +/// +/// This is not a near-miss worth a warning. `[PASS] testFoo(` and `[FAIL: …] testFoo(` +/// differ by a few characters, and a matrix built on the wide pattern reads exactly like +/// a correct one — a pure-constant assertion appeared as the killer of a guard mutant in +/// the incident this check exists for. Abort, the way a red baseline aborts. +fn fail_pattern_defect(baseline_output: &str, fail_pattern: ®ex::Regex) -> Option { + let names = captured_names(baseline_output, fail_pattern); + if names.is_empty() { + return None; + } + let shown = names.len().min(KILLED_BY_CAP); + Some(format!( + "suite.fail-pattern matches the GREEN baseline's own output: it captured {} \ + name(s) where nothing failed, so it matches PASSING result lines and every \ + killedBy it produced would be wrong. Captured: {}{}", + names.len(), + names[..shown].join(", "), + if names.len() > shown { ", ..." } else { "" } + )) +} + /// PURE: exit code from the pass's verdicts (baseline defects exit earlier, as 2). fn exit_code(verdicts: &[Verdict]) -> i32 { if verdicts.iter().all(|v| matches!(v, Verdict::Killed { .. })) { @@ -411,12 +556,9 @@ MUTANTS FILE (TOML) # here (wrapper script is fine): the probe runs # exactly this per verdict, and a suite that tests # stale artifacts is the #1 way a matrix lies. - proof = '(\d+) passed; (\d+) failed' - # 2 capture groups: passed, failed — read from the - # suite's own tally. Multiple matches SUM (cargo - # prints one line per test binary). No match = - # the suite did not provably run. - fail-pattern = 'test (\S+) \.\.\. FAILED' # optional: 1 group naming a killer + harness = "forge" # prefer this to hand-written patterns: it supplies + # a known-good proof + fail-pattern for that + # harness's output format (see HARNESSES below). timeout-secs = 1800 # optional; the suite's process group is killed [[mutants]] @@ -425,6 +567,31 @@ MUTANTS FILE (TOML) target = "if !ok {" # must occur EXACTLY once in the file replacement = "if ok {" +HARNESSES + harness = "forge" | "cargo" Supplies proof and fail-pattern. Each shipped + pattern is pinned by a test to real captured + output of that harness, green and red. + + Without a harness, write the two patterns yourself: + + proof = '(\d+) passed; (\d+) failed' + # 2 capture groups: passed, failed — read from the + # suite's own tally. Multiple matches SUM (cargo + # prints one line per test binary). No match = + # the suite did not provably run. REQUIRED unless + # harness supplies it. + fail-pattern = '(?m)^test (.+) \.\.\. FAILED$' # optional: 1 group, the killer + + Either given explicitly overrides the harness's. Both are worth avoiding: a + fail-pattern is easy to get wrong in two opposite directions, and neither + shows up as a wrong VERDICT — only as a wrong or empty killer column. + too wide also matches PASSING result lines, so mutants are credited to + tests that cannot kill them. The probe aborts on this: at the + green baseline the pattern must capture NOTHING. + too narrow matches nothing (e.g. `.` does not cross the newlines in a + multi-line assertion message), and says nothing. The probe + prints "killer NOT NAMED" per kill instead of shipping a blank. + VERDICTS KILLED suite ran and failed (failing tally, or non-zero exit with proof present — the tally is trusted over a lying wrapper exit code, @@ -435,7 +602,8 @@ VERDICTS HARNESS-ERROR the mutant is invalid: target not found exactly once INTEGRITY (enforced) - A red, silent, or zero-test baseline aborts before any probe. Writes are + A red, silent, or zero-test baseline aborts before any probe, and so does a + fail-pattern that matches that baseline's own output. Writes are atomic (temp + rename): no failure mode leaves a file truncated. Every restore is verified byte-exact, and each file is re-checked pristine before the next mutant. Suite output is capped per stream (oldest bytes dropped). @@ -444,7 +612,7 @@ EXIT CODES 0 baseline green and every probed mutant KILLED 1 the pass ran; something SURVIVED, was NO-RUN, or was a HARNESS-ERROR 2 the pass could not run or be trusted (config error, red baseline, - restore failure) + fail-pattern that matches the baseline, restore failure) "#; fn main() { @@ -484,12 +652,13 @@ fn main() { // Validate regexes at load, loudly: a proof with fewer than two capture groups can // never prove a run, which would score every mutant NO-RUN and look like a broken // suite instead of a broken config. - let proof = regex::Regex::new(&cfg.suite.proof) + let (proof_src, fail_pattern_src) = resolve_patterns(&cfg.suite).unwrap_or_else(|e| fail(&e)); + let proof = regex::Regex::new(&proof_src) .unwrap_or_else(|e| fail(&format!("suite.proof is not a valid regex: {e}"))); if proof.captures_len() < 3 { fail("suite.proof needs two capture groups: (passed) and (failed)"); } - let fail_pattern = cfg.suite.fail_pattern.as_deref().map(|p| { + let fail_pattern = fail_pattern_src.as_deref().map(|p| { let re = regex::Regex::new(p) .unwrap_or_else(|e| fail(&format!("suite.fail-pattern is not a valid regex: {e}"))); if re.captures_len() < 2 { @@ -536,10 +705,22 @@ fn main() { if let Some(defect) = baseline_defect(&baseline) { fail(&defect); } - let (base_passed, base_failed) = match &baseline { - SuiteOutcome::Ran { passed, failed, .. } => (*passed, *failed), + let (base_passed, base_failed, base_output) = match &baseline { + SuiteOutcome::Ran { + passed, + failed, + output, + .. + } => (*passed, *failed, output), _ => unreachable!("baseline_defect rejects non-Ran outcomes"), }; + // The green baseline is also the oracle for the fail-pattern: nothing failed, so + // anything it matches here it would go on to misreport as a killer. + if let Some(re) = fail_pattern.as_ref() { + if let Some(defect) = fail_pattern_defect(base_output, re) { + fail(&defect); + } + } println!("baseline: green ({base_passed} passed)"); let mut reports: Vec = Vec::new(); @@ -595,6 +776,18 @@ fn main() { Verdict::Killed { killed_by } if !killed_by.is_empty() => { format!("{} — killed by: {}", verdict.label(), killed_by.join(", ")) } + // The verdict is sound (it comes from the tally) but the killer column is + // blank, which is the OTHER way a fail-pattern fails: too narrow, and + // silent about it. The baseline check cannot see this one — a pattern that + // matches nothing matches nothing at baseline too — so say it here rather + // than let a matrix ship with no killers in it. + Verdict::Killed { killed_by } if killed_by.is_empty() && fail_pattern.is_some() => { + format!( + "{} — killer NOT NAMED: suite.fail-pattern matched nothing in this \ + mutant's failing output", + verdict.label() + ) + } Verdict::NoRun { detail } | Verdict::HarnessError { detail } => { format!("{} — {}", verdict.label(), detail) } @@ -844,4 +1037,246 @@ mod tests { assert_eq!(tail("abc\ndef", 4), " def"); assert_eq!(tail("ab", 4), "ab"); } + + // ------------------------------------------------- shipped harnesses ---- + // + // Real captured output, verbatim, from a throwaway project holding the shapes a + // shipped pattern has to survive: a PASS line, a single-line [FAIL: …], a + // multi-line assertEq message (generated Solidity source — the shape that broke + // the pattern in the incident), a fuzz counterexample with `]` inside the + // message, a custom-error revert, a snake_case test name, an invariant failure + // (name on a later line after a [Sequence] block), and forge's `Failing tests:` + // recap, which prints every failure a second time. + // + // forge-1.7.1-* forge 1.7.1, `forge test --offline [--fuzz-seed 1]` + // forge-1.0.0-nightly-* forge 1.0.0-nightly, same, `--color never` + // cargo-1.95.0-red cargo 1.95.0, `cargo test --no-fail-fast --color never` + // cargo-1.95.0-green the same, filtered to the passing test + // + // Two forge versions because a pattern that only reads the newest build is not a + // known-good pattern for the org's repos. + const FORGE_GREEN: &str = include_str!("../fixtures/forge-1.7.1-green.txt"); + const FORGE_RED: &str = include_str!("../fixtures/forge-1.7.1-red.txt"); + const FORGE_OLD_RED: &str = include_str!("../fixtures/forge-1.0.0-nightly-red.txt"); + const CARGO_GREEN: &str = include_str!("../fixtures/cargo-1.95.0-green.txt"); + const CARGO_RED: &str = include_str!("../fixtures/cargo-1.95.0-red.txt"); + + fn shipped(name: &str) -> (regex::Regex, regex::Regex) { + let h = harness_named(name).expect("shipped harness"); + ( + regex::Regex::new(h.proof).expect("proof compiles"), + regex::Regex::new(h.fail_pattern).expect("fail-pattern compiles"), + ) + } + + /// Every failure in both red forge fixtures, in the order forge first prints them. + const FORGE_KILLERS: &[&str] = &[ + "testGeneratedSourceMatchesSnapshot", // multi-line assertEq message + "testSingleLineFailure", + "testCustomErrorRevertIsUncaught", + "testFuzz_BoundedIsAlwaysSmall", // counterexample with `]` inside the message + "testPlainRevert", + "test_snake_case_name_fails", // snake_case, not just `test\w+` + "invariant_NeverIncrements", // name on its own line, after a [Sequence] block + ]; + + #[test] + fn shipped_forge_proof_reads_forges_own_tally() { + let (proof, _) = shipped("forge"); + // "4 tests passed, 7 failed" per forge's own run summary, reached by summing + // the per-contract `Suite result:` lines — and NOT double-counted off that + // summary line, which is comma-shaped. + for (fixture, label) in [(FORGE_RED, "1.7.1"), (FORGE_OLD_RED, "1.0.0-nightly")] { + match classify_suite(fixture, false, &proof) { + SuiteOutcome::Ran { passed, failed, .. } => { + assert_eq!((passed, failed), (4, 7), "forge {label}"); + } + other => panic!("forge {label}: expected Ran, got {other:?}"), + } + } + match classify_suite(FORGE_GREEN, true, &proof) { + SuiteOutcome::Ran { passed, failed, .. } => assert_eq!((passed, failed), (2, 0)), + other => panic!("expected Ran, got {other:?}"), + } + } + + #[test] + fn shipped_forge_fail_pattern_names_every_failure_and_no_passing_test() { + let (_, fp) = shipped("forge"); + for (fixture, label) in [(FORGE_RED, "1.7.1"), (FORGE_OLD_RED, "1.0.0-nightly")] { + assert_eq!( + captured_names(fixture, &fp), + FORGE_KILLERS, + "forge {label}: every failing test, once each — forge prints them twice, \ + and the run's own summary says 7" + ); + } + // …and the green run, which is nothing but [PASS] lines, yields none. + assert!(captured_names(FORGE_GREEN, &fp).is_empty()); + assert!(fail_pattern_defect(FORGE_GREEN, &fp).is_none()); + } + + #[test] + fn shipped_cargo_patterns_read_real_cargo_output() { + let (proof, fp) = shipped("cargo"); + match classify_suite(CARGO_RED, false, &proof) { + SuiteOutcome::Ran { passed, failed, .. } => { + // lib 1+1, integration 1+1, doctest 1+0 — the tallies sum across + // every target, which is why one proof works for a cargo workspace. + assert_eq!((passed, failed), (3, 2)); + } + other => panic!("expected Ran, got {other:?}"), + } + assert_eq!( + captured_names(CARGO_RED, &fp), + vec!["tests::unit_fails_multiline", "integration_fails"] + ); + match classify_suite(CARGO_GREEN, true, &proof) { + SuiteOutcome::Ran { passed, failed, .. } => assert_eq!((passed, failed), (1, 0)), + other => panic!("expected Ran, got {other:?}"), + } + assert!(fail_pattern_defect(CARGO_GREEN, &fp).is_none()); + } + + #[test] + fn the_incidents_wide_pattern_is_caught_by_the_green_baseline() { + // `'\] (test\w+)\('` — the pattern the campaign actually shipped. It matches + // `[PASS] testFoo(` as readily as `[FAIL: …] testFoo(`, and the green + // baseline is where that is provable: nothing failed, so anything captured + // here is a passing test. + let wide = regex::Regex::new(r"\] (test\w+)\(").unwrap(); + let defect = fail_pattern_defect(FORGE_GREEN, &wide).expect("must be rejected"); + assert!(defect.contains("testHeadGenesisIsNotZero"), "{defect}"); + assert!(defect.contains("testAppliedIsIdempotent"), "{defect}"); + } + + #[test] + fn a_sound_fail_pattern_captures_nothing_at_a_green_baseline() { + let fp = regex::Regex::new(r"(?m)^(\S+) \.\.\. FAILED$").unwrap(); + assert!(fail_pattern_defect("guard_test ... ok\n1 passed | 0 failed", &fp).is_none()); + } + + #[test] + fn the_s_flag_fixes_multiline_messages_and_then_misattributes_invariants() { + // Issue #13 proposed `(?s)` as the one-line fix, flagged as unverified. Both + // halves of that, against real forge output: + // + // It IS the reason the narrow pattern names nobody — `.` stops at the + // newlines inside a multi-line assertEq message, so `.*?` never reaches `]`. + let narrow = regex::Regex::new(r"\[FAIL.*?\] (test\w+)\(").unwrap(); + assert!(!captured_names(FORGE_RED, &narrow) + .iter() + .any(|n| n == "testGeneratedSourceMatchesSnapshot")); + let dotall = regex::Regex::new(r"(?s)\[FAIL.*?\] (test\w+)\(").unwrap(); + assert!(captured_names(FORGE_RED, &dotall) + .iter() + .any(|n| n == "testGeneratedSourceMatchesSnapshot")); + + // And it trades that blank for a WRONG cell, which is the worse half. A forge + // invariant failure has no `] name(` of its own — the name is on a later line, + // after a [Sequence] block — so `(?s)` runs on past the end of that entry and + // stops at the next `] name(` in the output. When the next one belongs to a + // [PASS] line, the pattern names a test that PASSED under the mutant: defect 1 + // again, arrived at from the opposite direction. + let dotall_any = regex::Regex::new(r"(?s)\[FAIL.*?\] (\w+)\(").unwrap(); + for (fixture, label) in [(FORGE_RED, "1.7.1"), (FORGE_OLD_RED, "1.0.0-nightly")] { + let names = captured_names(fixture, &dotall_any); + assert!( + !names.iter().any(|n| n == "invariant_NeverIncrements"), + "forge {label}: (?s) drops the invariant: {names:?}" + ); + assert!( + names.iter().any(|n| n == "testCounterStartsAtZero"), + "forge {label}: (?s) names a PASSING test: {names:?}" + ); + } + // The shipped pattern is line-anchored instead: all seven failures, no passers. + let (_, fp) = shipped("forge"); + assert_eq!(captured_names(FORGE_OLD_RED, &fp), FORGE_KILLERS); + } + + #[test] + fn killed_by_is_distinct_and_capped() { + let fp = regex::Regex::new(r"(?m)^(\S+) \.\.\. FAILED$").unwrap(); + let mut out = String::new(); + for i in 0..8 { + // Each name printed twice, the way forge repeats its failures. + out.push_str(&format!("t{i} ... FAILED\nt{i} ... FAILED\n")); + } + out.push_str("0 passed | 8 failed"); + let outcome = classify_suite(&out, false, &proof()); + match mutant_verdict(outcome, Some(&fp)) { + Verdict::Killed { killed_by } => { + assert_eq!(killed_by, vec!["t0", "t1", "t2", "t3", "t4"]); + } + other => panic!("expected Killed, got {other:?}"), + } + } + + fn suite_config(toml_body: &str) -> SuiteConfig { + let cfg: Config = toml::from_str(&format!( + "[suite]\nroot = \".\"\ncommand = [\"sh\"]\n{toml_body}\n" + )) + .expect("config parses"); + cfg.suite + } + + #[test] + fn a_named_harness_supplies_both_patterns() { + let (proof, fp) = resolve_patterns(&suite_config(r#"harness = "forge""#)).unwrap(); + let h = harness_named("forge").unwrap(); + assert_eq!(proof, h.proof); + assert_eq!(fp.as_deref(), Some(h.fail_pattern)); + } + + #[test] + fn explicit_patterns_override_the_harness() { + // A suite that wraps or reformats its harness's output must still be able to + // say so; silently using the harness's pattern would be its own wrong matrix. + let cfg = suite_config( + "harness = \"forge\"\nproof = '(\\d+) ok (\\d+) bad'\nfail-pattern = 'X(\\w+)'", + ); + let (proof, fp) = resolve_patterns(&cfg).unwrap(); + assert_eq!(proof, r"(\d+) ok (\d+) bad"); + assert_eq!(fp.as_deref(), Some(r"X(\w+)")); + } + + #[test] + fn an_unknown_harness_names_the_ones_that_exist() { + let err = resolve_patterns(&suite_config(r#"harness = "jest""#)).unwrap_err(); + assert!(err.contains("jest"), "{err}"); + assert!(err.contains("forge") && err.contains("cargo"), "{err}"); + } + + #[test] + fn without_a_harness_proof_is_still_required() { + let err = resolve_patterns(&suite_config("")).unwrap_err(); + assert!(err.contains("suite.proof"), "{err}"); + // …and a hand-written proof alone is enough, with no killer attribution. + let (proof, fp) = resolve_patterns(&suite_config(r"proof = '(\d+)/(\d+)'")).unwrap(); + assert_eq!(proof, r"(\d+)/(\d+)"); + assert_eq!(fp, None); + } + + #[test] + fn every_shipped_pattern_compiles_with_the_groups_the_probe_reads() { + for h in HARNESSES { + let proof = regex::Regex::new(h.proof) + .unwrap_or_else(|e| panic!("{}: proof does not compile: {e}", h.name)); + assert!( + proof.captures_len() >= 3, + "{}: proof needs (passed) and (failed)", + h.name + ); + let fp = regex::Regex::new(h.fail_pattern) + .unwrap_or_else(|e| panic!("{}: fail-pattern does not compile: {e}", h.name)); + assert_eq!( + fp.captures_len(), + 2, + "{}: fail-pattern needs exactly one group — the probe reads group 1, so a \ + second group would be silently ignored", + h.name + ); + } + } } diff --git a/mutation-probe-rs/tests/toy.rs b/mutation-probe-rs/tests/toy.rs index 161de70..1b08b01 100644 --- a/mutation-probe-rs/tests/toy.rs +++ b/mutation-probe-rs/tests/toy.rs @@ -268,6 +268,153 @@ replacement = "GUARD off" let _ = std::fs::remove_dir_all(&dir); } +#[test] +fn a_fail_pattern_matching_the_baseline_aborts_before_probing() { + // The toy suite prints ` ... ok` for passing tests, so a pattern that reads + // result lines without discriminating on the RESULT captures them at the green + // baseline — the shape of the incident, where `\] (test\w+)\(` matched forge's + // `[PASS]` lines. Nothing failed, so those captures can only be passing tests. + let dir = unique_dir("widepattern"); + let code = "GUARD on\nCAP 10\nMODE strict\n"; + toy(&dir, code); + let config = format!( + r#" +[suite] +root = "." +command = ["sh", "check.sh"] +proof = '(\d+) passed \| (\d+) failed' +fail-pattern = '(\S+) \.\.\. ' +timeout-secs = 60 +{ALL_FOUR} +"# + ); + let path = dir.join("mutants.toml"); + std::fs::write(&path, config).unwrap(); + let (exit, out, report) = run(&path, &[]); + assert_eq!( + exit, 2, + "an unusable fail-pattern is an abort; output:\n{out}" + ); + assert!( + out.contains("fail-pattern matches the GREEN baseline"), + "the abort names the fail-pattern:\n{out}" + ); + assert!( + out.contains("guard_test") && out.contains("cap_test"), + "the abort shows what it captured at baseline:\n{out}" + ); + assert_eq!( + report, + serde_json::Value::Null, + "no report on an aborted pass" + ); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn a_fail_pattern_that_names_nobody_says_so_per_kill() { + // The other direction: a pattern that matches nothing is invisible to the baseline + // check (it matches nothing there either) and leaves a KILLED row with no killer. + // The verdict is still read from the tally, so it stays correct. + let dir = unique_dir("silentpattern"); + toy(&dir, "GUARD on\nCAP 10\nMODE strict\n"); + let config = format!( + r#" +[suite] +root = "." +command = ["sh", "check.sh"] +proof = '(\d+) passed \| (\d+) failed' +fail-pattern = 'NOTHING MATCHES (\w+)' +timeout-secs = 60 +{} +"#, + r#" +[[mutants]] +name = "M-kill guard off" +file = "code.txt" +target = "GUARD on" +replacement = "GUARD off" +"# + ); + let path = dir.join("mutants.toml"); + std::fs::write(&path, config).unwrap(); + let (exit, out, report) = run(&path, &[]); + assert_eq!(exit, 0, "still a kill; output:\n{out}"); + assert_eq!(report["mutants"][0]["verdict"].as_str(), Some("KILLED")); + assert!( + out.contains("killer NOT NAMED"), + "a blank killer column must announce itself:\n{out}" + ); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn a_named_harness_replaces_the_hand_written_patterns() { + // The toy suite emits cargo-shaped result lines, so `harness = "cargo"` alone — + // no proof, no fail-pattern — must drive a full pass and name the killer. + let dir = unique_dir("harness"); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(dir.join("code.txt"), "GUARD on\nCAP 10\nMODE strict\n").unwrap(); + std::fs::write( + dir.join("check.sh"), + r#" +p=0; f=0 +if grep -q "GUARD on" code.txt; then echo "test guard_test ... ok"; p=$((p+1)); else echo "test guard_test ... FAILED"; f=$((f+1)); fi +echo "test cap_test ... ok"; p=$((p+1)) +if [ "$f" -eq 0 ]; then r=ok; else r=FAILED; fi +echo "test result: $r. $p passed; $f failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s" +[ "$f" -eq 0 ] || exit 101 +"#, + ) + .unwrap(); + let config = r#" +[suite] +harness = "cargo" +root = "." +command = ["sh", "check.sh"] +timeout-secs = 60 + +[[mutants]] +name = "M-kill guard off" +file = "code.txt" +target = "GUARD on" +replacement = "GUARD off" +"#; + let path = dir.join("mutants.toml"); + std::fs::write(&path, config).unwrap(); + let (exit, out, report) = run(&path, &[]); + assert_eq!(exit, 0, "an all-killed pass exits 0; output:\n{out}"); + assert_eq!(report["baseline"]["passed"].as_u64(), Some(2)); + assert_eq!( + report["mutants"][0]["killed_by"][0].as_str(), + Some("guard_test"), + "the shipped cargo pattern names the killer; output:\n{out}" + ); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn an_unknown_harness_is_a_config_abort() { + let dir = unique_dir("badharness"); + toy(&dir, "GUARD on\nCAP 10\nMODE strict\n"); + let config = format!( + r#" +[suite] +harness = "pytest" +root = "." +command = ["sh", "check.sh"] +timeout-secs = 60 +{ALL_FOUR} +"# + ); + let path = dir.join("mutants.toml"); + std::fs::write(&path, config).unwrap(); + let (exit, out, _) = run(&path, &[]); + assert_eq!(exit, 2, "output:\n{out}"); + assert!(out.contains("pytest"), "the abort names the value:\n{out}"); + let _ = std::fs::remove_dir_all(&dir); +} + #[test] fn only_filter_narrows_the_pass() { let dir = unique_dir("only"); From 0726e31cac2b3576afbbe968171b1fe0dd836f19 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 16 Aug 2026 16:51:47 +0000 Subject: [PATCH 2/3] Cover the doctest name shape and the no-fail-pattern case MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The cargo fixture had no FAILING doctest, so nothing discriminated `(.+)` from `(\S+)` in the cargo fail-pattern — the comment claimed a property the tests did not hold it to. And no end-to-end run killed a mutant with no fail-pattern configured at all, so the guard on the 'killer NOT NAMED' line was free. Co-Authored-By: Claude Opus 5 (1M context) --- .../fixtures/cargo-1.95.0-red.txt | 38 +++++++++++++++---- mutation-probe-rs/src/main.rs | 12 ++++-- mutation-probe-rs/tests/toy.rs | 32 ++++++++++++++++ 3 files changed, 71 insertions(+), 11 deletions(-) diff --git a/mutation-probe-rs/fixtures/cargo-1.95.0-red.txt b/mutation-probe-rs/fixtures/cargo-1.95.0-red.txt index 9ff842e..173a147 100644 --- a/mutation-probe-rs/fixtures/cargo-1.95.0-red.txt +++ b/mutation-probe-rs/fixtures/cargo-1.95.0-red.txt @@ -1,4 +1,5 @@ - Finished `test` profile [unoptimized + debuginfo] target(s) in 0.02s + Compiling cargo-toy v0.0.0 (/home/gildlab/code/amt-issue-13-forge-evidence/cargo-toy) + Finished `test` profile [unoptimized + debuginfo] target(s) in 0.18s Running unittests src/lib.rs (target/debug/deps/cargo_toy-9851a04534560ea2) running 2 tests @@ -9,7 +10,7 @@ failures: ---- tests::unit_fails_multiline stdout ---- -thread 'tests::unit_fails_multiline' (228965) panicked at src/lib.rs:18:9: +thread 'tests::unit_fails_multiline' (284639) panicked at src/lib.rs:27:9: assertion `left == right` failed: generated source differs: line one line two @@ -21,7 +22,7 @@ note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace failures: tests::unit_fails_multiline -test result: FAILED. 1 passed; 1 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.02s +test result: FAILED. 1 passed; 1 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s error: test failed, to rerun pass `--lib` Running tests/integration.rs (target/debug/deps/integration-4de883e90258a294) @@ -34,7 +35,7 @@ failures: ---- integration_fails stdout ---- -thread 'integration_fails' (228968) panicked at tests/integration.rs:8:5: +thread 'integration_fails' (284642) panicked at tests/integration.rs:8:5: assertion `left == right` failed left: 1 right: 2 @@ -44,16 +45,37 @@ note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace failures: integration_fails -test result: FAILED. 1 passed; 1 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.01s +test result: FAILED. 1 passed; 1 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s error: test failed, to rerun pass `--test integration` Doc-tests cargo_toy -running 1 test +running 2 tests test src/lib.rs - two (line 2) ... ok +test src/lib.rs - three (line 11) ... FAILED + +failures: + +---- src/lib.rs - three (line 11) stdout ---- +Test executable failed (exit status: 101). + +stderr: + +thread 'main' (284702) panicked at src/lib.rs:5:1: +assertion `left == right` failed + left: 3 + right: 4 +note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace + + + +failures: + src/lib.rs - three (line 11) -test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.47s +test result: FAILED. 1 passed; 1 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.12s -error: 2 targets failed: +error: doctest failed, to rerun pass `--doc` +error: 3 targets failed: `--lib` `--test integration` + `--doc` diff --git a/mutation-probe-rs/src/main.rs b/mutation-probe-rs/src/main.rs index 6ef63b8..b477cfe 100644 --- a/mutation-probe-rs/src/main.rs +++ b/mutation-probe-rs/src/main.rs @@ -1121,15 +1121,21 @@ mod tests { let (proof, fp) = shipped("cargo"); match classify_suite(CARGO_RED, false, &proof) { SuiteOutcome::Ran { passed, failed, .. } => { - // lib 1+1, integration 1+1, doctest 1+0 — the tallies sum across + // lib 1+1, integration 1+1, doctests 1+1 — the tallies sum across // every target, which is why one proof works for a cargo workspace. - assert_eq!((passed, failed), (3, 2)); + assert_eq!((passed, failed), (3, 3)); } other => panic!("expected Ran, got {other:?}"), } assert_eq!( captured_names(CARGO_RED, &fp), - vec!["tests::unit_fails_multiline", "integration_fails"] + vec![ + "tests::unit_fails_multiline", + "integration_fails", + // A doctest's libtest name contains spaces, so `(\S+)` would capture + // "src/lib.rs" and name a test that does not exist. + "src/lib.rs - three (line 11)", + ] ); match classify_suite(CARGO_GREEN, true, &proof) { SuiteOutcome::Ran { passed, failed, .. } => assert_eq!((passed, failed), (1, 0)), diff --git a/mutation-probe-rs/tests/toy.rs b/mutation-probe-rs/tests/toy.rs index 1b08b01..ac43bb3 100644 --- a/mutation-probe-rs/tests/toy.rs +++ b/mutation-probe-rs/tests/toy.rs @@ -348,6 +348,38 @@ replacement = "GUARD off" let _ = std::fs::remove_dir_all(&dir); } +#[test] +fn a_pass_with_no_fail_pattern_at_all_is_not_nagged() { + // fail-pattern is optional. A config that never asked for killer attribution has + // no blank column to complain about, and saying so per kill would train the + // reader to ignore the line that matters when a pattern IS configured. + let dir = unique_dir("nopattern"); + toy(&dir, "GUARD on\nCAP 10\nMODE strict\n"); + let config = r#" +[suite] +root = "." +command = ["sh", "check.sh"] +proof = '(\d+) passed \| (\d+) failed' +timeout-secs = 60 + +[[mutants]] +name = "M-kill guard off" +file = "code.txt" +target = "GUARD on" +replacement = "GUARD off" +"#; + let path = dir.join("mutants.toml"); + std::fs::write(&path, config).unwrap(); + let (exit, out, report) = run(&path, &[]); + assert_eq!(exit, 0, "output:\n{out}"); + assert_eq!(report["mutants"][0]["verdict"].as_str(), Some("KILLED")); + assert!( + !out.contains("NOT NAMED"), + "nothing was asked for, so nothing is missing:\n{out}" + ); + let _ = std::fs::remove_dir_all(&dir); +} + #[test] fn a_named_harness_replaces_the_hand_written_patterns() { // The toy suite emits cargo-shaped result lines, so `harness = "cargo"` alone — From b2d7e41057a8c3dea9464f299b6c3f336551a7ff Mon Sep 17 00:00:00 2001 From: David Meister Date: Sun, 16 Aug 2026 17:01:40 +0000 Subject: [PATCH 3/3] Pin the forge pattern's trailing metrics anchor to real -vvv output MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The mutation pass survived a mutant that deletes `\([^\n]*\) \((?:gas|runs):` from the shipped forge fail-pattern: nothing in the fixtures discriminated a name on a RESULT line from any other `] name(` in forge's output. Real output has one, and campaigns hit it routinely: at `-vvv` forge prints call traces, and a reverting contract's custom error comes out as `└─ ← [Revert] Custom(1, 2)`. Without the anchor the pattern captures `Custom` and names it as the test that killed the mutant. Same red run re-captured at `-vvv` on both pinned forge versions, and the existing test strengthened to walk all four fixtures. Co-Authored-By: Claude Opus 5 (1M context) --- .../forge-1.0.0-nightly-red-traces.txt | 146 +++++++++++++++ .../fixtures/forge-1.7.1-red-traces.txt | 177 ++++++++++++++++++ mutation-probe-rs/src/main.rs | 27 ++- 3 files changed, 347 insertions(+), 3 deletions(-) create mode 100644 mutation-probe-rs/fixtures/forge-1.0.0-nightly-red-traces.txt create mode 100644 mutation-probe-rs/fixtures/forge-1.7.1-red-traces.txt diff --git a/mutation-probe-rs/fixtures/forge-1.0.0-nightly-red-traces.txt b/mutation-probe-rs/fixtures/forge-1.0.0-nightly-red-traces.txt new file mode 100644 index 0000000..7995f13 --- /dev/null +++ b/mutation-probe-rs/fixtures/forge-1.0.0-nightly-red-traces.txt @@ -0,0 +1,146 @@ +Warning: This is a nightly build of Foundry. It is recommended to use the latest stable version. Visit https://book.getfoundry.sh/announcements for more information. +To mute this warning set `FOUNDRY_DISABLE_NIGHTLY_WARNING` in your environment. + +Compiling 22 files with Solc 0.8.25 +Solc 0.8.25 finished in 653.36ms +Compiler run successful! +proptest: Saving this and future failures in cache/fuzz/failures +proptest: If this test was run on a CI system, you may wish to add the following line to your copy of the file. (You may need to create it.) +cc 8b5dede949389349e6efb74f7812838e6847e5ecccff479e0cb60fa9bbb572bc + +Ran 4 tests for test/Evidence.t.sol:EvidenceTest +[PASS] testAppliedIsIdempotent() (gas: 266) +[FAIL: assertion failed: // SPDX-License-Identifier: MIT +pragma solidity =0.8.25; + +library LibGenerated { + bytes32 constant HEAD = bytes32(uint256(1)); +} + != // SPDX-License-Identifier: MIT +pragma solidity =0.8.25; + +library LibGenerated { + bytes32 constant HEAD = bytes32(uint256(2)); +} +] testGeneratedSourceMatchesSnapshot() (gas: 5529) +Traces: + [5529] EvidenceTest::testGeneratedSourceMatchesSnapshot() + ├─ [0] VM::assertEq("// SPDX-License-Identifier: MIT\npragma solidity =0.8.25;\n\nlibrary LibGenerated {\n bytes32 constant HEAD = bytes32(uint256(1));\n}\n", "// SPDX-License-Identifier: MIT\npragma solidity =0.8.25;\n\nlibrary LibGenerated {\n bytes32 constant HEAD = bytes32(uint256(2));\n}\n") [staticcall] + │ └─ ← [Revert] assertion failed: // SPDX-License-Identifier: MIT +pragma solidity =0.8.25; + +library LibGenerated { + bytes32 constant HEAD = bytes32(uint256(1)); +} + != // SPDX-License-Identifier: MIT +pragma solidity =0.8.25; + +library LibGenerated { + bytes32 constant HEAD = bytes32(uint256(2)); +} + + └─ ← [Revert] assertion failed: // SPDX-License-Identifier: MIT +pragma solidity =0.8.25; + +library LibGenerated { + bytes32 constant HEAD = bytes32(uint256(1)); +} + != // SPDX-License-Identifier: MIT +pragma solidity =0.8.25; + +library LibGenerated { + bytes32 constant HEAD = bytes32(uint256(2)); +} + + +[PASS] testHeadGenesisIsNotZero() (gas: 212) +[FAIL: assertion failed: 1 != 2] testSingleLineFailure() (gas: 3333) +Traces: + [3333] EvidenceTest::testSingleLineFailure() + ├─ [0] VM::assertEq(1, 2) [staticcall] + │ └─ ← [Revert] assertion failed: 1 != 2 + └─ ← [Revert] assertion failed: 1 != 2 + +Suite result: FAILED. 2 passed; 2 failed; 0 skipped; finished in 1.77ms (1.39ms CPU time) + +Ran 5 tests for test/Shapes.t.sol:ShapesTest +[FAIL: Custom(1, 2)] testCustomErrorRevertIsUncaught() (gas: 5877) +Traces: + [5877] ShapesTest::testCustomErrorRevertIsUncaught() + ├─ [663] Reverter::boom() [staticcall] + │ └─ ← [Revert] Custom(1, 2) + └─ ← [Revert] Custom(1, 2) + +[FAIL: assertion failed: 958 >= 10; counterexample: calldata=0x8054777000000000000000000000000000000000000000000000000000000000000007a6 args=[1958]] testFuzz_BoundedIsAlwaysSmall(uint256) (runs: 1, μ: 747, ~: 747) +Traces: + [3833] ShapesTest::testFuzz_BoundedIsAlwaysSmall(1958) + ├─ [0] VM::assertLt(958, 10) [staticcall] + │ └─ ← [Revert] assertion failed: 958 >= 10 + └─ ← [Revert] assertion failed: 958 >= 10 + +[FAIL: revert: plain string reason] testPlainRevert() (gas: 495) +Traces: + [495] ShapesTest::testPlainRevert() + └─ ← [Revert] revert: plain string reason + +[PASS] testThisOnePasses() (gas: 255) +[FAIL: assertion failed: 3 != 4] test_snake_case_name_fails() (gas: 3355) +Traces: + [3355] ShapesTest::test_snake_case_name_fails() + ├─ [0] VM::assertEq(3, 4) [staticcall] + │ └─ ← [Revert] assertion failed: 3 != 4 + └─ ← [Revert] assertion failed: 3 != 4 + +Suite result: FAILED. 1 passed; 4 failed; 0 skipped; finished in 1.84ms (2.76ms CPU time) + +Ran 2 tests for test/Invariant.t.sol:InvariantTest +[FAIL: assertion failed: 1 != 0] + [Sequence] (original: 1, shrunk: 1) + sender=0x0000000000000000000000000000000000000020 addr=[test/Invariant.t.sol:Counter]0x5615dEB798BB3E4dFa0139dFa1b3D433Cc23b72f calldata=inc() args=[] + invariant_NeverIncrements() (runs: 0, calls: 0, reverts: 0) +Traces: + [22454] Counter::inc() + └─ ← [Stop] + + [10932] InvariantTest::invariant_NeverIncrements() + ├─ [2402] Counter::n() [staticcall] + │ └─ ← [Return] 1 + ├─ [0] VM::assertEq(1, 0) [staticcall] + │ └─ ← [Revert] assertion failed: 1 != 0 + └─ ← [Revert] assertion failed: 1 != 0 + +[PASS] testCounterStartsAtZero() (gas: 7803) +Suite result: FAILED. 1 passed; 1 failed; 0 skipped; finished in 4.54ms (2.54ms CPU time) + +Ran 3 test suites in 13.14ms (8.15ms CPU time): 4 tests passed, 7 failed, 0 skipped (11 total tests) + +Failing tests: +Encountered 2 failing tests in test/Evidence.t.sol:EvidenceTest +[FAIL: assertion failed: // SPDX-License-Identifier: MIT +pragma solidity =0.8.25; + +library LibGenerated { + bytes32 constant HEAD = bytes32(uint256(1)); +} + != // SPDX-License-Identifier: MIT +pragma solidity =0.8.25; + +library LibGenerated { + bytes32 constant HEAD = bytes32(uint256(2)); +} +] testGeneratedSourceMatchesSnapshot() (gas: 5529) +[FAIL: assertion failed: 1 != 2] testSingleLineFailure() (gas: 3333) + +Encountered 1 failing test in test/Invariant.t.sol:InvariantTest +[FAIL: assertion failed: 1 != 0] + [Sequence] (original: 1, shrunk: 1) + sender=0x0000000000000000000000000000000000000020 addr=[test/Invariant.t.sol:Counter]0x5615dEB798BB3E4dFa0139dFa1b3D433Cc23b72f calldata=inc() args=[] + invariant_NeverIncrements() (runs: 0, calls: 0, reverts: 0) + +Encountered 4 failing tests in test/Shapes.t.sol:ShapesTest +[FAIL: Custom(1, 2)] testCustomErrorRevertIsUncaught() (gas: 5877) +[FAIL: assertion failed: 958 >= 10; counterexample: calldata=0x8054777000000000000000000000000000000000000000000000000000000000000007a6 args=[1958]] testFuzz_BoundedIsAlwaysSmall(uint256) (runs: 1, μ: 747, ~: 747) +[FAIL: revert: plain string reason] testPlainRevert() (gas: 495) +[FAIL: assertion failed: 3 != 4] test_snake_case_name_fails() (gas: 3355) + +Encountered a total of 7 failing tests, 4 tests succeeded diff --git a/mutation-probe-rs/fixtures/forge-1.7.1-red-traces.txt b/mutation-probe-rs/fixtures/forge-1.7.1-red-traces.txt new file mode 100644 index 0000000..6aa1ff0 --- /dev/null +++ b/mutation-probe-rs/fixtures/forge-1.7.1-red-traces.txt @@ -0,0 +1,177 @@ +2026-08-16T16:56:58.939193Z ERROR foundry_compilers::cache: error=missing field `preprocessed` at line 1 column 14991 +Compiling 22 files with Solc 0.8.25 +Solc 0.8.25 finished in 634.96ms +Compiler run successful! +2026-08-16T16:56:59.645018Z ERROR forge::runner: Failed to create fuzz failure dir err=failed to create dir "cache/fuzz/failures/ShapesTest": Not a directory (os error 20) + +Ran 4 tests for test/Evidence.t.sol:EvidenceTest +[PASS] testAppliedIsIdempotent() (gas: 266) +{"timestamp":1786899419,"event":"failure","invariant":"invariant_NeverIncrements","target":"test/Invariant.t.sol:InvariantTest","reason":"assertion failed: 1 != 0"} +[FAIL: assertion failed: // SPDX-License-Identifier: MIT +pragma solidity =0.8.25; + +library LibGenerated { + bytes32 constant HEAD = bytes32(uint256(1)); +} + != // SPDX-License-Identifier: MIT +pragma solidity =0.8.25; + +library LibGenerated { + bytes32 constant HEAD = bytes32(uint256(2)); +} +] testGeneratedSourceMatchesSnapshot() (gas: 5529) +Traces: + [5529] EvidenceTest::testGeneratedSourceMatchesSnapshot() + ├─ [0] VM::assertEq("// SPDX-License-Identifier: MIT\npragma solidity =0.8.25;\n\nlibrary LibGenerated {\n bytes32 constant HEAD = bytes32(uint256(1));\n}\n", "// SPDX-License-Identifier: MIT\npragma solidity =0.8.25;\n\nlibrary LibGenerated {\n bytes32 constant HEAD = bytes32(uint256(2));\n}\n") [staticcall] + │ └─ ← [Revert] assertion failed: // SPDX-License-Identifier: MIT +pragma solidity =0.8.25; + +library LibGenerated { + bytes32 constant HEAD = bytes32(uint256(1)); +} + != // SPDX-License-Identifier: MIT +pragma solidity =0.8.25; + +library LibGenerated { + bytes32 constant HEAD = bytes32(uint256(2)); +} + + └─ ← [Revert] assertion failed: // SPDX-License-Identifier: MIT +pragma solidity =0.8.25; + +library LibGenerated { + bytes32 constant HEAD = bytes32(uint256(1)); +} + != // SPDX-License-Identifier: MIT +pragma solidity =0.8.25; + +library LibGenerated { + bytes32 constant HEAD = bytes32(uint256(2)); +} + + +Backtrace: + at VM.assertEq + at EvidenceTest.testGeneratedSourceMatchesSnapshot + +[PASS] testHeadGenesisIsNotZero() (gas: 212) +[FAIL: assertion failed: 1 != 2] testSingleLineFailure() (gas: 3333) +Traces: + [3333] EvidenceTest::testSingleLineFailure() + ├─ [0] VM::assertEq(1, 2) [staticcall] + │ └─ ← [Revert] assertion failed: 1 != 2 + └─ ← [Revert] assertion failed: 1 != 2 + +Backtrace: + at VM.assertEq + at EvidenceTest.testSingleLineFailure + +Suite result: FAILED. 2 passed; 2 failed; 0 skipped; finished in 494.13µs (402.38µs CPU time) + +Ran 5 tests for test/Shapes.t.sol:ShapesTest +[FAIL: Custom(1, 2)] testCustomErrorRevertIsUncaught() (gas: 5877) +Traces: + [5877] ShapesTest::testCustomErrorRevertIsUncaught() + ├─ [663] Reverter::boom() [staticcall] + │ └─ ← [Revert] Custom(1, 2) + └─ ← [Revert] Custom(1, 2) + +Backtrace: + at Reverter.boom + at ShapesTest.testCustomErrorRevertIsUncaught + +[FAIL: assertion failed: 899 >= 10; counterexample: calldata=0x8054777000000000000000000000000000000000000000007f7d0614aa4fead90887bd3b args=[39455740690168716794427587899 [3.945e28]]] testFuzz_BoundedIsAlwaysSmall(uint256) (runs: 1, μ: 747, ~: 747) +Traces: + [3833] ShapesTest::testFuzz_BoundedIsAlwaysSmall(39455740690168716794427587899 [3.945e28]) + ├─ [0] VM::assertLt(899, 10) [staticcall] + │ └─ ← [Revert] assertion failed: 899 >= 10 + └─ ← [Revert] assertion failed: 899 >= 10 + +Backtrace: + at VM.assertLt + at ShapesTest.testFuzz_BoundedIsAlwaysSmall + +[FAIL: plain string reason] testPlainRevert() (gas: 495) +Traces: + [495] ShapesTest::testPlainRevert() + └─ ← [Revert] plain string reason + +Backtrace: + at ShapesTest.testPlainRevert + +[PASS] testThisOnePasses() (gas: 255) +[FAIL: assertion failed: 3 != 4] test_snake_case_name_fails() (gas: 3355) +Traces: + [3355] ShapesTest::test_snake_case_name_fails() + ├─ [0] VM::assertEq(3, 4) [staticcall] + │ └─ ← [Revert] assertion failed: 3 != 4 + └─ ← [Revert] assertion failed: 3 != 4 + +Backtrace: + at VM.assertEq + at ShapesTest.test_snake_case_name_fails + +Suite result: FAILED. 1 passed; 4 failed; 0 skipped; finished in 3.80ms (4.04ms CPU time) + +Ran 2 tests for test/Invariant.t.sol:InvariantTest +[FAIL: assertion failed: 1 != 0] + [Sequence] (original: 1, shrunk: 1) + sender=0x00000000000000000000000000000000000000E6 addr=[test/Invariant.t.sol:Counter]0x5615dEB798BB3E4dFa0139dFa1b3D433Cc23b72f calldata=inc() args=[] + invariant_NeverIncrements() (runs: 0, calls: 0, reverts: 0) + +╭----------+----------+-------+---------+----------╮ +| Contract | Selector | Calls | Reverts | Discards | ++==================================================+ +| Counter | inc | 1 | 0 | 0 | +╰----------+----------+-------+---------+----------╯ + +Traces: + [22454] Counter::inc() + └─ ← [Stop] + + [10932] InvariantTest::invariant_NeverIncrements() + ├─ [2402] Counter::n() [staticcall] + │ └─ ← [Return] 1 + ├─ [0] VM::assertEq(1, 0) [staticcall] + │ └─ ← [Revert] assertion failed: 1 != 0 + └─ ← [Revert] assertion failed: 1 != 0 + +[PASS] testCounterStartsAtZero() (gas: 7803) +Suite result: FAILED. 1 passed; 1 failed; 0 skipped; finished in 72.96ms (72.07ms CPU time) + +Ran 3 test suites in 74.02ms (77.26ms CPU time): 4 tests passed, 7 failed, 0 skipped (11 total tests) + +Failing tests: +Encountered 2 failing tests in test/Evidence.t.sol:EvidenceTest +[FAIL: assertion failed: // SPDX-License-Identifier: MIT +pragma solidity =0.8.25; + +library LibGenerated { + bytes32 constant HEAD = bytes32(uint256(1)); +} + != // SPDX-License-Identifier: MIT +pragma solidity =0.8.25; + +library LibGenerated { + bytes32 constant HEAD = bytes32(uint256(2)); +} +] testGeneratedSourceMatchesSnapshot() (gas: 5529) +[FAIL: assertion failed: 1 != 2] testSingleLineFailure() (gas: 3333) + +Encountered 1 failing test in test/Invariant.t.sol:InvariantTest +[FAIL: assertion failed: 1 != 0] + [Sequence] (original: 1, shrunk: 1) + sender=0x00000000000000000000000000000000000000E6 addr=[test/Invariant.t.sol:Counter]0x5615dEB798BB3E4dFa0139dFa1b3D433Cc23b72f calldata=inc() args=[] + invariant_NeverIncrements() (runs: 0, calls: 0, reverts: 0) + +Encountered 4 failing tests in test/Shapes.t.sol:ShapesTest +[FAIL: Custom(1, 2)] testCustomErrorRevertIsUncaught() (gas: 5877) +[FAIL: assertion failed: 899 >= 10; counterexample: calldata=0x8054777000000000000000000000000000000000000000007f7d0614aa4fead90887bd3b args=[39455740690168716794427587899 [3.945e28]]] testFuzz_BoundedIsAlwaysSmall(uint256) (runs: 1, μ: 747, ~: 747) +[FAIL: plain string reason] testPlainRevert() (gas: 495) +[FAIL: assertion failed: 3 != 4] test_snake_case_name_fails() (gas: 3355) + +Encountered a total of 7 failing tests, 4 tests succeeded + +Tip: Run `forge test --rerun` to retry only the 7 failed tests + +Fuzz seed: 0x1 (use `--fuzz-seed` to reproduce) diff --git a/mutation-probe-rs/src/main.rs b/mutation-probe-rs/src/main.rs index b477cfe..b319c75 100644 --- a/mutation-probe-rs/src/main.rs +++ b/mutation-probe-rs/src/main.rs @@ -109,6 +109,9 @@ const HARNESSES: &[Harness] = &[ // the start of a continuation line (multi-line message), by the tail of a // continuation line, or by one space (invariant). A `[PASS]`/`[SKIP]` line // reaches none of those: the alternation admits a leading `[` only for `[FAIL`. + // The trailing `(args) (gas:|runs:` pins the name to a RESULT line, which is + // what keeps `-vvv` traces out: `← [Revert] Custom(1, 2)` is `] name(` too, and + // without the anchor a reverting contract's error is named as the killer. fail_pattern: r"(?m)^(?:(?:(?:\[FAIL|[^\[\n])[^\n]*?)?\] | )(\w+)\([^\n]*\) \((?:gas|runs):", }, Harness { @@ -1048,8 +1051,14 @@ mod tests { // (name on a later line after a [Sequence] block), and forge's `Failing tests:` // recap, which prints every failure a second time. // + // The `-red-traces` pair is the same run at `-vvv`, which campaigns routinely use + // and which prints call traces the pattern must walk past: `└─ ← [Revert] Custom(1, + // 2)` is a `] name(` with nothing after it, so only the trailing `(args) (gas:|runs:` + // anchor stops a contract's custom error being named as the killing test. + // // forge-1.7.1-* forge 1.7.1, `forge test --offline [--fuzz-seed 1]` // forge-1.0.0-nightly-* forge 1.0.0-nightly, same, `--color never` + // *-red-traces the same red run again with `-vvv` // cargo-1.95.0-red cargo 1.95.0, `cargo test --no-fail-fast --color never` // cargo-1.95.0-green the same, filtered to the passing test // @@ -1058,6 +1067,9 @@ mod tests { const FORGE_GREEN: &str = include_str!("../fixtures/forge-1.7.1-green.txt"); const FORGE_RED: &str = include_str!("../fixtures/forge-1.7.1-red.txt"); const FORGE_OLD_RED: &str = include_str!("../fixtures/forge-1.0.0-nightly-red.txt"); + const FORGE_RED_TRACES: &str = include_str!("../fixtures/forge-1.7.1-red-traces.txt"); + const FORGE_OLD_RED_TRACES: &str = + include_str!("../fixtures/forge-1.0.0-nightly-red-traces.txt"); const CARGO_GREEN: &str = include_str!("../fixtures/cargo-1.95.0-green.txt"); const CARGO_RED: &str = include_str!("../fixtures/cargo-1.95.0-red.txt"); @@ -1103,12 +1115,21 @@ mod tests { #[test] fn shipped_forge_fail_pattern_names_every_failure_and_no_passing_test() { let (_, fp) = shipped("forge"); - for (fixture, label) in [(FORGE_RED, "1.7.1"), (FORGE_OLD_RED, "1.0.0-nightly")] { + // The `-vvv` pair carries a shape the default-verbosity pair cannot: a trace + // line `└─ ← [Revert] Custom(1, 2)`, which is `] name(` with no metrics after + // it. Nothing but the trailing `(args) (gas:|runs:` anchor keeps a reverting + // contract's custom error out of the killer column. + for (fixture, label) in [ + (FORGE_RED, "1.7.1"), + (FORGE_OLD_RED, "1.0.0-nightly"), + (FORGE_RED_TRACES, "1.7.1 -vvv"), + (FORGE_OLD_RED_TRACES, "1.0.0-nightly -vvv"), + ] { assert_eq!( captured_names(fixture, &fp), FORGE_KILLERS, - "forge {label}: every failing test, once each — forge prints them twice, \ - and the run's own summary says 7" + "forge {label}: every failing test, once each, and nothing that is not a \ + test — forge prints each failure twice, and the run's own summary says 7" ); } // …and the green run, which is nothing but [PASS] lines, yields none.