From 75b60b17fa2976856fb1c668829856a62aa6efe3 Mon Sep 17 00:00:00 2001 From: Bruno Garcia Date: Mon, 27 Jul 2026 14:16:10 +0200 Subject: [PATCH 1/2] feat: add parallelization --- README.md | 29 +- src/analyze.rs | 147 +++++- src/db.rs | 24 +- src/lib.rs | 4 +- src/main.rs | 70 ++- src/parallel.rs | 1167 +++++++++++++++++++++++++++++++++++++++++ src/report.rs | 24 +- src/workspace.rs | 372 +++++++++++++ tests/parallel_cli.rs | 269 ++++++++++ 9 files changed, 2049 insertions(+), 57 deletions(-) create mode 100644 src/parallel.rs create mode 100644 src/workspace.rs create mode 100644 tests/parallel_cli.rs diff --git a/README.md b/README.md index d3735f1..899f1a5 100644 --- a/README.md +++ b/README.md @@ -141,9 +141,12 @@ When `--sqlite` is used, the `mutate` command prints a `run_id` that you pass to | `--folder PATH` | `-f` | | Folder containing mutants (alternative to `--sqlite` / `--run-id`). | | `--timeout SECONDS` | `-t` | `300` | Timeout in seconds for each mutant's test run. | | `--jobs N` | `-j` | `0` | Number of parallel jobs passed to the compiler (e.g. `make -j N`). `0` uses the system default. | +| `--parallel N` | `-P` | `1` | Number of mutants to verify concurrently. Values larger than the number of mutants are capped automatically. | +| `--setup-command CMD` | | | Command run once in every isolated parallel worker before its baseline check. Useful for configuring a fresh `build/` directory. | +| `--keep-worktrees` | | | Preserve temporary parallel worker worktrees for debugging instead of removing them. | | `--survival-threshold RATE` | | `0.75` | Maximum acceptable mutant survival rate (e.g. `0.3` = 30%). The run exits with an error if the threshold is exceeded. | | `--min-score RATE` | | | CI gate: fail with a non-zero exit code if the final mutation score (killed / total) is below this value (e.g. `0.8` = 80%). Aggregated across all analyzed folders. When unset, the score is not enforced. | -| `--surviving` | | | Only analyze mutants that survived a previous run. Requires `--run-id`. | +| `--survivors-only` | | | Only analyze mutants that survived a previous run. Requires `--run-id`. | ### Examples @@ -163,7 +166,7 @@ bcore-mutation analyze --sqlite --run-id=1 --file-path="src/net_processing.cpp" **Retry only survivors from a previous run:** ```bash -bcore-mutation analyze --sqlite --run-id=1 --surviving \ +bcore-mutation analyze --sqlite --run-id=1 --survivors-only \ -c "cmake --build build && ./build/test/functional/wallet_test.py" ``` @@ -183,6 +186,28 @@ bcore-mutation analyze --sqlite --run-id=1 -t 120 -j 8 \ -c "cmake --build build && ./build/test/functional/wallet_test.py" ``` +**Verify mutants with three parallel workers:** +```bash +bcore-mutation analyze --sqlite --run-id=1 --parallel 3 --jobs 4 \ + --setup-command "cmake -B build -DENABLE_IPC=OFF && cmake --build build -j4" \ + -c "cmake --build build -j4 && ./build/bin/test_bitcoin" +``` + +Parallel workers are assigned mutants dynamically, so a worker receives the +next pending mutant as soon as it finishes its current one. Each worker uses a +detached Git worktree and its own `build/` directory; the checkout from which +`bcore-mutation` was started is not modified. A custom test command runs from +the worker directory. If the command references a sibling path such as +`../qa-assets`, parallel mode links the matching sibling from the original +checkout's parent into the temporary worker root, so existing Bitcoin Core fuzz +corpus commands keep working without copying the corpus. Fresh worktrees do not +contain an existing build, so provide `--setup-command` unless the test command +configures the build itself. + +`--parallel` controls concurrent mutants while `--jobs` controls parallel jobs +inside each build. For example, `--parallel 3 --jobs 4` can run approximately +12 compiler jobs and also requires disk space for three build trees. + **Set a survival rate threshold:** ```bash bcore-mutation analyze --sqlite --run-id=1 --survival-threshold=0.2 \ diff --git a/src/analyze.rs b/src/analyze.rs index 84fcca5..4a59680 100644 --- a/src/analyze.rs +++ b/src/analyze.rs @@ -1,6 +1,7 @@ use crate::commands::{self, ProjectCommands}; use crate::db::Database; use crate::error::{MutationError, Result}; +use crate::parallel; use crate::project::Project; use crate::report::generate_report; use std::fs; @@ -19,6 +20,24 @@ pub struct ScoreSummary { pub total: u64, } +/// Complete configuration for an analysis run. +pub struct AnalysisOptions { + pub project: Project, + pub folder: Option, + pub command: Option, + pub setup_command: Option, + pub jobs: u32, + pub parallel: usize, + pub timeout_secs: u64, + pub survival_threshold: f64, + pub min_score: Option, + pub sqlite_path: Option, + pub run_id: Option, + pub file_path: Option, + pub survivors_only: bool, + pub keep_worktrees: bool, +} + impl ScoreSummary { /// Mutation score as a fraction in `[0.0, 1.0]` (killed / total). /// Returns `0.0` when no mutants were analyzed. @@ -66,6 +85,7 @@ fn enforce_min_score(summary: ScoreSummary, min_score: Option) -> Result<() Ok(()) } +#[allow(clippy::too_many_arguments, dead_code)] pub async fn run_analysis( project: Project, folder: Option, @@ -79,10 +99,54 @@ pub async fn run_analysis( file_path: Option, survivors_only: bool, ) -> Result<()> { + run_analysis_with_options(AnalysisOptions { + project, + folder, + command, + setup_command: None, + jobs, + parallel: 1, + timeout_secs, + survival_threshold, + min_score, + sqlite_path, + run_id, + file_path, + survivors_only, + keep_worktrees: false, + }) + .await +} + +pub async fn run_analysis_with_options(options: AnalysisOptions) -> Result<()> { + let AnalysisOptions { + project, + folder, + command, + setup_command, + jobs, + parallel, + timeout_secs, + survival_threshold, + min_score, + sqlite_path, + run_id, + file_path, + survivors_only, + keep_worktrees, + } = options; + + if parallel == 0 { + return Err(MutationError::InvalidInput( + "--parallel must be at least 1".to_string(), + )); + } + println!("Analyzing mutants for project: {}", project.db_name()); + let project_commands = commands::for_project(project); // DB-based analysis mode: read mutants from DB and test them. - if let (Some(ref path), Some(rid)) = (sqlite_path.as_ref(), run_id) { + if let (Some(path), Some(rid)) = (sqlite_path.as_ref(), run_id) { let command = command.ok_or_else(|| { MutationError::InvalidInput( "--command is required when using --sqlite with --run_id".to_string(), @@ -91,15 +155,37 @@ pub async fn run_analysis( let db = Database::open(path)?; db.ensure_schema()?; db.seed_projects()?; - let summary = run_db_analysis( - &db, - rid, - &command, - timeout_secs, - file_path.as_deref(), - survivors_only, - ) - .await?; + let summary = if parallel > 1 { + parallel::analyze_database( + &db, + rid, + &command, + setup_command.as_deref(), + project_commands.build_timeout_secs(), + timeout_secs, + file_path.as_deref(), + survivors_only, + parallel, + keep_worktrees, + ) + .await? + } else { + if let Some(setup) = setup_command.as_deref() { + let success = run_command(setup, project_commands.build_timeout_secs()).await?; + if !success { + return Err(MutationError::Command("Setup command failed".to_string())); + } + } + run_db_analysis( + &db, + rid, + &command, + timeout_secs, + file_path.as_deref(), + survivors_only, + ) + .await? + }; return enforce_min_score(summary, min_score); } @@ -111,12 +197,31 @@ pub async fn run_analysis( find_mutation_folders()? }; - let project_commands = commands::for_project(project); + if parallel > 1 { + let summary = parallel::analyze_folders( + folders, + command, + setup_command, + jobs, + timeout_secs, + survival_threshold, + parallel, + keep_worktrees, + project_commands.as_ref(), + ) + .await?; + return enforce_min_score(summary, min_score); + } // When we derive the test command ourselves (no --command), do the one-time // clean build up front rather than once per folder. Each mutant still // triggers an incremental rebuild inside its test command. - if command.is_none() && !folders.is_empty() { + if let Some(setup) = setup_command.as_deref() { + let success = run_command(setup, project_commands.build_timeout_secs()).await?; + if !success { + return Err(MutationError::Command("Setup command failed".to_string())); + } + } else if command.is_none() && !folders.is_empty() { run_build_command(project_commands.as_ref()).await?; } @@ -307,7 +412,7 @@ pub async fn analyze_folder( let test_command = if let Some(cmd) = command { cmd } else { - project_commands.test_command(&target_file_path, jobs)? + project_commands.test_command(target_file_path, jobs)? }; // Get list of mutant files @@ -315,7 +420,7 @@ pub async fn analyze_folder( for entry in fs::read_dir(folder_path)? { let entry = entry?; let path = entry.path(); - if path.is_file() && !path.extension().map_or(true, |ext| ext == "txt") { + if path.is_file() && path.extension().is_some_and(|ext| ext != "txt") { if let Some(name) = path.file_name().and_then(|n| n.to_str()) { mutant_files.push(name.to_string()); } @@ -353,7 +458,7 @@ pub async fn analyze_folder( // Read and apply mutant let mutant_content = fs::read_to_string(&file_path)?; - fs::write(&target_file_path, &mutant_content)?; + fs::write(target_file_path, &mutant_content)?; //println!("Running: {}", test_command); let result = run_command(&test_command, timeout_secs).await?; @@ -367,6 +472,9 @@ pub async fn analyze_folder( } } + // Restore the original file before generating diffs for the report. + restore_file(target_file_path).await?; + // Generate report let score = num_killed as f64 / total_mutants as f64; println!("\nMUTATION SCORE: {:.2}%", score * 100.0); @@ -374,14 +482,11 @@ pub async fn analyze_folder( generate_report( ¬_killed, folder_path.to_str().unwrap(), - &target_file_path, + target_file_path, score, ) .await?; - // Restore the original file - restore_file(&target_file_path).await?; - Ok(ScoreSummary { killed: num_killed, total: total_mutants as u64, @@ -439,7 +544,6 @@ async fn run_command(command: &str, timeout_secs: u64) -> Result { /// Run the test command once against **unmutated** code before analyzing any /// mutants. If it does not pass, abort the whole run with an error. - async fn check_baseline(command: &str, timeout_secs: u64) -> Result<()> { println!("Baseline check: running the test command on unmutated code..."); let passed = run_command(command, timeout_secs).await?; @@ -562,9 +666,8 @@ mod tests { assert!(check_baseline("true", 5).await.is_ok()); // Test a command that fails on unmutated code aborts the run. - let result= check_baseline("exit 200", 5).await; + let result = check_baseline("exit 200", 5).await; assert!(matches!(result, Err(MutationError::InvalidInput(_)))); - } #[test] diff --git a/src/db.rs b/src/db.rs index f1015a3..4329768 100644 --- a/src/db.rs +++ b/src/db.rs @@ -57,6 +57,7 @@ pub struct MutantData { } /// A mutant row read back from the database. +#[derive(Clone, Debug)] pub struct MutantRow { pub id: i64, pub diff: String, @@ -184,7 +185,8 @@ impl Database { let rows: Vec = match (file_path, survivors_only) { (Some(fp), false) => { let mut stmt = self.conn.prepare( - "SELECT id, diff, file_path FROM mutants WHERE run_id = ?1 AND file_path = ?2", + "SELECT id, diff, file_path FROM mutants \ + WHERE run_id = ?1 AND file_path = ?2 ORDER BY id", )?; let rows = stmt .query_map(params![run_id, fp], map_row)? @@ -194,7 +196,7 @@ impl Database { (Some(fp), true) => { let mut stmt = self.conn.prepare( "SELECT id, diff, file_path FROM mutants \ - WHERE run_id = ?1 AND file_path = ?2 AND status = 'survived'", + WHERE run_id = ?1 AND file_path = ?2 AND status = 'survived' ORDER BY id", )?; let rows = stmt .query_map(params![run_id, fp], map_row)? @@ -202,9 +204,9 @@ impl Database { rows } (None, false) => { - let mut stmt = self - .conn - .prepare("SELECT id, diff, file_path FROM mutants WHERE run_id = ?1")?; + let mut stmt = self.conn.prepare( + "SELECT id, diff, file_path FROM mutants WHERE run_id = ?1 ORDER BY id", + )?; let rows = stmt .query_map(params![run_id], map_row)? .collect::>()?; @@ -213,7 +215,7 @@ impl Database { (None, true) => { let mut stmt = self.conn.prepare( "SELECT id, diff, file_path FROM mutants \ - WHERE run_id = ?1 AND status = 'survived'", + WHERE run_id = ?1 AND status = 'survived' ORDER BY id", )?; let rows = stmt .query_map(params![run_id], map_row)? @@ -225,6 +227,16 @@ impl Database { Ok(rows) } + /// Return the commit against which the mutants in `run_id` were generated. + pub fn get_run_commit_hash(&self, run_id: i64) -> Result { + let commit_hash = self.conn.query_row( + "SELECT commit_hash FROM runs WHERE id = ?1", + params![run_id], + |row| row.get(0), + )?; + Ok(commit_hash) + } + /// Update the status and command_to_test for a single mutant. pub fn update_mutant_status(&self, id: i64, status: &str, command: &str) -> Result<()> { self.conn.execute( diff --git a/src/lib.rs b/src/lib.rs index f6555e9..9942536 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -46,14 +46,16 @@ pub mod error; pub mod git_changes; pub mod mutation; pub mod operators; +pub mod parallel; pub mod project; pub mod report; +pub mod workspace; pub use error::{MutationError, Result}; /// Re-export commonly used types pub mod prelude { - pub use crate::analyze::run_analysis; + pub use crate::analyze::{run_analysis, run_analysis_with_options, AnalysisOptions}; pub use crate::ast_analysis::{AridNodeDetector, AstNode, AstNodeType}; pub use crate::coverage::parse_coverage_file; pub use crate::error::{MutationError, Result}; diff --git a/src/main.rs b/src/main.rs index 7341658..5b65fbf 100644 --- a/src/main.rs +++ b/src/main.rs @@ -11,8 +11,10 @@ mod error; mod git_changes; mod mutation; mod operators; +mod parallel; mod project; mod report; +mod workspace; use error::{MutationError, Result}; use project::Project; @@ -95,10 +97,22 @@ enum Commands { #[arg(short, long, default_value = "0")] jobs: u32, + /// Number of mutants to verify concurrently + #[arg(short = 'P', long, default_value = "1")] + parallel: usize, + /// Command to test the mutants #[arg(short, long)] command: Option, + /// One-time command used to prepare each isolated worker workspace + #[arg(long, value_name = "CMD")] + setup_command: Option, + + /// Keep temporary parallel worker worktrees for debugging + #[arg(long)] + keep_worktrees: bool, + /// Maximum acceptable survival rate (0.3 = 30%) #[arg(long, default_value = "0.75")] survival_threshold: f64, @@ -204,7 +218,10 @@ async fn main() -> Result<()> { folder, timeout, jobs, + parallel, command, + setup_command, + keep_worktrees, survival_threshold, min_score, sqlite, @@ -232,19 +249,41 @@ async fn main() -> Result<()> { } } - analyze::run_analysis( + if parallel == 0 { + return Err(MutationError::InvalidInput( + "--parallel must be at least 1".to_string(), + )); + } + + if parallel > 1 && jobs > 0 { + if let Ok(available) = std::thread::available_parallelism() { + let requested = parallel.saturating_mul(jobs as usize); + if requested > available.get() { + eprintln!( + "Warning: --parallel {parallel} × --jobs {jobs} may use \ + {requested} compiler jobs on {} available CPUs", + available.get() + ); + } + } + } + + analyze::run_analysis_with_options(analyze::AnalysisOptions { project, folder, command, + setup_command, jobs, - timeout, + parallel, + timeout_secs: timeout, survival_threshold, min_score, - sqlite, + sqlite_path: sqlite, run_id, file_path, survivors_only, - ) + keep_worktrees, + }) .await?; } } @@ -257,3 +296,26 @@ fn read_skip_lines(path: &PathBuf) -> Result>> { let map: HashMap> = serde_json::from_str(&content)?; Ok(map) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn analyze_parallel_defaults_to_one() { + let cli = Cli::try_parse_from(["bcore-mutation", "analyze"]).unwrap(); + match cli.command { + Commands::Analyze { parallel, .. } => assert_eq!(parallel, 1), + Commands::Mutate { .. } => panic!("expected analyze command"), + } + } + + #[test] + fn analyze_accepts_arbitrary_parallel_value() { + let cli = Cli::try_parse_from(["bcore-mutation", "analyze", "--parallel", "17"]).unwrap(); + match cli.command { + Commands::Analyze { parallel, .. } => assert_eq!(parallel, 17), + Commands::Mutate { .. } => panic!("expected analyze command"), + } + } +} diff --git a/src/parallel.rs b/src/parallel.rs new file mode 100644 index 0000000..72fe4bb --- /dev/null +++ b/src/parallel.rs @@ -0,0 +1,1167 @@ +//! Parallel mutant verification using isolated Git worktrees. + +use crate::analyze::ScoreSummary; +use crate::commands::ProjectCommands; +use crate::db::Database; +use crate::error::{MutationError, Result}; +use crate::report::generate_report; +use crate::workspace::{self, WorkerWorkspace, WorktreePool}; +use futures::future::join_all; +use std::collections::HashSet; +use std::fs; +use std::path::{Component, Path, PathBuf}; +use std::process::Stdio; +use std::time::Duration; +use tempfile::NamedTempFile; +use tokio::process::Command as TokioCommand; +use tokio::sync::mpsc; +use tokio::task::JoinHandle; +use tokio::time::timeout; + +#[derive(Clone, Debug)] +enum MutantIdentity { + Database(i64), + Folder(String), +} + +#[derive(Clone, Debug)] +enum MutantPayload { + Diff(String), + CompleteFile(String), +} + +#[derive(Clone, Debug)] +struct MutantJob { + sequence: usize, + identity: MutantIdentity, + target_file: PathBuf, + payload: MutantPayload, + command: String, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum MutantStatus { + Killed, + Survived, + Error, +} + +impl MutantStatus { + fn database_value(self) -> &'static str { + match self { + Self::Killed => "killed", + Self::Survived => "survived", + Self::Error => "error", + } + } +} + +#[derive(Debug)] +struct CommandExecution { + success: bool, + exit_code: Option, + stdout: String, + stderr: String, + timed_out: bool, + spawn_error: Option, +} + +#[derive(Debug)] +struct MutantResult { + worker_id: usize, + job: MutantJob, + status: MutantStatus, + command: Option, + error: Option, + workspace_usable: bool, +} + +struct WorkerMessage { + result: MutantResult, +} + +struct ParallelRun { + results: Vec, + skipped: usize, +} + +struct FolderPlan { + folder_path: PathBuf, + target_file: PathBuf, + command: String, + jobs: Vec, +} + +/// Analyze database-backed mutants with an arbitrary number of isolated workers. +#[allow(clippy::too_many_arguments)] +pub async fn analyze_database( + db: &Database, + run_id: i64, + command: &str, + setup_command: Option<&str>, + setup_timeout_secs: u64, + timeout_secs: u64, + file_path: Option<&str>, + survivors_only: bool, + parallel: usize, + keep_worktrees: bool, +) -> Result { + let mutants = db.get_mutants_for_run(run_id, file_path, survivors_only)?; + let total = mutants.len(); + + print_database_header(total, run_id, file_path, survivors_only); + if total == 0 { + return Err(MutationError::InvalidInput(format!( + "No mutants found for run_id={run_id}" + ))); + } + + let repository_root = workspace::repository_root(Path::new(".")).await?; + let recorded_commit = db.get_run_commit_hash(run_id)?; + let base_commit = if recorded_commit == "unknown" { + eprintln!( + "Warning: run_id={run_id} has no recorded commit; using the current HEAD instead" + ); + workspace::head_commit(&repository_root).await? + } else { + recorded_commit + }; + + let worker_count = effective_worker_count(parallel, total)?; + print_parallelism(worker_count, parallel, total); + + let mut pool = + WorktreePool::create(repository_root, &base_commit, worker_count, keep_worktrees).await?; + let referenced_commands = setup_command + .into_iter() + .chain(std::iter::once(command)) + .collect::>(); + pool.link_referenced_siblings(referenced_commands)?; + + let analysis_result = async { + if let Some(setup) = setup_command { + run_checked_on_all(pool.workspaces(), setup, setup_timeout_secs, "setup").await?; + } + run_checked_on_all(pool.workspaces(), command, timeout_secs, "baseline").await?; + + let jobs = mutants + .into_iter() + .enumerate() + .map(|(sequence, mutant)| { + let target = mutant.file_path.as_deref().ok_or_else(|| { + MutationError::InvalidInput(format!( + "mutant {} has no target file path", + mutant.id + )) + })?; + Ok(MutantJob { + sequence, + identity: MutantIdentity::Database(mutant.id), + target_file: validate_target_path(target)?, + payload: MutantPayload::Diff(mutant.diff), + command: command.to_string(), + }) + }) + .collect::>>()?; + + let run = execute_parallel_jobs( + pool.workspaces(), + jobs, + timeout_secs, + None, + |job| { + if let MutantIdentity::Database(id) = job.identity { + db.update_mutant_status(id, "running", command)?; + } + Ok(()) + }, + |result| { + if let MutantIdentity::Database(id) = result.job.identity { + db.update_mutant_status(id, result.status.database_value(), command)?; + } + Ok(()) + }, + ) + .await?; + + let killed = run + .results + .iter() + .filter(|result| result.status == MutantStatus::Killed) + .count() as u64; + let survived = run + .results + .iter() + .filter(|result| result.status == MutantStatus::Survived) + .count(); + let errors = run + .results + .iter() + .filter(|result| result.status == MutantStatus::Error) + .count(); + + let score = killed as f64 / total as f64; + println!( + "\nMUTATION SCORE: {:.2}% ({} killed / {} total)", + score * 100.0, + killed, + total + ); + println!("Survived: {survived}"); + if errors > 0 { + println!("Errors: {errors}"); + } + + Ok(ScoreSummary { + killed, + total: total as u64, + }) + } + .await; + + finish_with_cleanup(analysis_result, &mut pool).await +} + +/// Analyze folder-backed mutants with isolated worktrees. +#[allow(clippy::too_many_arguments)] +pub async fn analyze_folders( + folder_paths: Vec, + command: Option, + setup_command: Option, + jobs: u32, + timeout_secs: u64, + survival_threshold: f64, + parallel: usize, + keep_worktrees: bool, + project_commands: &dyn ProjectCommands, +) -> Result { + let repository_root = workspace::repository_root(Path::new(".")).await?; + let uses_derived_commands = command.is_none(); + let plans = load_folder_plans( + folder_paths, + command, + jobs, + project_commands, + &repository_root, + )?; + + let maximum_folder_size = plans.iter().map(|plan| plan.jobs.len()).max().unwrap_or(0); + if maximum_folder_size == 0 { + return Err(MutationError::InvalidInput( + "No mutants found in the selected folders".to_string(), + )); + } + + let worker_count = effective_worker_count(parallel, maximum_folder_size)?; + let total_mutants: usize = plans.iter().map(|plan| plan.jobs.len()).sum(); + print_parallelism(worker_count, parallel, total_mutants); + + let base_commit = workspace::head_commit(&repository_root).await?; + let setup = match setup_command.as_deref() { + Some(command) => Some(command.to_string()), + None if uses_derived_commands => Some(project_commands.build_command()), + None => None, + }; + let mut pool = WorktreePool::create( + repository_root.clone(), + &base_commit, + worker_count, + keep_worktrees, + ) + .await?; + let referenced_commands = setup + .as_deref() + .into_iter() + .chain(plans.iter().map(|plan| plan.command.as_str())) + .collect::>(); + pool.link_referenced_siblings(referenced_commands)?; + + let analysis_result = async { + if let Some(setup) = setup.as_deref() { + run_checked_on_all( + pool.workspaces(), + setup, + project_commands.build_timeout_secs(), + "setup", + ) + .await?; + } + + let mut checked_commands = HashSet::new(); + let mut overall = ScoreSummary::default(); + + for plan in plans { + if checked_commands.insert(plan.command.clone()) { + run_checked_on_all(pool.workspaces(), &plan.command, timeout_secs, "baseline") + .await?; + } + + let total = plan.jobs.len(); + println!("* {total} MUTANTS in {} *", plan.folder_path.display()); + let run = execute_parallel_jobs( + pool.workspaces(), + plan.jobs, + timeout_secs, + Some((survival_threshold, total)), + |_| Ok(()), + |_| Ok(()), + ) + .await?; + + let killed = run + .results + .iter() + .filter(|result| result.status == MutantStatus::Killed) + .count() as u64; + let mut survivors = run + .results + .iter() + .filter_map(|result| { + if result.status == MutantStatus::Survived { + match &result.job.identity { + MutantIdentity::Folder(name) => { + Some((result.job.sequence, name.clone())) + } + MutantIdentity::Database(_) => None, + } + } else { + None + } + }) + .collect::>(); + survivors.sort_by_key(|(sequence, _)| *sequence); + let survivor_names = survivors + .into_iter() + .map(|(_, name)| name) + .collect::>(); + + let score = killed as f64 / total as f64; + println!("\nMUTATION SCORE: {:.2}%", score * 100.0); + if run.skipped > 0 { + println!( + "Skipped {} mutants after the survival threshold was exceeded", + run.skipped + ); + } + + generate_report( + &survivor_names, + plan.folder_path.to_string_lossy().as_ref(), + plan.target_file.to_string_lossy().as_ref(), + score, + ) + .await?; + + overall.killed += killed; + overall.total += total as u64; + } + + Ok(overall) + } + .await; + + finish_with_cleanup(analysis_result, &mut pool).await +} + +async fn finish_with_cleanup(analysis_result: Result, pool: &mut WorktreePool) -> Result { + let cleanup_result = pool.cleanup().await; + match (analysis_result, cleanup_result) { + (Ok(value), Ok(())) => Ok(value), + (Err(error), _) => Err(error), + (Ok(_), Err(error)) => Err(error), + } +} + +fn load_folder_plans( + folder_paths: Vec, + command: Option, + jobs: u32, + project_commands: &dyn ProjectCommands, + repository_root: &Path, +) -> Result> { + let mut plans = Vec::with_capacity(folder_paths.len()); + + for folder_path in folder_paths { + let folder_path = if folder_path.is_absolute() { + folder_path + } else { + std::env::current_dir()?.join(folder_path) + }; + let original_file_path = folder_path.join("original_file.txt"); + let raw_target = fs::read_to_string(&original_file_path)?; + let target_file = repository_relative_path(repository_root, raw_target.trim())?; + ensure_target_clean(repository_root, &target_file)?; + + let test_command = match command.as_ref() { + Some(command) => command.clone(), + None => project_commands.test_command(target_file.to_string_lossy().as_ref(), jobs)?, + }; + + let mut mutant_files = fs::read_dir(&folder_path)? + .filter_map(|entry| entry.ok()) + .map(|entry| entry.path()) + .filter(|path| { + path.is_file() && path.extension().is_some_and(|extension| extension != "txt") + }) + .collect::>(); + mutant_files.sort(); + + if mutant_files.is_empty() { + return Err(MutationError::InvalidInput(format!( + "No mutants in the provided folder path ({})", + folder_path.display() + ))); + } + + let jobs = mutant_files + .into_iter() + .enumerate() + .map(|(sequence, path)| { + let name = path + .file_name() + .and_then(|name| name.to_str()) + .ok_or_else(|| { + MutationError::InvalidInput(format!( + "invalid mutant filename: {}", + path.display() + )) + })? + .to_string(); + + Ok(MutantJob { + sequence, + identity: MutantIdentity::Folder(name), + target_file: target_file.clone(), + payload: MutantPayload::CompleteFile(fs::read_to_string(path)?), + command: test_command.clone(), + }) + }) + .collect::>>()?; + + plans.push(FolderPlan { + folder_path, + target_file, + command: test_command, + jobs, + }); + } + + Ok(plans) +} + +fn ensure_target_clean(repository_root: &Path, target: &Path) -> Result<()> { + for args in [ + vec!["diff", "--quiet", "--"], + vec!["diff", "--cached", "--quiet", "--"], + ] { + let status = std::process::Command::new("git") + .current_dir(repository_root) + .args(args) + .arg(target) + .status() + .map_err(|e| { + MutationError::Git(format!( + "failed to check whether {} is clean: {e}", + target.display() + )) + })?; + if !status.success() { + return Err(MutationError::InvalidInput(format!( + "parallel folder analysis requires a clean target file: {}", + target.display() + ))); + } + } + Ok(()) +} + +fn repository_relative_path(repository_root: &Path, raw_path: &str) -> Result { + let path = Path::new(raw_path); + if path.is_absolute() { + let relative = path.strip_prefix(repository_root).map_err(|_| { + MutationError::InvalidInput(format!( + "target file {} is outside repository {}", + path.display(), + repository_root.display() + )) + })?; + validate_target_path(relative) + } else { + validate_target_path(raw_path) + } +} + +fn validate_target_path(path: impl AsRef) -> Result { + let path = path.as_ref(); + if path.as_os_str().is_empty() || path.is_absolute() { + return Err(MutationError::InvalidInput(format!( + "mutant target must be a non-empty repository-relative path: {}", + path.display() + ))); + } + + if path.components().any(|component| { + matches!( + component, + Component::ParentDir | Component::RootDir | Component::Prefix(_) + ) + }) { + return Err(MutationError::InvalidInput(format!( + "mutant target escapes the repository: {}", + path.display() + ))); + } + + Ok(path.to_path_buf()) +} + +fn effective_worker_count(requested: usize, mutant_count: usize) -> Result { + if requested == 0 { + return Err(MutationError::InvalidInput( + "--parallel must be at least 1".to_string(), + )); + } + Ok(requested.min(mutant_count)) +} + +fn print_parallelism(worker_count: usize, requested: usize, mutant_count: usize) { + println!( + "Parallel analysis: {worker_count} worker(s) for {mutant_count} mutant(s) \ + (requested {requested})" + ); +} + +fn print_database_header(total: usize, run_id: i64, file_path: Option<&str>, survivors_only: bool) { + match (file_path, survivors_only) { + (Some(path), true) => { + println!("* {total} SURVIVING MUTANTS in run_id={run_id} (file: {path}) *") + } + (Some(path), false) => { + println!("* {total} MUTANTS in run_id={run_id} (file: {path}) *") + } + (None, true) => println!("* {total} SURVIVING MUTANTS in run_id={run_id} *"), + (None, false) => println!("* {total} MUTANTS in run_id={run_id} *"), + } +} + +async fn run_checked_on_all( + workspaces: &[WorkerWorkspace], + command: &str, + timeout_secs: u64, + phase: &str, +) -> Result<()> { + println!( + "Running {phase} command in {} worker workspace(s)...", + workspaces.len() + ); + + let command_runs = join_all(workspaces.iter().map(|workspace| async move { + ( + workspace.id, + capture_command(&workspace.path, command, timeout_secs).await, + ) + })); + let executions = tokio::select! { + executions = command_runs => executions, + signal = tokio::signal::ctrl_c() => { + let signal_note = signal + .err() + .map(|error| format!(": {error}")) + .unwrap_or_default(); + return Err(MutationError::Command(format!( + "parallel analysis cancelled during {phase}{signal_note}" + ))); + } + }; + + for (worker_id, execution) in executions { + print_command_execution( + &format!("[worker {worker_id}][{phase}]"), + command, + &execution, + ); + if !execution.success { + let setup_hint = if phase == "baseline" { + " A fresh parallel worktree has no existing build directory; use \ + --setup-command if the test command does not configure it." + } else { + "" + }; + return Err(MutationError::InvalidInput(format!( + "{phase} command failed in worker {worker_id}.{setup_hint}" + ))); + } + } + + Ok(()) +} + +async fn execute_parallel_jobs( + workspaces: &[WorkerWorkspace], + jobs: Vec, + timeout_secs: u64, + survival_threshold: Option<(f64, usize)>, + mut on_dispatch: OnDispatch, + mut on_result: OnResult, +) -> Result +where + OnDispatch: FnMut(&MutantJob) -> Result<()>, + OnResult: FnMut(&MutantResult) -> Result<()>, +{ + if jobs.is_empty() { + return Ok(ParallelRun { + results: Vec::new(), + skipped: 0, + }); + } + + let worker_count = workspaces.len().min(jobs.len()); + let (result_tx, mut result_rx) = mpsc::channel::(worker_count); + let mut job_senders = Vec::with_capacity(worker_count); + let mut handles: Vec> = Vec::with_capacity(worker_count); + + for workspace in workspaces.iter().take(worker_count).cloned() { + let (job_tx, job_rx) = mpsc::channel::(1); + let worker_result_tx = result_tx.clone(); + handles.push(tokio::spawn(worker_loop( + workspace, + job_rx, + worker_result_tx, + timeout_secs, + ))); + job_senders.push(job_tx); + } + drop(result_tx); + + let mut next_job = 0usize; + let mut in_flight = 0usize; + let mut active_jobs = vec![None; worker_count]; + for sender in &job_senders { + let job = jobs[next_job].clone(); + if let Err(error) = on_dispatch(&job) { + abort_workers(job_senders, handles).await; + return Err(error); + } + active_jobs[next_job] = Some(job.clone()); + if sender.send(job).await.is_err() { + abort_workers(job_senders, handles).await; + return Err(MutationError::Command( + "parallel worker stopped before receiving a job".to_string(), + )); + } + next_job += 1; + in_flight += 1; + } + + let mut results = Vec::with_capacity(jobs.len()); + let mut stop_scheduling = false; + let mut fatal_error = None; + + while in_flight > 0 { + let message = tokio::select! { + message = result_rx.recv() => { + match message { + Some(message) => message, + None => { + abort_workers(job_senders, handles).await; + return Err(MutationError::Command( + "all parallel workers stopped before returning their results" + .to_string(), + )); + } + } + } + signal = tokio::signal::ctrl_c() => { + let signal_note = signal + .err() + .map(|error| format!(": {error}")) + .unwrap_or_default(); + for (worker_id, job) in active_jobs.iter().enumerate() { + if let Some(job) = job { + let cancelled = error_result( + worker_id, + job.clone(), + "analysis cancelled before the mutant completed".to_string(), + false, + ); + let _ = on_result(&cancelled); + } + } + abort_workers(job_senders, handles).await; + return Err(MutationError::Command(format!( + "parallel analysis cancelled{signal_note}" + ))); + } + }; + in_flight -= 1; + + print_mutant_result(&message.result, jobs.len()); + + let worker_id = message.result.worker_id; + active_jobs[worker_id] = None; + if let Err(error) = on_result(&message.result) { + stop_scheduling = true; + if fatal_error.is_none() { + fatal_error = Some(error); + } + } + if !message.result.workspace_usable { + stop_scheduling = true; + if fatal_error.is_none() { + fatal_error = Some(MutationError::Command( + message + .result + .error + .clone() + .unwrap_or_else(|| format!("worker {worker_id} became unusable")), + )); + } + } + results.push(message.result); + + if let Some((threshold, total)) = survival_threshold { + let survived = results + .iter() + .filter(|result| result.status == MutantStatus::Survived) + .count(); + let rate = survived as f64 / total as f64; + if !stop_scheduling && rate > threshold { + println!( + "\nTerminating early: {:.2}% mutants surviving after {} completed mutants", + rate * 100.0, + results.len() + ); + println!( + "Survival rate exceeds threshold of {:.0}%", + threshold * 100.0 + ); + stop_scheduling = true; + } + } + + if !stop_scheduling && next_job < jobs.len() { + let job = jobs[next_job].clone(); + match on_dispatch(&job) { + Ok(()) => { + active_jobs[worker_id] = Some(job.clone()); + if job_senders[worker_id].send(job).await.is_err() { + active_jobs[worker_id] = None; + stop_scheduling = true; + if fatal_error.is_none() { + fatal_error = Some(MutationError::Command(format!( + "worker {worker_id} stopped before receiving its next job" + ))); + } + } else { + next_job += 1; + in_flight += 1; + } + } + Err(error) => { + stop_scheduling = true; + if fatal_error.is_none() { + fatal_error = Some(error); + } + } + } + } + } + + drop(job_senders); + for handle in handles { + handle + .await + .map_err(|e| MutationError::Command(format!("parallel worker task failed: {e}")))?; + } + + if let Some(error) = fatal_error { + return Err(error); + } + + results.sort_by_key(|result| result.job.sequence); + Ok(ParallelRun { + skipped: jobs.len() - next_job, + results, + }) +} + +async fn abort_workers(job_senders: Vec>, handles: Vec>) { + drop(job_senders); + for handle in &handles { + handle.abort(); + } + for handle in handles { + let _ = handle.await; + } +} + +async fn worker_loop( + workspace: WorkerWorkspace, + mut jobs: mpsc::Receiver, + results: mpsc::Sender, + timeout_secs: u64, +) { + while let Some(job) = jobs.recv().await { + let result = execute_mutant(&workspace, job, timeout_secs).await; + let usable = result.workspace_usable; + if results.send(WorkerMessage { result }).await.is_err() || !usable { + break; + } + } +} + +async fn execute_mutant( + workspace: &WorkerWorkspace, + job: MutantJob, + timeout_secs: u64, +) -> MutantResult { + if let Err(error) = restore_file(&workspace.path, &job.target_file).await { + return error_result(workspace.id, job, error.to_string(), false); + } + + let apply_result = match &job.payload { + MutantPayload::Diff(diff) => apply_diff(&workspace.path, diff).await, + MutantPayload::CompleteFile(content) => { + fs::write(workspace.path.join(&job.target_file), content).map_err(Into::into) + } + }; + + if let Err(error) = apply_result { + let restore_result = restore_file(&workspace.path, &job.target_file).await; + let usable = restore_result.is_ok(); + let detail = match restore_result { + Ok(()) => format!("failed to apply mutant: {error}"), + Err(restore_error) => { + format!("failed to apply mutant: {error}; restore also failed: {restore_error}") + } + }; + return error_result(workspace.id, job, detail, usable); + } + + let execution = capture_command(&workspace.path, &job.command, timeout_secs).await; + let status = if execution.success { + MutantStatus::Survived + } else { + MutantStatus::Killed + }; + + match restore_file(&workspace.path, &job.target_file).await { + Ok(()) => MutantResult { + worker_id: workspace.id, + job, + status, + command: Some(execution), + error: None, + workspace_usable: true, + }, + Err(error) => MutantResult { + worker_id: workspace.id, + job, + status: MutantStatus::Error, + command: Some(execution), + error: Some(format!("failed to restore worker workspace: {error}")), + workspace_usable: false, + }, + } +} + +fn error_result( + worker_id: usize, + job: MutantJob, + error: String, + workspace_usable: bool, +) -> MutantResult { + MutantResult { + worker_id, + job, + status: MutantStatus::Error, + command: None, + error: Some(error), + workspace_usable, + } +} + +async fn apply_diff(workspace: &Path, diff: &str) -> Result<()> { + use std::io::Write; + + let mut patch = NamedTempFile::new()?; + patch.write_all(diff.as_bytes())?; + patch.flush()?; + + let output = TokioCommand::new("git") + .current_dir(workspace) + .args(["apply", "--whitespace=nowarn"]) + .arg(patch.path()) + .output() + .await + .map_err(|e| MutationError::Git(format!("git apply failed: {e}")))?; + + if !output.status.success() { + return Err(MutationError::Git(format!( + "git apply error: {}", + String::from_utf8_lossy(&output.stderr).trim() + ))); + } + Ok(()) +} + +async fn restore_file(workspace: &Path, target_file: &Path) -> Result<()> { + let output = TokioCommand::new("git") + .current_dir(workspace) + .args(["restore", "--worktree", "--"]) + .arg(target_file) + .output() + .await + .map_err(|e| MutationError::Git(format!("git restore failed: {e}")))?; + + if !output.status.success() { + return Err(MutationError::Git(format!( + "git restore failed for {}: {}", + target_file.display(), + String::from_utf8_lossy(&output.stderr).trim() + ))); + } + Ok(()) +} + +async fn capture_command(workspace: &Path, command: &str, timeout_secs: u64) -> CommandExecution { + let (shell, shell_arg) = if cfg!(target_os = "windows") { + ("cmd", "/C") + } else { + ("sh", "-c") + }; + + let mut child = TokioCommand::new(shell); + child + .current_dir(workspace) + .arg(shell_arg) + .arg(command) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .kill_on_drop(true); + + match timeout(Duration::from_secs(timeout_secs), child.output()).await { + Ok(Ok(output)) => CommandExecution { + success: output.status.success(), + exit_code: output.status.code(), + stdout: String::from_utf8_lossy(&output.stdout).to_string(), + stderr: String::from_utf8_lossy(&output.stderr).to_string(), + timed_out: false, + spawn_error: None, + }, + Ok(Err(error)) => CommandExecution { + success: false, + exit_code: None, + stdout: String::new(), + stderr: String::new(), + timed_out: false, + spawn_error: Some(error.to_string()), + }, + Err(_) => CommandExecution { + success: false, + exit_code: None, + stdout: String::new(), + stderr: String::new(), + timed_out: true, + spawn_error: None, + }, + } +} + +fn print_mutant_result(result: &MutantResult, total: usize) { + let identity = match &result.job.identity { + MutantIdentity::Database(id) => format!("mutant {id}"), + MutantIdentity::Folder(name) => name.clone(), + }; + let status = match result.status { + MutantStatus::Killed => "KILLED ✅", + MutantStatus::Survived => "NOT KILLED ❌", + MutantStatus::Error => "ERROR", + }; + let prefix = format!( + "[worker {}][{}/{}][{}]", + result.worker_id, + result.job.sequence + 1, + total, + identity + ); + println!("{prefix} {status}"); + + if let Some(execution) = &result.command { + print_command_execution(&prefix, &result.job.command, execution); + } + if let Some(error) = &result.error { + eprintln!("{prefix} {error}"); + } +} + +fn print_command_execution(prefix: &str, command: &str, execution: &CommandExecution) { + println!("{prefix} Command: {command}"); + if execution.timed_out { + println!("{prefix} Command timed out"); + } else if let Some(error) = &execution.spawn_error { + println!("{prefix} Command execution failed: {error}"); + } else { + println!("{prefix} Exit code: {}", execution.exit_code.unwrap_or(-1)); + } + if !execution.stdout.is_empty() { + println!("{prefix} STDOUT:\n{}", execution.stdout); + } + if !execution.stderr.is_empty() { + println!("{prefix} STDERR:\n{}", execution.stderr); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::process::Command; + use tempfile::tempdir; + + #[test] + fn worker_count_accepts_any_positive_value_and_caps_to_mutants() { + assert_eq!(effective_worker_count(1, 9).unwrap(), 1); + assert_eq!(effective_worker_count(3, 9).unwrap(), 3); + assert_eq!(effective_worker_count(20, 9).unwrap(), 9); + assert!(effective_worker_count(0, 9).is_err()); + } + + #[test] + fn target_path_must_stay_inside_repository() { + assert_eq!( + validate_target_path("src/example.cpp").unwrap(), + PathBuf::from("src/example.cpp") + ); + assert!(validate_target_path("../example.cpp").is_err()); + assert!(validate_target_path("/tmp/example.cpp").is_err()); + assert!(validate_target_path("").is_err()); + } + + #[tokio::test] + async fn arbitrary_worker_pool_processes_each_mutant_in_isolation() { + let repository = initialized_repository(); + + let base = workspace::head_commit(repository.path()).await.unwrap(); + let mut pool = WorktreePool::create(repository.path().to_path_buf(), &base, 4, false) + .await + .unwrap(); + let workspace_paths = pool + .workspaces() + .iter() + .map(|workspace| workspace.path.clone()) + .collect::>(); + + let jobs = (0..9) + .map(|sequence| MutantJob { + sequence, + identity: MutantIdentity::Folder(format!("mutant-{sequence}")), + target_file: PathBuf::from("value.txt"), + payload: MutantPayload::CompleteFile(if sequence % 2 == 0 { + "killed\n".to_string() + } else { + "base\n".to_string() + }), + command: "! grep -q '^killed$' value.txt".to_string(), + }) + .collect::>(); + + let run = execute_parallel_jobs(pool.workspaces(), jobs, 5, None, |_| Ok(()), |_| Ok(())) + .await + .unwrap(); + + assert_eq!(run.results.len(), 9); + assert_eq!( + run.results + .iter() + .filter(|result| result.status == MutantStatus::Killed) + .count(), + 5 + ); + assert_eq!( + run.results + .iter() + .filter(|result| result.status == MutantStatus::Survived) + .count(), + 4 + ); + let used_workers = run + .results + .iter() + .map(|result| result.worker_id) + .collect::>(); + assert_eq!(used_workers.len(), 4); + assert_eq!( + fs::read_to_string(repository.path().join("value.txt")).unwrap(), + "base\n" + ); + + pool.cleanup().await.unwrap(); + assert!(workspace_paths.iter().all(|path| !path.exists())); + } + + #[tokio::test] + async fn threshold_stops_dispatching_new_jobs_but_finishes_in_flight_jobs() { + let repository = initialized_repository(); + let base = workspace::head_commit(repository.path()).await.unwrap(); + let mut pool = WorktreePool::create(repository.path().to_path_buf(), &base, 2, false) + .await + .unwrap(); + let jobs = (0..8) + .map(|sequence| MutantJob { + sequence, + identity: MutantIdentity::Folder(format!("mutant-{sequence}")), + target_file: PathBuf::from("value.txt"), + payload: MutantPayload::CompleteFile("survives\n".to_string()), + command: "true".to_string(), + }) + .collect::>(); + + let run = execute_parallel_jobs( + pool.workspaces(), + jobs, + 5, + Some((0.0, 8)), + |_| Ok(()), + |_| Ok(()), + ) + .await + .unwrap(); + + assert_eq!(run.results.len(), 2); + assert_eq!(run.skipped, 6); + assert!(run + .results + .iter() + .all(|result| result.status == MutantStatus::Survived)); + pool.cleanup().await.unwrap(); + } + + fn initialized_repository() -> tempfile::TempDir { + let repository = tempdir().unwrap(); + run_git(repository.path(), &["init", "-q"]); + run_git( + repository.path(), + &["config", "user.email", "test@example.com"], + ); + run_git(repository.path(), &["config", "user.name", "Test"]); + run_git(repository.path(), &["config", "commit.gpgsign", "false"]); + fs::write(repository.path().join("value.txt"), "base\n").unwrap(); + run_git(repository.path(), &["add", "value.txt"]); + run_git(repository.path(), &["commit", "-qm", "base"]); + repository + } + + fn run_git(repository: &Path, args: &[&str]) { + let output = Command::new("git") + .current_dir(repository) + .args(args) + .output() + .unwrap(); + assert!( + output.status.success(), + "git {:?} failed: {}", + args, + String::from_utf8_lossy(&output.stderr) + ); + } +} diff --git a/src/report.rs b/src/report.rs index ac7d984..ec21c10 100644 --- a/src/report.rs +++ b/src/report.rs @@ -44,9 +44,6 @@ pub async fn generate_report( } } - // Restore original file - restore_original_file(&original_file_path).await?; - println!("Surviving mutants:"); let mut diffs = Vec::new(); @@ -79,26 +76,9 @@ pub async fn generate_report( Ok(()) } -async fn restore_original_file(file_path: &str) -> Result<()> { - let output = Command::new("git") - .args(&["checkout", "--", file_path]) - .output() - .map_err(|e| MutationError::Git(format!("Failed to restore file: {}", e)))?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - return Err(MutationError::Git(format!( - "Git checkout failed: {}", - stderr - ))); - } - - Ok(()) -} - async fn get_git_diff(original_file: &str, modified_file: &str) -> Result { let output = Command::new("git") - .args(&["diff", "--no-index", original_file, modified_file]) + .args(["diff", "--no-index", original_file, modified_file]) .output() .map_err(|e| MutationError::Git(format!("Failed to get git diff: {}", e)))?; @@ -142,7 +122,7 @@ async fn parse_diffs_to_json(diffs_list: &[String]) -> Result Result { let output = Command::new("git") - .args(&["log", "--pretty=format:%h", "-n", "1"]) + .args(["log", "--pretty=format:%h", "-n", "1"]) .output() .map_err(|e| MutationError::Git(format!("Failed to get git hash: {}", e)))?; diff --git a/src/workspace.rs b/src/workspace.rs new file mode 100644 index 0000000..1961c7a --- /dev/null +++ b/src/workspace.rs @@ -0,0 +1,372 @@ +//! Isolated Git worktrees used by parallel mutant analysis. + +use crate::error::{MutationError, Result}; +use std::collections::BTreeSet; +use std::path::{Path, PathBuf}; +use std::process::Command; +use tempfile::TempDir; +use tokio::process::Command as TokioCommand; + +#[derive(Clone, Debug)] +pub struct WorkerWorkspace { + pub id: usize, + pub path: PathBuf, +} + +/// Owns the temporary Git worktrees created for one parallel analysis run. +/// +/// Normal cleanup is asynchronous and should be performed with [`cleanup`]. +/// `Drop` is a best-effort fallback for early returns and panics. +pub struct WorktreePool { + repository_root: PathBuf, + temporary_root: Option, + workspaces: Vec, + keep: bool, + cleaned: bool, +} + +impl WorktreePool { + pub async fn create( + repository_root: PathBuf, + base_commit: &str, + worker_count: usize, + keep: bool, + ) -> Result { + if worker_count == 0 { + return Err(MutationError::InvalidInput( + "parallel worker count must be at least 1".to_string(), + )); + } + + verify_commit(&repository_root, base_commit).await?; + + let temporary_root = tempfile::Builder::new() + .prefix("bcore-mutation-workers-") + .tempdir()?; + let mut pool = Self { + repository_root, + temporary_root: Some(temporary_root), + workspaces: Vec::with_capacity(worker_count), + keep, + cleaned: false, + }; + + for id in 0..worker_count { + let path = pool + .temporary_root + .as_ref() + .expect("temporary root exists while creating worktrees") + .path() + .join(format!("worker-{id}")); + + let add_worktree = TokioCommand::new("git") + .current_dir(&pool.repository_root) + .args(["worktree", "add", "--detach"]) + .arg(&path) + .arg(base_commit) + .output(); + let output = tokio::select! { + output = add_worktree => output.map_err(|e| { + MutationError::Git(format!("failed to create worker {id} worktree: {e}")) + })?, + signal = tokio::signal::ctrl_c() => { + let signal_note = signal + .err() + .map(|error| format!(": {error}")) + .unwrap_or_default(); + let cleanup_error = pool.cleanup().await.err(); + let cleanup_note = cleanup_error + .map(|e| format!("; cleanup also failed: {e}")) + .unwrap_or_default(); + return Err(MutationError::Command(format!( + "parallel analysis cancelled while creating worker {id} worktree{signal_note}{cleanup_note}" + ))); + } + }; + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + let cleanup_error = pool.cleanup().await.err(); + let cleanup_note = cleanup_error + .map(|e| format!("; cleanup also failed: {e}")) + .unwrap_or_default(); + return Err(MutationError::Git(format!( + "git worktree add failed for worker {id}: {}{}", + stderr.trim(), + cleanup_note + ))); + } + + pool.workspaces.push(WorkerWorkspace { id, path }); + } + + Ok(pool) + } + + pub fn workspaces(&self) -> &[WorkerWorkspace] { + &self.workspaces + } + + /// Mirror repository sibling paths referenced by commands as `../name`. + /// + /// Parallel workers live under a temporary root, so a command that worked + /// from the original checkout with `../qa-assets` would otherwise point at + /// the wrong parent directory. Symlinking referenced siblings into the + /// temporary root preserves that common Bitcoin Core layout without copying + /// large corpora. + pub fn link_referenced_siblings<'a>( + &self, + commands: impl IntoIterator, + ) -> Result<()> { + let Some(temporary_root) = &self.temporary_root else { + return Ok(()); + }; + let Some(repository_parent) = self.repository_root.parent() else { + return Ok(()); + }; + + for name in referenced_sibling_names(commands) { + let source = repository_parent.join(&name); + if !source.exists() { + continue; + } + + let destination = temporary_root.path().join(&name); + if destination.exists() { + continue; + } + + symlink_path(&source, &destination)?; + println!( + "Parallel workers linked ../{} to {}", + name, + source.display() + ); + } + + Ok(()) + } + + pub async fn cleanup(&mut self) -> Result<()> { + if self.cleaned { + return Ok(()); + } + + if self.keep { + if let Some(root) = self.temporary_root.take() { + let path = root.keep(); + println!("Parallel worker worktrees kept at {}", path.display()); + } + self.cleaned = true; + return Ok(()); + } + + let mut first_error = None; + for workspace in self.workspaces.iter().rev() { + let output = TokioCommand::new("git") + .current_dir(&self.repository_root) + .args(["worktree", "remove", "--force"]) + .arg(&workspace.path) + .output() + .await; + + match output { + Ok(output) if output.status.success() => {} + Ok(output) => { + if first_error.is_none() { + first_error = Some(MutationError::Git(format!( + "failed to remove worker {} worktree: {}", + workspace.id, + String::from_utf8_lossy(&output.stderr).trim() + ))); + } + } + Err(e) => { + if first_error.is_none() { + first_error = Some(MutationError::Git(format!( + "failed to remove worker {} worktree: {e}", + workspace.id + ))); + } + } + } + } + + self.workspaces.clear(); + self.temporary_root.take(); + self.cleaned = true; + + let _ = TokioCommand::new("git") + .current_dir(&self.repository_root) + .args(["worktree", "prune"]) + .output() + .await; + + match first_error { + Some(error) => Err(error), + None => Ok(()), + } + } +} + +impl Drop for WorktreePool { + fn drop(&mut self) { + if self.cleaned || self.keep { + return; + } + + for workspace in self.workspaces.iter().rev() { + let _ = Command::new("git") + .current_dir(&self.repository_root) + .args(["worktree", "remove", "--force"]) + .arg(&workspace.path) + .output(); + } + + let _ = Command::new("git") + .current_dir(&self.repository_root) + .args(["worktree", "prune"]) + .output(); + } +} + +pub async fn repository_root(start: &Path) -> Result { + let output = TokioCommand::new("git") + .current_dir(start) + .args(["rev-parse", "--show-toplevel"]) + .output() + .await + .map_err(|e| MutationError::Git(format!("failed to locate Git repository: {e}")))?; + + if !output.status.success() { + return Err(MutationError::Git(format!( + "not inside a Git repository: {}", + String::from_utf8_lossy(&output.stderr).trim() + ))); + } + + Ok(PathBuf::from( + String::from_utf8_lossy(&output.stdout).trim(), + )) +} + +pub async fn head_commit(repository_root: &Path) -> Result { + let output = TokioCommand::new("git") + .current_dir(repository_root) + .args(["rev-parse", "HEAD"]) + .output() + .await + .map_err(|e| MutationError::Git(format!("failed to resolve HEAD: {e}")))?; + + if !output.status.success() { + return Err(MutationError::Git(format!( + "failed to resolve HEAD: {}", + String::from_utf8_lossy(&output.stderr).trim() + ))); + } + + Ok(String::from_utf8_lossy(&output.stdout).trim().to_string()) +} + +async fn verify_commit(repository_root: &Path, commit: &str) -> Result<()> { + let object = format!("{commit}^{{commit}}"); + let output = TokioCommand::new("git") + .current_dir(repository_root) + .args(["cat-file", "-e", &object]) + .output() + .await + .map_err(|e| MutationError::Git(format!("failed to verify commit {commit}: {e}")))?; + + if !output.status.success() { + return Err(MutationError::Git(format!( + "commit {commit} is not available in the current repository" + ))); + } + + Ok(()) +} + +fn referenced_sibling_names<'a>(commands: impl IntoIterator) -> BTreeSet { + let mut names = BTreeSet::new(); + for command in commands { + let mut search_start = 0usize; + while let Some(relative_index) = command[search_start..].find("../") { + let index = search_start + relative_index; + let after_parent = &command[index + 3..]; + let preceded_by_path_component = command[..index] + .chars() + .next_back() + .is_some_and(|character| character == '.' || character == '/'); + if preceded_by_path_component { + search_start = index + 3; + continue; + } + let name = after_parent + .split(|character: char| { + character == '/' + || character.is_whitespace() + || matches!( + character, + '"' | '\'' | '`' | ';' | '|' | '&' | '(' | ')' | '<' | '>' | '\\' + ) + }) + .next() + .unwrap_or_default(); + + if is_valid_sibling_name(name) { + names.insert(name.to_string()); + } + + search_start = index + 3 + name.len(); + } + } + names +} + +fn is_valid_sibling_name(name: &str) -> bool { + !name.is_empty() && name != "." && name != ".." +} + +#[cfg(unix)] +fn symlink_path(source: &Path, destination: &Path) -> std::io::Result<()> { + std::os::unix::fs::symlink(source, destination) +} + +#[cfg(windows)] +fn symlink_path(source: &Path, destination: &Path) -> std::io::Result<()> { + if source.is_dir() { + std::os::windows::fs::symlink_dir(source, destination) + } else { + std::os::windows::fs::symlink_file(source, destination) + } +} + +#[cfg(not(any(unix, windows)))] +fn symlink_path(source: &Path, destination: &Path) -> std::io::Result<()> { + if source.is_dir() { + std::fs::create_dir_all(destination) + } else { + std::fs::copy(source, destination).map(|_| ()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn sibling_references_are_extracted_from_shell_commands() { + let names = referenced_sibling_names([ + "FUZZ=x ./build/bin/fuzz ../qa-assets/fuzz_corpora/x", + "FOO='../depends' ./script --path ../qa-assets", + "echo ../../outside ../.hidden ../two-words", + ]); + + assert!(names.contains("qa-assets")); + assert!(names.contains("depends")); + assert!(names.contains(".hidden")); + assert!(names.contains("two-words")); + assert!(!names.contains("..")); + assert_eq!(names.len(), 4); + } +} diff --git a/tests/parallel_cli.rs b/tests/parallel_cli.rs new file mode 100644 index 0000000..11518c6 --- /dev/null +++ b/tests/parallel_cli.rs @@ -0,0 +1,269 @@ +use bcore_mutation::db::{compute_patch_hash, Database, MutantData}; +use std::fs; +use std::path::Path; +use std::process::Command; +use tempfile::tempdir; + +#[test] +fn cli_verifies_nine_mutants_with_four_workers() { + let repository = initialized_repository("value.txt"); + + let commit = git_stdout(repository.path(), &["rev-parse", "HEAD"]); + let database_path = repository.path().join("mutation.db"); + let run_id = { + let mut database = Database::open(&database_path).unwrap(); + database.ensure_schema().unwrap(); + database.seed_projects().unwrap(); + let project_id = database.get_project_id("Bitcoin Core").unwrap(); + let run_id = database + .create_run(project_id, &commit, "test", None, None) + .unwrap(); + + let mutants = (0..9) + .map(|index| { + let diff = format!( + "diff --git a/value.txt b/value.txt\n\ + --- a/value.txt\n\ + +++ b/value.txt\n\ + @@ -1 +1 @@\n\ + -base\n\ + +mutant-{index}\n" + ); + MutantData { + patch_hash: compute_patch_hash(&diff), + diff, + file_path: "value.txt".to_string(), + operator: "integration-test".to_string(), + } + }) + .collect::>(); + database.insert_mutant_batch(run_id, &mutants).unwrap(); + run_id + }; + + let output = Command::new(env!("CARGO_BIN_EXE_bcore-mutation")) + .current_dir(repository.path()) + .args([ + "analyze", + "--sqlite", + database_path.to_str().unwrap(), + "--run-id", + &run_id.to_string(), + "--parallel", + "4", + "--command", + "test \"$(cat value.txt)\" = base", + ]) + .output() + .unwrap(); + + assert!( + output.status.success(), + "parallel CLI failed\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + let stdout = String::from_utf8_lossy(&output.stdout); + assert!(stdout.contains("Parallel analysis: 4 worker(s) for 9 mutant(s)")); + assert!(stdout.contains("MUTATION SCORE: 100.00% (9 killed / 9 total)")); + + let connection = rusqlite::Connection::open(&database_path).unwrap(); + let killed: usize = connection + .query_row( + "SELECT COUNT(*) FROM mutants WHERE run_id = ?1 AND status = 'killed'", + [run_id], + |row| row.get(0), + ) + .unwrap(); + let running: usize = connection + .query_row( + "SELECT COUNT(*) FROM mutants WHERE run_id = ?1 AND status = 'running'", + [run_id], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(killed, 9); + assert_eq!(running, 0); + assert_eq!( + fs::read_to_string(repository.path().join("value.txt")).unwrap(), + "base\n" + ); + + let worktree_list = git_stdout(repository.path(), &["worktree", "list", "--porcelain"]); + assert_eq!( + worktree_list + .lines() + .filter(|line| line.starts_with("worktree ")) + .count(), + 1 + ); +} + +#[test] +fn folder_mode_uses_the_same_parallel_worker_pool() { + let repository = initialized_repository("value.cpp"); + let mutation_folder = repository.path().join("muts-value"); + fs::create_dir(&mutation_folder).unwrap(); + fs::write(mutation_folder.join("original_file.txt"), "value.cpp\n").unwrap(); + for index in 0..7 { + fs::write( + mutation_folder.join(format!("mutant-{index}.cpp")), + format!("mutant-{index}\n"), + ) + .unwrap(); + } + + let output = Command::new(env!("CARGO_BIN_EXE_bcore-mutation")) + .current_dir(repository.path()) + .args([ + "analyze", + "--folder", + mutation_folder.to_str().unwrap(), + "--parallel", + "3", + "--survival-threshold", + "1.0", + "--command", + "test \"$(cat value.cpp)\" = base", + ]) + .output() + .unwrap(); + + assert!( + output.status.success(), + "parallel folder analysis failed\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + let stdout = String::from_utf8_lossy(&output.stdout); + assert!(stdout.contains("Parallel analysis: 3 worker(s) for 7 mutant(s)")); + assert!(stdout.contains("MUTATION SCORE: 100.00%")); + assert_eq!( + fs::read_to_string(repository.path().join("value.cpp")).unwrap(), + "base\n" + ); + + let worktree_list = git_stdout(repository.path(), &["worktree", "list", "--porcelain"]); + assert_eq!( + worktree_list + .lines() + .filter(|line| line.starts_with("worktree ")) + .count(), + 1 + ); +} + +#[test] +fn parallel_workers_can_use_referenced_sibling_assets() { + let parent = tempdir().unwrap(); + let repository_path = parent.path().join("repository"); + fs::create_dir(&repository_path).unwrap(); + initialize_repository_at(&repository_path, "value.txt"); + + let assets = parent.path().join("qa-assets"); + fs::create_dir(&assets).unwrap(); + fs::write(assets.join("token.txt"), "asset\n").unwrap(); + + let commit = git_stdout(&repository_path, &["rev-parse", "HEAD"]); + let database_path = repository_path.join("mutation.db"); + let run_id = { + let mut database = Database::open(&database_path).unwrap(); + database.ensure_schema().unwrap(); + database.seed_projects().unwrap(); + let project_id = database.get_project_id("Bitcoin Core").unwrap(); + let run_id = database + .create_run(project_id, &commit, "test", None, None) + .unwrap(); + + let diff = "diff --git a/value.txt b/value.txt\n\ + --- a/value.txt\n\ + +++ b/value.txt\n\ + @@ -1 +1 @@\n\ + -base\n\ + +mutant\n" + .to_string(); + database + .insert_mutant_batch( + run_id, + &[MutantData { + patch_hash: compute_patch_hash(&diff), + diff, + file_path: "value.txt".to_string(), + operator: "integration-test".to_string(), + }], + ) + .unwrap(); + run_id + }; + + let output = Command::new(env!("CARGO_BIN_EXE_bcore-mutation")) + .current_dir(&repository_path) + .args([ + "analyze", + "--sqlite", + database_path.to_str().unwrap(), + "--run-id", + &run_id.to_string(), + "--parallel", + "2", + "--command", + "test \"$(cat ../qa-assets/token.txt)\" = asset && test \"$(cat value.txt)\" = base", + ]) + .output() + .unwrap(); + + assert!( + output.status.success(), + "parallel CLI with sibling assets failed\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + let stdout = String::from_utf8_lossy(&output.stdout); + assert!(stdout.contains("Parallel workers linked ../qa-assets")); + assert!(stdout.contains("MUTATION SCORE: 100.00% (1 killed / 1 total)")); +} + +fn initialized_repository(filename: &str) -> tempfile::TempDir { + let repository = tempdir().unwrap(); + initialize_repository_at(repository.path(), filename); + repository +} + +fn initialize_repository_at(repository: &Path, filename: &str) { + run_git(repository, &["init", "-q"]); + run_git(repository, &["config", "user.email", "test@example.com"]); + run_git(repository, &["config", "user.name", "Test"]); + run_git(repository, &["config", "commit.gpgsign", "false"]); + fs::write(repository.join(filename), "base\n").unwrap(); + run_git(repository, &["add", filename]); + run_git(repository, &["commit", "-qm", "base"]); +} + +fn run_git(repository: &Path, args: &[&str]) { + let output = Command::new("git") + .current_dir(repository) + .args(args) + .output() + .unwrap(); + assert!( + output.status.success(), + "git {:?} failed: {}", + args, + String::from_utf8_lossy(&output.stderr) + ); +} + +fn git_stdout(repository: &Path, args: &[&str]) -> String { + let output = Command::new("git") + .current_dir(repository) + .args(args) + .output() + .unwrap(); + assert!( + output.status.success(), + "git {:?} failed: {}", + args, + String::from_utf8_lossy(&output.stderr) + ); + String::from_utf8(output.stdout).unwrap().trim().to_string() +} From af4c07175bc22fb27606f16ae3ef575dbf04dc91 Mon Sep 17 00:00:00 2001 From: Bruno Garcia Date: Wed, 29 Jul 2026 11:10:51 +0200 Subject: [PATCH 2/2] [REMOVE] benchmark --- .dockerignore | 7 + README.md | 19 + benchmark/Dockerfile | 25 ++ benchmark/README.md | 68 +++ .../benchmark_parallel.cpython-311.pyc | Bin 0 -> 20862 bytes benchmark/benchmark_parallel.py | 425 ++++++++++++++++++ benchmark/run-in-docker.sh | 31 ++ 7 files changed, 575 insertions(+) create mode 100644 .dockerignore create mode 100644 benchmark/Dockerfile create mode 100644 benchmark/README.md create mode 100644 benchmark/__pycache__/benchmark_parallel.cpython-311.pyc create mode 100755 benchmark/benchmark_parallel.py create mode 100755 benchmark/run-in-docker.sh diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..1930e53 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,7 @@ +target/ +.git/ +.DS_Store +mutation.db +benchmark-results/ +*.log + diff --git a/README.md b/README.md index 899f1a5..03b8390 100644 --- a/README.md +++ b/README.md @@ -222,6 +222,25 @@ bcore-mutation analyze --sqlite --run-id=1 --survival-threshold=0.2 \ cargo test ``` +## Benchmarking Parallel Analysis + +The `benchmark/` directory contains a Docker-based harness that runs the same +SQLite mutation run with multiple `--parallel` / `--jobs` configurations and +generates CSV results plus timing and speedup charts. + +```bash +bash benchmark/run-in-docker.sh /path/to/bitcoin \ + --db mutation.db \ + --run-id 123 \ + --repeats 3 \ + --case sequential:1:3 \ + --case parallel-3x3:3:3 \ + --command "cmake --build build -j3 && ./build/bin/test_bitcoin" +``` + +See `benchmark/README.md` for the fuzzing-oriented example that keeps +`../qa-assets` available inside Docker. + ## Contributing 1. Fork the repository. diff --git a/benchmark/Dockerfile b/benchmark/Dockerfile new file mode 100644 index 0000000..c6a32a7 --- /dev/null +++ b/benchmark/Dockerfile @@ -0,0 +1,25 @@ +FROM rust:1-bookworm + +RUN apt-get update && apt-get install -y --no-install-recommends \ + bash \ + build-essential \ + ca-certificates \ + clang \ + cmake \ + git \ + libboost-dev \ + libevent-dev \ + libsqlite3-dev \ + ninja-build \ + pkg-config \ + python3 \ + python3-matplotlib \ + sqlite3 \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /opt/bcore-mutation +COPY . /opt/bcore-mutation +RUN cargo install --path /opt/bcore-mutation + +ENTRYPOINT ["python3", "/opt/bcore-mutation/benchmark/benchmark_parallel.py"] + diff --git a/benchmark/README.md b/benchmark/README.md new file mode 100644 index 0000000..9cc9f49 --- /dev/null +++ b/benchmark/README.md @@ -0,0 +1,68 @@ +# Parallel Benchmark Harness + +This harness compares `bcore-mutation analyze --parallel 1` with one or more +parallel configurations inside Docker. It records wall-clock time, keeps logs, +and generates charts. + +## Build and Run + +From this repository: + +```bash +bash benchmark/run-in-docker.sh /path/to/bitcoin \ + --db mutation.db \ + --run-id 123 \ + --repeats 3 \ + --timeout 900 \ + --case sequential:1:3 \ + --case parallel-2x4:2:4 \ + --case parallel-3x3:3:3 \ + --case parallel-4x2:4:2 \ + --setup-command "cmake -B build_corecheck -DBUILD_FOR_FUZZING=ON && cmake --build build_corecheck -j3" \ + --command "FUZZ=coin_grinder_is_optimal ./build_corecheck/bin/fuzz ../qa-assets/fuzz_corpora/coin_grinder_is_optimal" +``` + +The Docker runner mounts the subject repository's parent directory at `/bench`. +That matters for Bitcoin Core fuzz benchmarks because a sibling directory such +as `../qa-assets` remains visible inside the container. + +By default the runner limits Docker to 10 CPUs. Override it with: + +```bash +BCORE_BENCH_CPUS=8 bash benchmark/run-in-docker.sh /path/to/bitcoin ... +``` + +## Outputs + +Results are written to `benchmark-results/` by default: + +- `results.csv`: one row per case/repeat. +- `summary.md`: median timing and speedup table. +- `time-by-case.png`: median wall-clock seconds. +- `speedup-by-case.png`: speedup relative to the `sequential` case. +- `logs/*.stdout.log` and `logs/*.stderr.log`: raw analyze output. +- `db/*.db`: copied SQLite database used for each run. + +Each run uses a copy of the input SQLite database, so the original DB is not +modified by the benchmark. + +## Case Format + +Cases use this format: + +```text +LABEL:PARALLEL:JOBS +``` + +For a 10 CPU machine, useful starting points are: + +```text +sequential:1:3 +parallel-2x4:2:4 +parallel-3x3:3:3 +parallel-4x2:4:2 +``` + +`PARALLEL * JOBS` is the approximate maximum compiler/test parallelism. For +example, `parallel-3x3` can run about 9 build jobs across 3 active mutants. + diff --git a/benchmark/__pycache__/benchmark_parallel.cpython-311.pyc b/benchmark/__pycache__/benchmark_parallel.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c3a7618020b8085e00a228c8545ded0282104dbd GIT binary patch literal 20862 zcmch9d2Ab3nr9VRB#XDGTe4-9k65N7DL!Lc@e!Z$sW`D?cdHX9w5lZ2rbMNR^1)CY z#^c?VJ(`VXHZxIoJ3HFmbXwV{XN}(O?ffwqbf>#9jm7*?GzhprKtO;+2Ac&IP>?hx z@E?o)eXqzOiS9zU%QP!Jv=B_4+U4m$+>l_dn>Rbh*p`|K%9Z zaqn_6*U!nk>`3q=)V7+wWyhU%!t% z{r!IS4D<)^6cWLaP=AQ!?Mj44YWiz<&cP`m**)+10JZoC|K{c3dO2CV$;sXi9Eiui zd3}J#N2M38Kca*frVp6@M`G5>opSi5&|fEa$q_v3Sak&ULRLRQ}I|rx}r&AF*TM*C<$p~JQW*FNmmp#J`^8} zrQ*raZlCY$MMcsE)%aLSQpZO%>9q@kNmc10WWtbQqp`&GwDOuXlvJhUs3PGp0t{&| zrYTydq$o6NhI;fu~h5=3Xm=+C|8jb)cQJAP|}i;5*xh8ikGw$ zW2uX05pW}` zG&<6n76>@a7yip`1m5KouAj#Qcf5>gtvF@JZp6w?gaj+(Vj(w%##0auq8SCJmQo9@ zMC^i+C^+Mzse)JZRmW)r&e71e&y}pMq{x8B<~@H9$=ZQjBXTk&LAZ z9t|^kRMrY!+H@6%?67s#ak1gUKi-j3H1gGm|-*0 ztTNTM;JOk^j4NjUqEl1j>gb?Fdx1c`fhEL$`A>*_m&J;8VqgN4S}) zud?GZ`>NKbihB=XU9wWhPJ3=Q_zeFFzD_tMoD)LEp)HqAs7~kQQZ#jnSEef83773_ zRq6@%XvbPE#r91zgLRzRI^kKS&Mte|8P5%e%uRT{>$%Eb<*qru&0Xa)F4=vVWwmU} zt9C!M+%1cbN#nJbe1lKXiYQ(gN5++L&-*?=|9xcjFxHp9?*ZTQImM*+(ZT}d)l$+p z0w?z$IDE3_^!}IjpFD}c@fQx9>0}#-axFG8mQbusscY-Ctv&clx3UZvfdpwMQVr5C zDI;U4>*+NmLdNzKB9~@KK)`xbh80!&oMv2lg@wp!g72}Elu%-*)n=9U)pV0Bwe@vV z)Tz=am8k(Jw@VvK#8U-lLK!WHF?AT3gOW$~tHa|X*j~?GA5#v~f;Z`j?;I@yGlGb1 zR8-gDQE(wyN@KYf0;T#5sIjXBk?KOBa;XKUa&1tf7Deu6V)U7O~cKR{bQvfBHI zwzb&GZk1O&LD?(iHhI+(w6k57MV#V z#GSYb{!2)KB&$0y$sTySx(1UYBdAY$%Rw`M%r#AsrBqT{rXg0Es6*{RE@oEYt}YE+ z&6t(p{ujnbGJ`G)-r-O1Qb2~8=AGT5A+qw4G^ z)hX>pRY|OH5!4Fq*w~nYrtU9~728f@+EP1lxAYQYec6UQf^Lq}j?8lEj=Bm0hAV|E z2dR-}!B%IaQE`)R*3=ECO?p+|_{arRG&ux8a7;I}$LNJSbx-K(Yjr3<$~5lqF56FQW#dmW; zNLMc^qfG3l^)?iXCp0V$mM^WdG#Wgb>?#gT(+i`C>!!@l1b(6hG>B1`+6m-B$iDj) z0x?-0D7B4oQ7SPvAQ|COEXlCgs5RPvC6gC&n^Ed9L>d!7Z z9Zl{pH~=s^P44UqU^HCy_XX*mAk8-3ZPO3GtP4_3IF}dB>B2cfs4ISJy(|B&PCxMq z`}%5LcvTl(wZ0l+oe^m;g8LUeoNM(LoZv$C0E-OxtP%{d#i*$_f~OygSGN$XF>30J zVCSNjW$B_U0TvkuRAgCW1nZ4((?YO`)#3DFkR|X?0)RyZYAO<#c}I+3*JJZ;G-}ow z!M%^I?kb~3GJ>Zbi&w{*zw5Dh4GizO$Kti2;YLlZ5j_44ni5`N1bZHvWrIO@>zBa` ztk8v4Cf^nWOhfH~HxbuT-+=<>RhwMv!Q5l~;?EFRo?jg1Gu)4$3rzv5bX~IeMO!R# zGcVaWFMI&G_9IKSoq+s$$>IZLS5f+P$WSuji-k99@l&O|==lNuKe8gmcuE!Z^c9@L z@zmKPP(2AL&>qrqkfwq?p{2m?a#0z)%mk-`XMFVXX!7bPu46J!C} zvty?Z7d(d!pX+=1kL=YuteQ`>4Rqu zpFDiR9z-Equ!6|r@Ys%nwN-2m# zN^0;T={8+Vj}djNY+<~D`BU%}(TOQsg$T8sfCQk?d^gn<<_;YkOpcGH226W{rYiXS zBf{H*Wsv0-9W^d#Ay_{(FmpZ^d@3J&YLN?iJLg)BhW7gn9rqeKjGC8?NYk|U3zxGo z^c4s2FK#Z}bo=71OH-F-B42r&jVmXQK!MS`=6+N3UQ^VlIbTZ9ObMDFO3=J=@+cBC zN~IFcmJ&R=g!n`zr$FxTQj*?06*B*MC^a1?r~ukOD<2 z10&q7vzv4#5^~YDqZ@D3A9QekbD;LXcIPJ>0tkPyT{y7I^T{qJg?IB5-b3NvIKl@T zoWH5(0Uy+2D`SjPcOPsF+E{E*QAuYqSV=@kBh3ic|6+tI!(BpzdBNo7uqkA#)F86c z_LVDdadsjn3srbl`Biu>8(zicSB2-U!n3x*Dm)KbLlc!{3@|oU;`Q zic&n3IASHrL?4?iLoZzbN6xsdviU607chg>YYC30D&4M;^M4|dsm1@$Kb zqQ9@;DGCy42-_xwP>GW0pyh zAff`}E!d1UX%;EGH6rvxm`t)1E+r`DqrQ&M|BdkeH$ayAR1DyJM!WVUEBebyr+e5p6825dUJGcTmI=ideh#VxGyj6)5U#;;QwCVjlSt^`MPLM z*q9eK>cU3D5BreG^$=@N@R{=Y0Oa#Q>aWrjMu5?RRe&(tIAs2Zg6sgZS;F9c$h}9# z6sPP=V}50XA9Awro`b>@*sf)l73(g>x^1zZVyvff1()dtX01gVGOp_F-ez_I3k(ck zm(3FsV#=EHY!%z;EqgN#)YgaASTq6ee!N>W0j)Dts$lKPP6IrbidY>`vG(n3*Mv7k zdi&y4UVRzlm?CRm=?hsO*|l`69GVYT2?D-vQhvsp2A!DjXWX+DhMZDa7%Sf;qV?>O zYvv=>y_@kYn~vtivguO9FBC5rjkrrzU)f7wrntaFa3X}7)Rt1nwMFL#hg>(x%k>js z+ZZ<3=U#CP8#8{n2{W@f6JB=2I!YywA+bDvVc4n||H`q?_$%v`31+9G(O7oENPLc z&R~m52YaM4rIql(q_q-xCcrv9u3bE$_Mt5gT;r*suALw91rcWdqzrY9DWy4*m*K(y zEsY|ly+@$O(Jn&cnRfI@51idwhSFZG$f)l5FlZ7>q5)-ctgfKfl?2)d*cm;OvaX>I zYXPG5rd7+WU4$Mp0dX-oc72O_g}xJMh3QL$xKz#^VkXp_u!1S-5QsT=M0CbkLVuaS zIZESWE+YdZk22v=eUUy93(2++^;;AoyK}Urg5r44~3QCQ>D ze#(S3wW@*CXsrf-c8$u*+(B68q}Nm@fs-TT&7cl2!qqnG;it`eCUqxmhO$SEQ1i^T zT&R8aTl$(U=A8>|$@UtN6*EJ*$hx@=ddEKV&PDcRk1rShnc%y5a=QMVrkReMuqH38 z(S ze*H95yXb*Rt4g``2Ok%V zSeZRCd31WnXjqkRh|axuuc2FS=$L*N8wqCWDw<)1HYt==-sr#}JHLsBfmq-}+^ZLf5tyxHAgoGdgtC4LVQa%h z*xGCnw%BikEtXpL{7}HV#q=UhtFS1rdb~ z5U*Wun*~6rhlxpY6r+G9>GQI01He^(fG<#+aM}(4&^~YQ?Uvgu(_3d9bLS^pa^CHE z@AgIRDOc#@=7mW8JFAR9{h}k(9KzHu0av8f_XP*=6@f|j;yTV7zAv`k6I*lQ%DlKz z7gri}O^aN_i(~+k2aQ1O+b`UH;nS7vca9pXR^Pc~NNe8@80>Y;Q|~{mw{OL5tX}ti zoB6qn!$R$A?z~|Dv6}&`%1A9L6si5fSrdfL0AR5UoK$WUOycdpIV<^rAhKxH7I~ zG$w9)?${uf!}O%;N~MsDCVQ)O8YH8=SLJf|i2VFEsmt0l?0xx!c?MF%Z5FTchVxLK z!6&F^=o|M?_#sTk(vq3IKA;3QU%|SbaKqWq+A!fD=`OWIuED(UJbGSCh#5ikLuQR! zW*fWef(l+Fsh-UkZ^k1>i1RkD317zhmOJB>Yaxx*z2{&OoPXJ|Z^*bY5{;FTXz|_J zu<6|C31rl#lL0h_TTY8s*R zQ){bTU`pyeI9(P`5%_#C?n?)ua*L;Wq}S43Rk=c1KzN-9M-T5mlyYLp5f`_s0#J`k$!V-Y4*5fJ$?v`V;EM#}3t@s-fyWt#_aP zSew)4-$0=1HiwF4h5B890)H*-9DOF1c4VZq6Wv6>7mcv}5T;uN?opUBfQ(giGoKWV zT+DHBLrcX6;TJT*dRF}_%FbnScVu40>4QWp1%I27f+Gn>_rbAoQywq4kk^P-7TMLO zfgTN%*+um~u=I2`gZSD)xD*;N%YuE$oVaYyN7g`O+jt>lA`Fz=>phw|%Ft}YRd^f) zitkMJ*NC$ROJH3lZxA%O&{7fq0Hp2I#bkJdshA|=2wbI%^hKYJv%xi`PL_xH8A&Xf7hli8!6?uEPS>A&znt81>H zyUlp!T6*$;VozS|(Z!zMA6j(6(E_$4U#Q6THWAYs&3mJZT%9X4cb+t|!P`N-abHe+ zJ}*A6i_cq8y+7`q`_9Mf_0IjdbqDh64(N@&=A9Fd=EbABc+^7e`SFJQnytAt+w*I- z>y16;ofDtSi_huea~9gZpRSqj{IGkz8|9hz!=;|fiRbeodEi>fDx%ug|Lw#NCcgjX zoi`V`0P))Zw_cb$G#y=N>&Ulto741HaXfR~r}Kbqr}J&6C;Q-7Q@`R#YkUmg2(=(nhK3EQY!t=H^^ zFkehwFSC+|awp z`H!1~-ff0bhqMAU~aIUon;Ll4!!J6plYmlN6h9_4#yE36-7ybLdg ziLYGJ7U7(&u%%R_sp>i8{-^9^@u4>QW*f_;mqT)`TqoDhH`wh@j~JE4CHi{lQE6HN z^Z)OtH2?9VLUbY8(q|edoco7|(=U(2Qe%l^DiOaR#o@1tvp#UhP0=Y;`8gOvpu(4~ z#^JwP;ap0`PU&P3ofNfrROt|oo5IyNJ`x*NY`h*xvY3(lmt-&aYyMqsm{|1nI8;_a zLCdaKGIx`^1#3`w`B$%D`$+tyW2ytK;%#L1LjnElUEV8iEmftU@Tb zP{nCVy}HzUw{EuX{S^pQ-Hg+OKi+>q=cso9pq8PfYMy~24zH+^$uhR!7p z8GiK*CGil4+shJqy3LX=ZM7;&73sQSptAgEQ^8lJMFl4aOd(Q^z`1E6Qw4YN=|46y zmQ+*BH?9zX@Jj7rZ>CHp&?-?sLcP^_0zW12cLc13MyG`@P^NI5y%1BGl62s@c`n=w zh3NoJP(Ty!hxm4xeBWauS|N~%4`0Oj;n;O@&h==qD+)#l9g|3+zMj~%i_|yKpedxO zRGtn{aE>KX>J|#QSpizXV~Sv!N?tFhiYb7hDwLlic1@*O$0R(BtSOf8B}vaeZwg?6 z$~B|@Gmu`Q36}-!`x-ND(IL9v@zSD~?oY*9@^YCT&xvh$u}v4-46#Fh=*AhsM)2Y; zD0NFGQ6zA%-U1p9Le+9k*Vc zdNq5*C{ZDzF)x1yFc*3yA9`g8A@S(aLSnA-Xuk7k_UJ;RlyB@Xm;T4s@XU1_%>yvXxmFm7JCv46On{{FH zzoPD2ay45%c2eDU=Y`$6u)C`6S=67IRBpDf+pML4uk1!`>LzW#EAdemMMV9~x>yx>#skZYI1%6KkWLYG`uJZ*A7_P|WUU9!X=ZI^9^ z)gzT(<90ker5|K58@VFE)392{Y?Xf}lv>ZJ5a>#(Esdjr-l25bh~B9PVmb7K3` z_0|v8kTf#{-zK?7Vy;kU$qN8GIZhRV7s5}9Xen4W)vHLQo&qSi;aHFTmmF`06Uhs) zMB3HWg_CjVkQN(Ky5W~;o{u~d<<*y{EPwIrWH*fQX&lBe?F1t-v%aR=&!m!LV|0Ko zHUt*}c4iMgV34W0r89~mrCsb-6dv&1=@sX}npKKZ#f<6PBoo)7E7>W#($5djX{dsy zbpG=ORoh(SiyuFCrUobhs)PTxx6-zumBp5|Tfa z`aFRj5Fo{F(TYqb76;ggMRg6u(vo2wY6b6Ta=;Wr7$broTsThORuiDIO@3&CLNu35 zBlPzXTH*&51UmJo{tG_7LHx`q3@Y@={>ZJJQ#*J0Y8ZaEd-LL61dMRaYT<~r1+t?b~~A@6GvF;209w ztRpOC_w0E+)Q$TWXMWxfHE~&UOF0;!hU}r6y`M!Irsa2%xybr_WIbHOT;b_;IalL- zq3xc~HY?+I5QMgz(32PFcM$07`kbrzzR-S8XrB}ArgWj*+BH8DLO1(xmTFp@**H6_ zKeO+5b-z3N+i(54|F`|WZvSn&F1|>9TrMa1fB`^7N{0&x__(%qv^`jxJ+e^SG{etq zyR&<4b#BGex!TTrEuDXGbu%#OHtL(n=_=qw1i<8>h1%AcZL{^WSLV{W+MaxE4>@zx zRQ_m%9$2{$Kr3gC&$Z=NKAQ{d$Om@lfgKBhmYH?4zPVR&(ym;fCm-n113gB#b}Bt{ z_Fj0kjuT)lZPRX}X~jFxUArRP3-v8V^J^ck{rhgc`8C|X*Z%3dW(aVhO}f*M)-^Pu zbq$SZT>~+l03ruBl0Ev0uDShseOKlE^Qe-W zOMpbq!rLfZ^k_d}VGwTVL=IbQuh1t@_+k#P()Z#T7@}Arz2@&eu_eLMu>2{{sLxz2 zRhVp{usqocDyJd3j?!;Yp0^47If3%`Df%u{d}k&7kYX!SRprW36XyG>%^r!jE97;Mq8KhFJag6#B0OenucmV1NJ5kWHx!dF{WFD#R;Z++E%?q{*I+d} zj9-hY`Z=cu;%@VIy!NX92bjcDYF!{Lut)IxqTu2M%vyjD$FJ47HI?_LTsX`A49=Hj ze+K8xvOlBzDVSw{1{cb*KZA?trQ061GVy2GpTVux?YF@->-HP=9fv>LGQCdcn{s?p zo^M)oIC&UD9tK!ZJegGZ&=*e5(QKgxki()Q$m17c9tJo+#W&#m6yHFJ{5M-nOsSE7 zZjrNtEk}7Cg!3VI&f()XFLHL!8sTw@$__R(^AIlW;Mq+)$eSGu@$0FDcJLDKqSSU! z(@KbT(6gTK?BFGSBfn~qvx9z~69SXzd(IZ!*<$$WCpS)?oj&`Q=iWJ&^R3MLNXUXA LO=!heMmqj4l*4)x literal 0 HcmV?d00001 diff --git a/benchmark/benchmark_parallel.py b/benchmark/benchmark_parallel.py new file mode 100755 index 0000000..8f773cc --- /dev/null +++ b/benchmark/benchmark_parallel.py @@ -0,0 +1,425 @@ +#!/usr/bin/env python3 +"""Benchmark sequential vs parallel mutant verification. + +The script runs `bcore-mutation analyze` for one or more cases, records wall +clock time, and writes a CSV plus PNG charts. It copies the input SQLite +database before every run so each case starts from the same mutant state. +""" + +from __future__ import annotations + +import argparse +import csv +import os +import platform +import shutil +import sqlite3 +import statistics +import subprocess +import sys +import time +from dataclasses import dataclass +from pathlib import Path + + +@dataclass(frozen=True) +class Case: + label: str + parallel: int + jobs: int + + +@dataclass +class ResultRow: + case: str + repeat: int + parallel: int + jobs: int + seconds: float + exit_code: int + status: str + killed: int | None + survived: int | None + errors: int | None + db_path: Path + stdout_log: Path + stderr_log: Path + + +def parse_case(value: str) -> Case: + parts = value.split(":") + if len(parts) != 3: + raise argparse.ArgumentTypeError( + "case must use LABEL:PARALLEL:JOBS, for example parallel-3x3:3:3" + ) + + label, parallel_raw, jobs_raw = parts + if not label: + raise argparse.ArgumentTypeError("case label cannot be empty") + + try: + parallel = int(parallel_raw) + jobs = int(jobs_raw) + except ValueError as exc: + raise argparse.ArgumentTypeError("parallel and jobs must be integers") from exc + + if parallel < 1: + raise argparse.ArgumentTypeError("parallel must be at least 1") + if jobs < 0: + raise argparse.ArgumentTypeError("jobs must be at least 0") + + return Case(label=label, parallel=parallel, jobs=jobs) + + +def parser() -> argparse.ArgumentParser: + argument_parser = argparse.ArgumentParser( + description="Benchmark bcore-mutation analyze sequential and parallel runs." + ) + argument_parser.add_argument("--db", default="mutation.db", help="SQLite DB path") + argument_parser.add_argument("--run-id", required=True, type=int, help="Mutation run ID") + argument_parser.add_argument( + "--command", + required=True, + help="Command passed to bcore-mutation analyze --command", + ) + argument_parser.add_argument( + "--setup-command", + default=None, + help="Optional command passed to bcore-mutation analyze --setup-command", + ) + argument_parser.add_argument( + "--timeout", + type=int, + default=300, + help="Timeout in seconds per mutant", + ) + argument_parser.add_argument( + "--case", + dest="cases", + action="append", + type=parse_case, + required=True, + help="Benchmark case as LABEL:PARALLEL:JOBS. Repeat for multiple cases.", + ) + argument_parser.add_argument( + "--repeats", + type=int, + default=3, + help="Number of repetitions per case", + ) + argument_parser.add_argument( + "--output-dir", + default="benchmark-results", + help="Directory for copied DBs, logs, CSV, and charts", + ) + argument_parser.add_argument( + "--bcore-mutation-bin", + default="bcore-mutation", + help="Path to bcore-mutation binary inside the benchmark environment", + ) + argument_parser.add_argument( + "--project", + default=None, + help="Optional --project value passed to analyze", + ) + argument_parser.add_argument( + "--file-path", + default=None, + help="Optional --file-path value passed to analyze", + ) + argument_parser.add_argument( + "--extra-analyze-arg", + action="append", + default=[], + help="Extra argument passed through to analyze. Repeat as needed.", + ) + argument_parser.add_argument( + "--keep-going", + action="store_true", + help="Continue remaining cases when one run fails", + ) + argument_parser.add_argument( + "--no-charts", + action="store_true", + help="Only write CSV and logs", + ) + return argument_parser + + +def run_git(args: list[str]) -> str: + completed = subprocess.run( + ["git", *args], + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + check=False, + ) + if completed.returncode != 0: + return "unknown" + return completed.stdout.strip() or "unknown" + + +def count_mutants(db_path: Path, run_id: int) -> tuple[int | None, int | None, int | None]: + try: + with sqlite3.connect(db_path) as connection: + killed = connection.execute( + "SELECT COUNT(*) FROM mutants WHERE run_id = ? AND status = 'killed'", + (run_id,), + ).fetchone()[0] + survived = connection.execute( + "SELECT COUNT(*) FROM mutants WHERE run_id = ? AND status = 'survived'", + (run_id,), + ).fetchone()[0] + errors = connection.execute( + "SELECT COUNT(*) FROM mutants WHERE run_id = ? AND status = 'error'", + (run_id,), + ).fetchone()[0] + return killed, survived, errors + except sqlite3.Error: + return None, None, None + + +def analyze_command(args: argparse.Namespace, case: Case, db_path: Path) -> list[str]: + command = [ + args.bcore_mutation_bin, + "analyze", + "--sqlite", + str(db_path), + "--run-id", + str(args.run_id), + "--timeout", + str(args.timeout), + "--parallel", + str(case.parallel), + "--jobs", + str(case.jobs), + "--command", + args.command, + ] + if args.project: + command.extend(["--project", args.project]) + if args.file_path: + command.extend(["--file-path", args.file_path]) + if args.setup_command: + command.extend(["--setup-command", args.setup_command]) + command.extend(args.extra_analyze_arg) + return command + + +def run_case(args: argparse.Namespace, case: Case, repeat: int, output_dir: Path) -> ResultRow: + run_name = f"{repeat:02d}-{case.label}" + db_path = output_dir / "db" / f"{run_name}.db" + stdout_log = output_dir / "logs" / f"{run_name}.stdout.log" + stderr_log = output_dir / "logs" / f"{run_name}.stderr.log" + + shutil.copy2(args.db, db_path) + command = analyze_command(args, case, db_path) + + print( + f"Running {case.label} repeat {repeat}: " + f"--parallel {case.parallel} --jobs {case.jobs}", + flush=True, + ) + start = time.perf_counter() + completed = subprocess.run( + command, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + seconds = time.perf_counter() - start + + stdout_log.write_text(completed.stdout, encoding="utf-8") + stderr_log.write_text(completed.stderr, encoding="utf-8") + killed, survived, errors = count_mutants(db_path, args.run_id) + status = "ok" if completed.returncode == 0 else "failed" + + print( + f"Finished {case.label} repeat {repeat}: " + f"{seconds:.2f}s status={status}", + flush=True, + ) + return ResultRow( + case=case.label, + repeat=repeat, + parallel=case.parallel, + jobs=case.jobs, + seconds=seconds, + exit_code=completed.returncode, + status=status, + killed=killed, + survived=survived, + errors=errors, + db_path=db_path, + stdout_log=stdout_log, + stderr_log=stderr_log, + ) + + +def write_csv(rows: list[ResultRow], path: Path) -> None: + with path.open("w", newline="", encoding="utf-8") as csv_file: + writer = csv.DictWriter( + csv_file, + fieldnames=[ + "case", + "repeat", + "parallel", + "jobs", + "seconds", + "exit_code", + "status", + "killed", + "survived", + "errors", + "db_path", + "stdout_log", + "stderr_log", + ], + ) + writer.writeheader() + for row in rows: + writer.writerow( + { + "case": row.case, + "repeat": row.repeat, + "parallel": row.parallel, + "jobs": row.jobs, + "seconds": f"{row.seconds:.6f}", + "exit_code": row.exit_code, + "status": row.status, + "killed": row.killed, + "survived": row.survived, + "errors": row.errors, + "db_path": row.db_path, + "stdout_log": row.stdout_log, + "stderr_log": row.stderr_log, + } + ) + + +def grouped_ok_rows(rows: list[ResultRow]) -> dict[str, list[ResultRow]]: + grouped: dict[str, list[ResultRow]] = {} + for row in rows: + if row.status == "ok": + grouped.setdefault(row.case, []).append(row) + return grouped + + +def write_summary(rows: list[ResultRow], path: Path) -> None: + grouped = grouped_ok_rows(rows) + baseline = grouped.get("sequential") or next(iter(grouped.values()), []) + baseline_median = ( + statistics.median(row.seconds for row in baseline) if baseline else None + ) + + with path.open("w", encoding="utf-8") as summary: + summary.write("# bcore-mutation parallel benchmark\n\n") + summary.write(f"- Commit: `{run_git(['rev-parse', 'HEAD'])}`\n") + summary.write(f"- Host: `{platform.platform()}`\n") + summary.write(f"- CPUs visible: `{os.cpu_count()}`\n\n") + summary.write("| Case | Parallel | Jobs | Runs | Median seconds | Speedup |\n") + summary.write("|------|----------|------|------|----------------|---------|\n") + for case, case_rows in grouped.items(): + median_seconds = statistics.median(row.seconds for row in case_rows) + speedup = ( + baseline_median / median_seconds + if baseline_median and median_seconds > 0 + else None + ) + first = case_rows[0] + speedup_text = f"{speedup:.2f}x" if speedup is not None else "n/a" + summary.write( + f"| {case} | {first.parallel} | {first.jobs} | {len(case_rows)} | " + f"{median_seconds:.2f} | {speedup_text} |\n" + ) + + +def write_charts(rows: list[ResultRow], output_dir: Path) -> None: + try: + import matplotlib + + matplotlib.use("Agg") + import matplotlib.pyplot as plt + except ImportError as exc: + raise RuntimeError( + "matplotlib is not installed; rerun with --no-charts or use the benchmark Docker image" + ) from exc + + grouped = grouped_ok_rows(rows) + if not grouped: + return + + labels = list(grouped) + medians = [statistics.median(row.seconds for row in grouped[label]) for label in labels] + baseline = grouped.get("sequential") or next(iter(grouped.values())) + baseline_median = statistics.median(row.seconds for row in baseline) + speedups = [baseline_median / seconds for seconds in medians] + + fig, ax = plt.subplots(figsize=(9, 4.8)) + ax.bar(labels, medians) + ax.set_ylabel("Median wall time (seconds)") + ax.set_title("Mutation verification time") + ax.tick_params(axis="x", rotation=20) + fig.tight_layout() + fig.savefig(output_dir / "time-by-case.png", dpi=160) + plt.close(fig) + + fig, ax = plt.subplots(figsize=(9, 4.8)) + ax.bar(labels, speedups) + ax.axhline(1.0, color="black", linewidth=1) + ax.set_ylabel("Speedup vs sequential") + ax.set_title("Parallel verification speedup") + ax.tick_params(axis="x", rotation=20) + fig.tight_layout() + fig.savefig(output_dir / "speedup-by-case.png", dpi=160) + plt.close(fig) + + +def main() -> int: + args = parser().parse_args() + if args.repeats < 1: + print("--repeats must be at least 1", file=sys.stderr) + return 2 + + db_path = Path(args.db) + if not db_path.exists(): + print(f"database not found: {db_path}", file=sys.stderr) + return 2 + args.db = db_path + + output_dir = Path(args.output_dir) + (output_dir / "db").mkdir(parents=True, exist_ok=True) + (output_dir / "logs").mkdir(parents=True, exist_ok=True) + + subprocess.run( + ["git", "config", "--global", "--add", "safe.directory", str(Path.cwd())], + check=False, + ) + + rows: list[ResultRow] = [] + failed = False + for repeat in range(1, args.repeats + 1): + for case in args.cases: + row = run_case(args, case, repeat, output_dir) + rows.append(row) + write_csv(rows, output_dir / "results.csv") + write_summary(rows, output_dir / "summary.md") + if row.status != "ok": + failed = True + if not args.keep_going: + print( + f"Stopping after failed run. See {row.stdout_log} and {row.stderr_log}.", + file=sys.stderr, + ) + return row.exit_code or 1 + + if not args.no_charts: + write_charts(rows, output_dir) + + print(f"Wrote benchmark results to {output_dir}") + return 1 if failed else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmark/run-in-docker.sh b/benchmark/run-in-docker.sh new file mode 100755 index 0000000..4707cfa --- /dev/null +++ b/benchmark/run-in-docker.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ $# -lt 1 ]]; then + echo "usage: benchmark/run-in-docker.sh SUBJECT_REPO [benchmark args...]" >&2 + exit 2 +fi + +subject_repo=$1 +shift + +image=${BCORE_BENCH_IMAGE:-bcore-mutation-bench} +cpus=${BCORE_BENCH_CPUS:-10} +results_dir=${BCORE_BENCH_RESULTS:-"$(pwd)/benchmark-results"} + +subject_repo=$(cd "$subject_repo" && pwd -P) +subject_parent=$(dirname "$subject_repo") +subject_name=$(basename "$subject_repo") +results_dir=$(mkdir -p "$results_dir" && cd "$results_dir" && pwd -P) + +docker build -f benchmark/Dockerfile -t "$image" . + +docker run --rm \ + --cpus="$cpus" \ + -v "$subject_parent:/bench" \ + -v "$results_dir:/results" \ + -w "/bench/$subject_name" \ + "$image" \ + --output-dir /results \ + "$@" +