diff --git a/bench-orchestrator/bench_orchestrator/cli.py b/bench-orchestrator/bench_orchestrator/cli.py index 53952b02661..cd8654b9fe2 100644 --- a/bench-orchestrator/bench_orchestrator/cli.py +++ b/bench-orchestrator/bench_orchestrator/cli.py @@ -22,6 +22,7 @@ from .comparison import analyzer from .comparison.reporter import pivot_comparison_table from .config import ( + BENCHMARK_SCAN_V2, Benchmark, BenchmarkTarget, BuildConfig, @@ -399,6 +400,8 @@ def run( console.print(f"\n[bold]Running {benchmark.value} benchmark[/bold]") console.print(f" Targets: {', '.join(map(str, config.targets))}") console.print(f" Iterations: {iterations}") + if any(target.engine in {Engine.DATAFUSION, Engine.DUCKDB} for target in config.targets): + console.print(f" Vortex scan: {'V2' if BENCHMARK_SCAN_V2[benchmark] else 'V1'}") if label: console.print(f" Label: {label}") console.print() diff --git a/bench-orchestrator/bench_orchestrator/config.py b/bench-orchestrator/bench_orchestrator/config.py index 5d51fb8ecc3..920efd4813b 100644 --- a/bench-orchestrator/bench_orchestrator/config.py +++ b/bench-orchestrator/bench_orchestrator/config.py @@ -92,6 +92,25 @@ class Benchmark(Enum): } +# Whether each benchmark runs its Vortex scans on the planner-driven V2 scan (`True`) or on V1. +# Every benchmark is listed so its scan version is explicit. +BENCHMARK_SCAN_V2: dict[Benchmark, bool] = { + Benchmark.APPIAN: False, + Benchmark.TPCH: True, + Benchmark.TPCDS: True, + Benchmark.CLICKBENCH: True, + Benchmark.CLICKBENCH_SORTED: False, + Benchmark.FINEWEB: False, + Benchmark.GHARCHIVE: False, + Benchmark.JOB: False, + Benchmark.POLARSIGNALS: False, + Benchmark.PUBLIC_BI: False, + Benchmark.STATPOPGEN: False, + Benchmark.SPATIALBENCH: False, + Benchmark.VORTEX_QUERIES: False, +} + + def engines_for_benchmark(benchmark: Benchmark) -> frozenset[Engine]: """Return the engines `benchmark` supports, defaulting to every engine when unrestricted.""" return BENCHMARK_ENGINES.get(benchmark, frozenset(Engine)) diff --git a/bench-orchestrator/bench_orchestrator/runner/executor.py b/bench-orchestrator/bench_orchestrator/runner/executor.py index b7afee7e948..171a7c7479f 100644 --- a/bench-orchestrator/bench_orchestrator/runner/executor.py +++ b/bench-orchestrator/bench_orchestrator/runner/executor.py @@ -14,7 +14,7 @@ from rich.console import Console from rich.progress import Progress, SpinnerColumn, TextColumn -from ..config import Benchmark, Engine, Format +from ..config import BENCHMARK_SCAN_V2, Benchmark, Engine, Format console = Console() @@ -55,6 +55,7 @@ def build_command( if self.backend in {Engine.DATAFUSION, Engine.DUCKDB}: cmd.extend(["--formats", ",".join(fmt.value for fmt in formats)]) + cmd.extend(["--scan-v2", str(BENCHMARK_SCAN_V2[benchmark]).lower()]) if self.backend == Engine.DUCKDB: cmd.append("--delete-duckdb-database") diff --git a/bench-orchestrator/tests/test_executor.py b/bench-orchestrator/tests/test_executor.py index b8afb066e75..0d5cb38a0f0 100644 --- a/bench-orchestrator/tests/test_executor.py +++ b/bench-orchestrator/tests/test_executor.py @@ -5,7 +5,8 @@ import textwrap from pathlib import Path -from bench_orchestrator.config import Benchmark, Engine, Format +import pytest +from bench_orchestrator.config import BENCHMARK_SCAN_V2, Benchmark, Engine, Format from bench_orchestrator.runner.executor import BenchmarkExecutor @@ -46,6 +47,20 @@ def test_build_command_serializes_vortex_spatial_native_format() -> None: assert cmd[cmd.index("--formats") + 1] == "parquet,vortex,vortex-spatial-native" +def test_every_benchmark_has_a_scan_version() -> None: + assert set(BENCHMARK_SCAN_V2) == set(Benchmark) + + +@pytest.mark.parametrize("engine", [Engine.DATAFUSION, Engine.DUCKDB]) +@pytest.mark.parametrize("benchmark", list(Benchmark)) +def test_build_command_selects_scan_version(engine: Engine, benchmark: Benchmark) -> None: + executor = BenchmarkExecutor(Path("/tmp/bench"), engine) + + cmd = executor.build_command(benchmark=benchmark, formats=[Format.VORTEX]) + + assert cmd[cmd.index("--scan-v2") + 1] == str(BENCHMARK_SCAN_V2[benchmark]).lower() + + def test_build_command_omits_formats_for_lance_backend() -> None: executor = BenchmarkExecutor(Path("/tmp/lance-bench"), Engine.LANCE) @@ -57,6 +72,7 @@ def test_build_command_omits_formats_for_lance_backend() -> None: assert cmd[0] == "/tmp/lance-bench" assert "--formats" not in cmd + assert "--scan-v2" not in cmd assert cmd[cmd.index("--queries") + 1] == "3" diff --git a/benchmarks/datafusion-bench/src/main.rs b/benchmarks/datafusion-bench/src/main.rs index 3d3983f43ee..765b34fafcf 100644 --- a/benchmarks/datafusion-bench/src/main.rs +++ b/benchmarks/datafusion-bench/src/main.rs @@ -120,9 +120,9 @@ struct Args { #[arg(long, value_delimiter = ',', value_parser = value_parser!(Format))] formats: Vec, - /// Use the planner-driven V2 Vortex scan (defaults to true for TPC-H, TPC-DS and ClickBench). - #[arg(long, action = clap::ArgAction::Set)] - scan_v2: Option, + /// Use the planner-driven V2 Vortex scan. + #[arg(long, default_value_t = false, action = clap::ArgAction::Set)] + scan_v2: bool, #[arg(long = "opt", value_delimiter = ',', value_parser = value_parser!(Opt))] options: Vec, @@ -136,12 +136,7 @@ async fn main() -> anyhow::Result<()> { vortex_bench::configure_allocator(); let args = Args::parse(); let opts = Opts::from(args.options); - let scan_v2 = args.scan_v2.unwrap_or_else(|| { - matches!( - args.benchmark, - BenchmarkArg::TpcH | BenchmarkArg::TpcDS | BenchmarkArg::ClickBench - ) && !std::env::var("VORTEX_SCAN_V2").is_ok_and(|v| v == "0") - }); + let scan_v2 = args.scan_v2; set_join_set_tracer(get_static_tracer())?; setup_logging_and_tracing(args.verbose, args.tracing)?; diff --git a/benchmarks/duckdb-bench/src/main.rs b/benchmarks/duckdb-bench/src/main.rs index 903cd958b54..5a9a0f2f4c9 100644 --- a/benchmarks/duckdb-bench/src/main.rs +++ b/benchmarks/duckdb-bench/src/main.rs @@ -103,9 +103,9 @@ struct Args { )] reuse: bool, - /// Use the planner-driven V2 Vortex scan (defaults to true for TPC-H, TPC-DS and ClickBench). - #[arg(long, action = clap::ArgAction::Set)] - scan_v2: Option, + /// Use the planner-driven V2 Vortex scan. + #[arg(long, default_value_t = false, action = clap::ArgAction::Set)] + scan_v2: bool, } #[global_allocator] @@ -116,13 +116,7 @@ fn main() -> anyhow::Result<()> { let args = Args::parse(); let opts = Opts::from(args.options); - let scan_v2 = args.scan_v2.unwrap_or_else(|| { - matches!( - args.benchmark, - BenchmarkArg::TpcH | BenchmarkArg::TpcDS | BenchmarkArg::ClickBench - ) && !std::env::var("VORTEX_SCAN_V2").is_ok_and(|v| v == "0") - }); - vortex_duckdb::set_scan_v2(scan_v2); + vortex_duckdb::set_scan_v2(args.scan_v2); setup_logging_and_tracing(args.verbose, args.tracing)?;