Choose the scan version per benchmark in the bench orchestrator

bench-orchestrator/bench_orchestrator/cli.py

@@ -22,6 +22,7 @@
 from .comparison import analyzer
 from .comparison.reporter import pivot_comparison_table
 from .config import (
+    BENCHMARK_SCAN_V2,
     Benchmark,
     BenchmarkTarget,
     BuildConfig,
@@ -399,6 +400,8 @@ def run(
     console.print(f"\n[bold]Running {benchmark.value} benchmark[/bold]")
     console.print(f"  Targets: {', '.join(map(str, config.targets))}")
     console.print(f"  Iterations: {iterations}")
+    if any(target.engine in {Engine.DATAFUSION, Engine.DUCKDB} for target in config.targets):
+        console.print(f"  Vortex scan: {'V2' if BENCHMARK_SCAN_V2[benchmark] else 'V1'}")
     if label:
         console.print(f"  Label: {label}")
     console.print()

bench-orchestrator/bench_orchestrator/config.py

@@ -92,6 +92,25 @@ class Benchmark(Enum):
 }
 
 
+# Whether each benchmark runs its Vortex scans on the planner-driven V2 scan (`True`) or on V1.
+# Every benchmark is listed so its scan version is explicit.
+BENCHMARK_SCAN_V2: dict[Benchmark, bool] = {
+    Benchmark.APPIAN: False,
+    Benchmark.TPCH: True,
+    Benchmark.TPCDS: True,
+    Benchmark.CLICKBENCH: True,
+    Benchmark.CLICKBENCH_SORTED: False,
+    Benchmark.FINEWEB: False,
+    Benchmark.GHARCHIVE: False,
+    Benchmark.JOB: False,
+    Benchmark.POLARSIGNALS: False,
+    Benchmark.PUBLIC_BI: False,
+    Benchmark.STATPOPGEN: False,
+    Benchmark.SPATIALBENCH: False,
+    Benchmark.VORTEX_QUERIES: False,
+}
+
+
 def engines_for_benchmark(benchmark: Benchmark) -> frozenset[Engine]:
     """Return the engines `benchmark` supports, defaulting to every engine when unrestricted."""
     return BENCHMARK_ENGINES.get(benchmark, frozenset(Engine))

bench-orchestrator/bench_orchestrator/runner/executor.py

@@ -14,7 +14,7 @@
 from rich.console import Console
 from rich.progress import Progress, SpinnerColumn, TextColumn
 
-from ..config import Benchmark, Engine, Format
+from ..config import BENCHMARK_SCAN_V2, Benchmark, Engine, Format
 
 console = Console()
 
@@ -55,6 +55,7 @@ def build_command(
 
         if self.backend in {Engine.DATAFUSION, Engine.DUCKDB}:
             cmd.extend(["--formats", ",".join(fmt.value for fmt in formats)])
+            cmd.extend(["--scan-v2", str(BENCHMARK_SCAN_V2[benchmark]).lower()])
         if self.backend == Engine.DUCKDB:
             cmd.append("--delete-duckdb-database")
 

bench-orchestrator/tests/test_executor.py

@@ -5,7 +5,8 @@
 import textwrap
 from pathlib import Path
 
-from bench_orchestrator.config import Benchmark, Engine, Format
+import pytest
+from bench_orchestrator.config import BENCHMARK_SCAN_V2, Benchmark, Engine, Format
 from bench_orchestrator.runner.executor import BenchmarkExecutor
 
 
@@ -46,6 +47,20 @@ def test_build_command_serializes_vortex_spatial_native_format() -> None:
     assert cmd[cmd.index("--formats") + 1] == "parquet,vortex,vortex-spatial-native"
 
 
+def test_every_benchmark_has_a_scan_version() -> None:
+    assert set(BENCHMARK_SCAN_V2) == set(Benchmark)
+
+
+@pytest.mark.parametrize("engine", [Engine.DATAFUSION, Engine.DUCKDB])
+@pytest.mark.parametrize("benchmark", list(Benchmark))
+def test_build_command_selects_scan_version(engine: Engine, benchmark: Benchmark) -> None:
+    executor = BenchmarkExecutor(Path("/tmp/bench"), engine)
+
+    cmd = executor.build_command(benchmark=benchmark, formats=[Format.VORTEX])
+
+    assert cmd[cmd.index("--scan-v2") + 1] == str(BENCHMARK_SCAN_V2[benchmark]).lower()
+
+
 def test_build_command_omits_formats_for_lance_backend() -> None:
     executor = BenchmarkExecutor(Path("/tmp/lance-bench"), Engine.LANCE)
 
@@ -57,6 +72,7 @@ def test_build_command_omits_formats_for_lance_backend() -> None:
 
     assert cmd[0] == "/tmp/lance-bench"
     assert "--formats" not in cmd
+    assert "--scan-v2" not in cmd
     assert cmd[cmd.index("--queries") + 1] == "3"
 
 

benchmarks/datafusion-bench/src/main.rs

@@ -120,9 +120,9 @@ struct Args {
     #[arg(long, value_delimiter = ',', value_parser = value_parser!(Format))]
     formats: Vec<Format>,
 
-    /// Use the planner-driven V2 Vortex scan (defaults to true for TPC-H, TPC-DS and ClickBench).
-    #[arg(long, action = clap::ArgAction::Set)]
-    scan_v2: Option<bool>,
+    /// Use the planner-driven V2 Vortex scan.
+    #[arg(long, default_value_t = false, action = clap::ArgAction::Set)]
+    scan_v2: bool,
 
     #[arg(long = "opt", value_delimiter = ',', value_parser = value_parser!(Opt))]
     options: Vec<Opt>,
@@ -136,12 +136,7 @@ async fn main() -> anyhow::Result<()> {
     vortex_bench::configure_allocator();
     let args = Args::parse();
     let opts = Opts::from(args.options);
-    let scan_v2 = args.scan_v2.unwrap_or_else(|| {
-        matches!(
-            args.benchmark,
-            BenchmarkArg::TpcH | BenchmarkArg::TpcDS | BenchmarkArg::ClickBench
-        ) && !std::env::var("VORTEX_SCAN_V2").is_ok_and(|v| v == "0")
-    });
+    let scan_v2 = args.scan_v2;
 
     set_join_set_tracer(get_static_tracer())?;
     setup_logging_and_tracing(args.verbose, args.tracing)?;

benchmarks/duckdb-bench/src/main.rs

@@ -103,9 +103,9 @@ struct Args {
     )]
     reuse: bool,
 
-    /// Use the planner-driven V2 Vortex scan (defaults to true for TPC-H, TPC-DS and ClickBench).
-    #[arg(long, action = clap::ArgAction::Set)]
-    scan_v2: Option<bool>,
+    /// Use the planner-driven V2 Vortex scan.
+    #[arg(long, default_value_t = false, action = clap::ArgAction::Set)]
+    scan_v2: bool,
 }
 
 #[global_allocator]
@@ -116,13 +116,7 @@ fn main() -> anyhow::Result<()> {
     let args = Args::parse();
     let opts = Opts::from(args.options);
 
-    let scan_v2 = args.scan_v2.unwrap_or_else(|| {
-        matches!(
-            args.benchmark,
-            BenchmarkArg::TpcH | BenchmarkArg::TpcDS | BenchmarkArg::ClickBench
-        ) && !std::env::var("VORTEX_SCAN_V2").is_ok_and(|v| v == "0")
-    });
-    vortex_duckdb::set_scan_v2(scan_v2);
+    vortex_duckdb::set_scan_v2(args.scan_v2);
 
     setup_logging_and_tracing(args.verbose, args.tracing)?;