Skip to content
Linters and Tests

Default DuckDB to Scan V2 and optimize sparse and wide scans #43781

Sign in to view logs

Default DuckDB to Scan V2 and optimize sparse and wide scans

Default DuckDB to Scan V2 and optimize sparse and wide scans #43781

Workflow file for this run

.github/workflows/ci.yml at 7dcaf56
name: Linters and Tests
# Concurrency control:
# - PRs: new commits on a feature branch will cancel in-progress (outdated) runs.
# - Push to develop: runs queue sequentially, never cancelled.
# - `workflow_dispatch`: groups by branch and queues if run on develop.
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: ${{ github.ref != 'refs/heads/develop' }}
on:
push:
branches: [develop]
pull_request: { }
workflow_dispatch: { }
permissions:
actions: read
contents: read
checks: write # audit-check creates checks
issues: write # audit-check creates issues
env:
CARGO_TERM_COLOR: always
RUST_BACKTRACE: 1
NIGHTLY_TOOLCHAIN: nightly-2026-09-10
jobs:
duckdb-mirror:
name: "Mirror DuckDB to R2"
if: github.event_name == 'pull_request'
uses: ./.github/workflows/duckdb-r2.yml
secrets: inherit
lightweight-checks:
name: "Formatting, Python lint and script tests"
runs-on: >-
${{ github.repository == 'vortex-data/vortex'
&& format('runs-on={0}/region=us-east-1/runner=amd64-ci-small/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=lightweight-checks', github.run_id)
|| 'ubuntu-latest' }}
timeout-minutes: 10
steps:
- parallel:
- uses: runs-on/action@v2
if: github.repository == 'vortex-data/vortex'
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
- uses: spiraldb/actions/.github/actions/setup-uv@a746510eafaa926484c354541cfc49b2ec06cc63 # 0.18.6
with:
sync: false
- name: Install missing formatting tools
run: |
packages=()
command -v yamllint >/dev/null || packages+=(yamllint)
command -v clang-format >/dev/null || packages+=(clang-format)
if [ ${#packages[@]} -gt 0 ]; then
sudo apt-get update
sudo apt-get install -y "${packages[@]}"
fi
- name: TOML Lint
if: "!cancelled()"
uses: spiraldb/actions/.github/actions/lint-toml@a746510eafaa926484c354541cfc49b2ec06cc63 # 0.18.6
- name: Validate YAML file
if: "!cancelled()"
run: |
# Lint the workflows and yamllint's configuration file.
yamllint \
--strict \
-c .yamllint.yaml \
.github/
- name: C/C++ Lint - clang-format
if: "!cancelled()"
run: |
git ls-files lang/cpp vortex-cuda vortex-duckdb vortex-ffi \
| grep -E '\.(cpp|hpp|cu|cuh|h)$' \
| grep -v 'arrow/reference/arrow_c_device\.h$' \
| grep -v 'kernels/src/bit_unpack_.*\.cu$' \
| grep -v 'kernels/src/bit_unpack_.*_lanes\.cuh$' \
| xargs clang-format --dry-run --Werror --style=file
# Use uvx for ruff to avoid building the Rust extension (saves ~4.5 min)
- name: Python Lint - Format
if: "!cancelled()"
run: uvx ruff format --check .
- name: Python Lint - Ruff
if: "!cancelled()"
run: uvx ruff check .
# ty needs the Python dependencies for type information, but resolves the
# vortex package itself from the in-tree sources and stubs, so skip the
# Rust extension build entirely.
- name: Python Lint - ty
if: "!cancelled()"
run: |
uv sync --all-packages --no-install-workspace
uvx ty check vortex-python vortex-python-cuda vortex-ffi/cmake/tests scripts/tests
# Pins the measurement_id hash that scripts/post-ingest.py uses as the primary key of every
# benchmarks-database fact row. The golden vectors are a frozen artifact (see the note inside
# the JSON); this job fails if a change to scripts/_measurement_id.py drifts the hash, which
# would break ON CONFLICT idempotency against the rows already in production.
- name: Pytest - measurement_id, benchmark reporting and ingestion
if: "!cancelled()"
run: |
uv run --no-project --with pytest --with pandas --with numpy --with orjson \
--with tabulate --with xxhash \
pytest scripts/tests/test_measurement_id.py \
scripts/tests/test_benchmark_reporting.py scripts/tests/test_post_ingest.py
python-test:
name: "Python (test)"
runs-on: >-
${{ github.repository == 'vortex-data/vortex'
&& format('runs-on={0}/region=us-east-1/runner=amd64-ci-large/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=python-test', github.run_id)
|| 'ubuntu-latest' }}
timeout-minutes: 30
env:
RUST_LOG: "info,maturin=off,uv=debug"
MATURIN_PEP517_ARGS: "--profile ci"
steps:
- parallel:
- uses: runs-on/action@v2
if: github.repository == 'vortex-data/vortex'
with:
sccache: s3
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
- uses: ./.github/actions/setup-prebuild
with:
enable-sccache: "true"
- name: Pytest - Vortex
run: |
uv run --all-packages pytest --benchmark-disable -n auto test/
working-directory: vortex-python/
# The datasets integration relies on private `datasets` APIs, so exercise the oldest
# supported release in addition to the locked one.
- name: Pytest - Vortex (datasets 4.x floor)
run: |
uv run --all-packages --with 'datasets==4.0.0' --with 'huggingface-hub<1' \
pytest --benchmark-disable test/test_hf_datasets.py
working-directory: vortex-python/
# The lockfile tracks Polars 2, so also exercise the oldest supported 1.x release. The
# Polars integration parses Polars' serialized expression format, which differs across
# versions.
- name: Pytest - Vortex (Polars 1.x floor)
run: |
uv run --all-packages --with 'polars==1.32.0' \
pytest --benchmark-disable test/test_polars_.py test/test_dataset.py
working-directory: vortex-python/
- name: Check Python documentation
run: |
uv run --all-packages make check
working-directory: docs/
python-windows:
name: "Python (windows-x64)"
timeout-minutes: 60
runs-on: >-
${{ github.repository == 'vortex-data/vortex'
&& format('runs-on={0}/region=us-east-1/pool=windows-x64-pre/extras=s3-cache', github.run_id)
|| 'windows-latest' }}
steps:
- uses: runs-on/action@v2
if: github.repository == 'vortex-data/vortex'
with:
sccache: s3
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
- name: Setup (Windows)
run: |
echo "C:\rust\cargo\bin" >> $env:GITHUB_PATH
- uses: ./.github/actions/setup-prebuild
with:
enable-sccache: "true"
- uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7
with:
python-version: "3.11"
# Mirrors the wheel build in package.yml, which spells the Windows interpreter
# `python` because maturin has no `python3.11` executable to resolve there.
- name: Build wheel
uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1
with:
command: build
maturin-version: 1.15.0
target: x86_64-pc-windows-msvc
args: >
--profile ci
--manifest-path vortex-python/Cargo.toml
--out dist
--interpreter python
env:
# Keep this constant to avoid pyo3 invalidating itself
PYO3_ENVIRONMENT_SIGNATURE: "cpython3"
# Also applies to host proc-macro links when Cargo receives an explicit target.
LINK: /Brepro
# rust-test-windows excludes vortex-python, so this is the only check that the
# win_amd64 wheel imports and can round-trip a file. Runs outside the checkout so
# the repo-root `vortex/` crate directory cannot shadow the installed package.
- name: Import and round-trip the wheel
shell: bash
run: |
python -m pip install dist/*.whl
cd "$RUNNER_TEMP"
python - <<'PY'
import pyarrow as pa
import vortex as vx
table = pa.table({"a": [1, 2, 3], "b": ["x", "y", "z"]})
vx.io.write(table, "smoke.vortex")
# Vortex reads utf8 back as string_view, so compare column-wise rather than by table.
round_tripped = vx.open("smoke.vortex").to_arrow().read_all()
assert round_tripped.column_names == table.column_names
assert round_tripped.column("a").to_pylist() == [1, 2, 3]
assert round_tripped.column("b").to_pylist() == ["x", "y", "z"]
print("vortex", vx.__version__, "round-tripped a file on Windows")
PY
python-cuda-test:
name: "Python CUDA (test)"
if: github.repository == 'vortex-data/vortex'
runs-on: runs-on=${{ github.run_id }}/region=us-east-1/runner=gpu/extras=s3-cache/tag=python-cuda-test
timeout-minutes: 30
env:
RUST_LOG: "info,maturin=off,uv=debug"
MATURIN_PEP517_ARGS: "--profile ci"
steps:
- parallel:
- uses: runs-on/action@v2
with:
sccache: s3
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
- parallel:
- uses: ./.github/actions/setup-rust
with:
repo-token: ${{ secrets.GITHUB_TOKEN }}
components: cargo
enable-sccache: "true"
- name: Install uv
uses: spiraldb/actions/.github/actions/setup-uv@a746510eafaa926484c354541cfc49b2ec06cc63 # 0.18.6
with:
sync: false
- name: Pin rustup proxy to repository toolchain
run: |
TOOLCHAIN="$(grep '^channel' rust-toolchain.toml | cut -d '"' -f 2)"
echo "RUSTUP_TOOLCHAIN=$TOOLCHAIN" >> "$GITHUB_ENV"
- name: Pytest - PyVortex CUDA bridge
run: |
uv run --extra cuda \
pytest --benchmark-disable ../vortex-python-cuda/test/test_native_bridge.py
working-directory: vortex-python/
rust-docs:
name: "Rust (docs)"
needs: duckdb-mirror
# The PR-only mirror is a skipped ancestor on develop; override implicit success().
if: ${{ !cancelled() && (needs.duckdb-mirror.result == 'success' || needs.duckdb-mirror.result == 'skipped') }}
timeout-minutes: 30
runs-on: >-
${{ github.repository == 'vortex-data/vortex'
&& format('runs-on={0}/region=us-east-1/runner=amd64-ci-docs/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=rust-docs', github.run_id)
|| 'ubuntu-latest' }}
steps:
- parallel:
- uses: runs-on/action@v2
if: github.repository == 'vortex-data/vortex'
with:
sccache: s3
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
- uses: ./.github/actions/setup-prebuild
with:
enable-sccache: "true"
- name: Docs
env:
# required to make sure docs build for features
RUSTDOCFLAGS: "-D warnings --cfg docsrs"
run: |
# Build docs for the whole workspace (aside from vortex-python that is having hard time with the python-docs syntax),
# including all private docs.
cargo doc --profile ci --no-deps --document-private-items --workspace --exclude vortex-python --exclude vortex-python-cuda
# nextest doesn't support doc tests, so we run it here
cargo test --profile ci --doc --workspace --all-features --exclude vortex-jni --exclude vortex-ffi --exclude xtask --exclude vortex-python-cuda --no-fail-fast
build-rust:
name: "Rust build (${{matrix.config.name}})"
timeout-minutes: 30
runs-on: >-
${{ github.repository == 'vortex-data/vortex'
&& format('runs-on={0}/region=us-east-1/runner={1}/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag={2}', github.run_id, matrix.config.runner, matrix.config.name)
|| 'ubuntu-latest' }}
strategy:
fail-fast: false
matrix:
config:
- name: "all-features"
runner: amd64-ci-large
args: "--all-features --all-targets"
- name: "default features"
runner: amd64-ci-large
args: "--all-targets"
- name: "with tokio dispatcher"
runner: amd64-ci-small
# Only build the crates that have the tokio features, not re-building other crates with no-default-features
args: "--no-default-features --features tokio --all-targets -p vortex -p vortex-io -p vortex-file -p vortex-layout"
steps:
- parallel:
- uses: runs-on/action@v2
if: github.repository == 'vortex-data/vortex'
with:
sccache: s3
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
- uses: ./.github/actions/setup-prebuild
with:
enable-sccache: "true"
# Feature-unification errors appear at type check. Codegen and linking add most of the
# time and sccache cannot cache links; the test jobs link the workspace already.
- uses: ./.github/actions/check-rebuild
with:
command: "cargo hack check --profile ci --locked ${{matrix.config.args}} --ignore-private"
- name: "Make sure no files changed after build"
run: |
git status --porcelain
test -z "$(git status --porcelain)"
- name: sccache stats
if: always() && github.repository == 'vortex-data/vortex'
run: sccache --show-stats
check-min-deps:
name: "Check build with minimal dependencies"
timeout-minutes: 30
runs-on: >-
${{ github.repository == 'vortex-data/vortex'
&& format('runs-on={0}/region=us-east-1/runner=amd64-ci-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=rust-min-deps', github.run_id)
|| 'ubuntu-latest' }}
steps:
- parallel:
- uses: runs-on/action@v2
if: github.repository == 'vortex-data/vortex'
with:
sccache: s3
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
- uses: ./.github/actions/setup-prebuild
with:
enable-sccache: "true"
- run: cargo minimal-versions check --direct --workspace --ignore-private
rust-msrv:
name: "Rust (MSRV)"
timeout-minutes: 30
runs-on: >-
${{ github.repository == 'vortex-data/vortex'
&& format('runs-on={0}/region=us-east-1/runner=amd64-ci-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=rust-msrv', github.run_id)
|| 'ubuntu-latest' }}
env:
RUSTFLAGS: "-A warnings"
steps:
- parallel:
- uses: runs-on/action@v2
if: github.repository == 'vortex-data/vortex'
with:
sccache: s3
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
- uses: ./.github/actions/setup-prebuild
with:
enable-sccache: "true"
- name: Check publishable crates against the MSRV
run: cargo hack --rust-version --no-dev-deps --ignore-private check --profile ci --all-features
rust-lint:
name: "Rust (lint)"
needs: duckdb-mirror
if: ${{ !cancelled() && (needs.duckdb-mirror.result == 'success' || needs.duckdb-mirror.result == 'skipped') }}
timeout-minutes: 30
runs-on: >-
${{ github.repository == 'vortex-data/vortex'
&& format('runs-on={0}/region=us-east-1/runner=amd64-ci-large/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=rust-lint', github.run_id)
|| 'ubuntu-latest' }}
steps:
- parallel:
- uses: runs-on/action@v2
if: github.repository == 'vortex-data/vortex'
with:
sccache: s3
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
- uses: ./.github/actions/setup-prebuild
with:
enable-sccache: "true"
- name: Install nightly for fmt
run: rustup toolchain install $NIGHTLY_TOOLCHAIN --component rustfmt
# Cargo locks each profile's target directory, so the release check runs
# alongside clippy while the two ci-profile clippy runs take turns.
- parallel:
- name: Rust Lint - Format
id: fmt
continue-on-error: true
run: cargo +$NIGHTLY_TOOLCHAIN fmt --all --check
- name: Rustc check (release)
id: check-release
continue-on-error: true
run: RUSTFLAGS="-D warnings" cargo check --locked --all-features --all-targets --release
- name: Rust Lint - Clippy All Features
id: clippy-all
continue-on-error: true
run: cargo clippy --profile ci --locked --all-features --all-targets -- -D warnings
- name: Rust Lint - Clippy Default Features
id: clippy-default
continue-on-error: true
run: cargo clippy --profile ci --locked --all-targets -- -D warnings
- name: Check lint results
if: always()
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9
with:
script: |
const failed = Object.entries({
fmt: '${{ steps.fmt.outcome }}',
'check-release': '${{ steps.check-release.outcome }}',
'clippy-all': '${{ steps.clippy-all.outcome }}',
'clippy-default': '${{ steps.clippy-default.outcome }}',
}).filter(([, o]) => o === 'failure').map(([n]) => n);
if (!failed.length) return;
const { data: { jobs } } = await github.rest.actions.listJobsForWorkflowRun({
...context.repo, run_id: context.runId,
});
const url = jobs.find(j => j.name === context.job)?.check_run_url;
if (url) {
await github.rest.checks.update({
...context.repo,
check_run_id: Number(url.split('/').pop()),
output: { title: `${failed.join(', ')} - failing`, summary: '' },
});
}
core.setFailed(`Lint failed: ${failed.join(', ')}`);
- name: sccache stats
if: always() && github.repository == 'vortex-data/vortex'
run: sccache --show-stats
rust-lint-no-default:
name: "Rust (lint, no default)"
timeout-minutes: 30
runs-on: >-
${{ github.repository == 'vortex-data/vortex'
&& format('runs-on={0}/region=us-east-1/runner=amd64-ci-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=rust-lint-no-default', github.run_id)
|| 'ubuntu-latest' }}
steps:
- parallel:
- uses: runs-on/action@v2
if: github.repository == 'vortex-data/vortex'
with:
sccache: s3
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
- uses: ./.github/actions/setup-prebuild
with:
enable-sccache: "true"
- name: Rust Lint - Clippy No Default Features
shell: bash
run: |
cargo hack --no-dev-deps --ignore-private clippy --profile ci --no-default-features -- -D warnings
rust-test-windows:
name: "Rust tests (windows-x64)"
env:
# Stable proc-macro DLL contents let dependent crates reuse sccache entries.
# duckdb_static.lib is built with /MT and thus requires crt-static
RUSTFLAGS: "-C link-arg=/Brepro -C target-feature=+crt-static"
# link.exe spends ~1.5 min on the test binaries after compilation finishes.
CARGO_TARGET_X86_64_PC_WINDOWS_MSVC_LINKER: rust-lld
needs: duckdb-mirror
if: ${{ !cancelled() && (needs.duckdb-mirror.result == 'success' || needs.duckdb-mirror.result == 'skipped') }}
timeout-minutes: 30
runs-on: >-
${{ github.repository == 'vortex-data/vortex'
&& format('runs-on={0}/region=us-east-1/pool=windows-x64-pre/extras=s3-cache', github.run_id)
|| 'windows-latest' }}
steps:
- parallel:
- uses: runs-on/action@v2
if: github.repository == 'vortex-data/vortex'
with:
sccache: s3
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
- name: Setup (Windows)
run: |
echo "C:\rust\cargo\bin" >> $env:GITHUB_PATH
- uses: ./.github/actions/setup-prebuild
with:
enable-sccache: "true"
- name: Rust Tests (Windows)
run: |
cargo nextest run --cargo-profile ci --locked --workspace --all-features --no-fail-fast `
--exclude vortex-bench `
--exclude vortex-python `
--exclude vortex-fuzz --exclude vortex-cuda --exclude vortex-cuda-ffi `
--exclude vortex-nvcomp --exclude vortex-cub --exclude vortex-test-e2e-cuda --exclude vortex-python-cuda `
--exclude duckdb-bench `
--exclude lance-bench --exclude datafusion-bench --exclude random-access-bench `
--exclude compress-bench --exclude xtask --exclude vortex-datafusion `
--exclude gpu-scan-cli
- name: sccache stats
if: always() && github.repository == 'vortex-data/vortex'
run: sccache --show-stats
- name: Alert incident.io
if: failure() && github.event_name == 'push' && github.ref == 'refs/heads/develop'
uses: ./.github/actions/alert-incident-io
with:
api-key: ${{ secrets.INCIDENT_IO_ALERT_TOKEN }}
alert-title: "Rust tests (windows-x64) failed on develop"
deduplication-key: ci-rust-test-windows-x64-failure
rust-test-linux-arm64:
name: "Rust tests (linux-arm64)"
needs: duckdb-mirror
if: >-
!cancelled() && (needs.duckdb-mirror.result == 'success' || needs.duckdb-mirror.result == 'skipped') &&
github.repository == 'vortex-data/vortex'
timeout-minutes: 30
runs-on: runs-on=${{ github.run_id }}/region=us-east-1/runner=arm64-ci-test/image=ubuntu24-full-arm64-pre-v2/extras=s3-cache/tag=rust-test-linux-arm64
steps:
- parallel:
- uses: runs-on/action@v2
with:
sccache: s3
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
- uses: ./.github/actions/setup-prebuild
with:
enable-sccache: "true"
- name: Rust Tests
run: |
cargo nextest run --cargo-profile ci --locked --workspace --all-features --no-fail-fast --exclude vortex-bench --exclude xtask --exclude vortex-sqllogictest
- uses: ./.github/actions/check-rebuild
with:
command: "cargo test --profile ci --locked --workspace --all-features --no-run --exclude vortex-bench --exclude xtask --exclude vortex-sqllogictest"
- name: Alert incident.io
if: failure() && github.event_name == 'push' && github.ref == 'refs/heads/develop'
uses: ./.github/actions/alert-incident-io
with:
api-key: ${{ secrets.INCIDENT_IO_ALERT_TOKEN }}
alert-title: "Rust tests (linux-arm64) failed on develop"
deduplication-key: ci-rust-test-linux-arm64-failure
# Runs the crates with hand-written SIMD kernels on EC2 instance families chosen for
# their CPU features, so that every runtime-dispatched kernel tier is exercised:
# c6a (AMD Zen 3): AVX2 and BMI2, no AVX-512
# c6i (Intel Ice Lake): AVX-512 F/BW/VBMI2/VPOPCNTDQ
# c8g (Graviton4, Neoverse V2): SVE2 with BITPERM
# `VORTEX_EXPECT_CPU_FEATURES` lists every feature the instance must have; vortex-array-macros'
# `cpu_features` test fails if any of them is missing or any other known feature (e.g.
# AVX-512 on the avx2 entry) is detected.
rust-test-cpu-features:
name: "Rust tests (cpu-features, ${{ matrix.config.name }})"
if: github.repository == 'vortex-data/vortex'
timeout-minutes: 30
strategy:
fail-fast: false
matrix:
config:
- name: avx2
runner: amd64-ci-medium
family: c6a.2xlarge
image: ubuntu24-full-x64-pre-v2
expect: "sse2,popcnt,bmi1,bmi2,avx,avx2"
- name: avx512
runner: amd64-ci-medium
family: c6i.2xlarge
image: ubuntu24-full-x64-pre-v2
expect: "sse2,popcnt,bmi1,bmi2,avx,avx2,avx512f,avx512bw,avx512vbmi2,avx512vpopcntdq"
- name: sve
runner: arm64-medium
family: c8g.2xlarge
image: ubuntu24-full-arm64-pre-v2
expect: "neon,sve,sve2,sve2-bitperm"
runs-on: >-
runs-on=${{ github.run_id }}/region=us-east-1/runner=${{ matrix.config.runner }}/family=${{ matrix.config.family }}/spot=price-capacity-optimized/image=${{ matrix.config.image }}/extras=s3-cache/tag=rust-test-cpu-features-${{ matrix.config.name }}
env:
VORTEX_EXPECT_CPU_FEATURES: ${{ matrix.config.expect }}
steps:
- parallel:
- uses: runs-on/action@v2
with:
sccache: s3
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
- uses: ./.github/actions/setup-prebuild
with:
enable-sccache: "true"
# Print the detected features and fail fast if the instance is not the expected tier.
- name: Check CPU features
run: |
cargo nextest run --cargo-profile ci --locked --no-capture \
-p vortex-array-macros --test cpu_features
- name: Rust Tests
run: |
cargo nextest run --cargo-profile ci --locked --no-fail-fast \
-p vortex-buffer -p vortex-mask -p vortex-array
build-java:
name: "Java"
runs-on: >-
${{ github.repository == 'vortex-data/vortex'
&& format('runs-on={0}/region=us-east-1/runner=amd64-ci-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=java', github.run_id)
|| 'ubuntu-latest' }}
timeout-minutes: 30
steps:
- parallel:
- uses: runs-on/action@v2
if: github.repository == 'vortex-data/vortex'
with:
sccache: s3
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
- parallel:
- uses: ./.github/actions/setup-prebuild
with:
enable-sccache: "true"
- uses: gradle/actions/setup-gradle@3f5f9adaf7d9fecd50b5935e54106014257a94e6 # v6
- name: Build documentation and run Java checks
run: ./gradlew --parallel javadoc check
working-directory: ./java
license-check-and-audit-check:
# The name is a required status check on develop; keep it unchanged.
name: License Check and Audit Check (bans licenses sources)
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
# Prevent sudden announcement of a new advisory from failing ci:
- name: cargo deny check advisories
id: advisories
continue-on-error: true
uses: EmbarkStudios/cargo-deny-action@3c6349835b2b7b196a839186cb8b78e02f7b5f25 # v2
with:
command: check advisories
- name: Warn on advisory failures
if: steps.advisories.outcome == 'failure'
run: >-
echo "::warning title=cargo deny advisories::cargo deny check advisories
failed; see the 'cargo deny check advisories' step log."
- name: cargo deny check bans licenses sources
uses: EmbarkStudios/cargo-deny-action@3c6349835b2b7b196a839186cb8b78e02f7b5f25 # v2
with:
command: check bans licenses sources
sqllogic-test:
name: "SQL logic tests"
needs: duckdb-mirror
if: ${{ !cancelled() && (needs.duckdb-mirror.result == 'success' || needs.duckdb-mirror.result == 'skipped') }}
runs-on: >-
${{ github.repository == 'vortex-data/vortex'
&& format('runs-on={0}/region=us-east-1/runner=amd64-ci-sql-logic-test/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=sql-logic-test', github.run_id)
|| 'ubuntu-latest' }}
timeout-minutes: 30
steps:
- parallel:
- uses: runs-on/action@v2
if: github.repository == 'vortex-data/vortex'
with:
sccache: s3
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
- uses: ./.github/actions/setup-prebuild
with:
enable-sccache: "true"
- name: Run sqllogictest tests
run: |
VORTEX_SLT_PROFILE=ci ./vortex-sqllogictest/slt/generate_data.sh
cargo nextest run --cargo-profile ci -p vortex-sqllogictest
- uses: ./.github/actions/check-rebuild
with:
command: "cargo nextest run --cargo-profile ci -p vortex-sqllogictest --no-run"
message-format-flag: "--cargo-message-format"
c-cxx-generated:
name: "Generated sources, C/C++ API and CMake integration"
# Leave room for cold, per-build-tree Cargo artifacts, including sanitized std.
timeout-minutes: 30
runs-on: >-
${{ github.repository == 'vortex-data/vortex'
&& format('runs-on={0}/region=us-east-1/runner=amd64-ci-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=c-cxx-generated', github.run_id)
|| 'ubuntu-latest' }}
steps:
- parallel:
- uses: runs-on/action@v2
if: github.repository == 'vortex-data/vortex'
with:
sccache: s3
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
# check-editions compares against the merge base, so it needs real history.
fetch-depth: 0
- uses: ./.github/actions/setup-prebuild
with:
enable-sccache: "true"
- parallel:
# The only job that needs flatc: builds use the checked-in bindings.
- uses: ./.github/actions/setup-flatc
- name: Install nightly for cbindgen macro expansion
run: |
rustup toolchain install $NIGHTLY_TOOLCHAIN \
--component rust-src,rustfmt,clippy,llvm-tools-preview
- name: Install Ninja for CMake integration tests
run: |
sudo apt-get update
sudo apt-get install -y ninja-build
- parallel:
- name: "regenerate the FlatBuffers and Protocol Buffers bindings"
# Fails on a flatc version mismatch, and the porcelain check below fails on stale bindings.
run: |
cargo run --profile ci -p xtask -- generate-flatbuffers
cargo run --profile ci -p xtask -- generate-proto
- name: "regenerate FFI header and test the C API"
env:
# Nightly regenerates cinclude/vortex.h with cbindgen during the build.
RUSTUP_TOOLCHAIN: ${{ env.NIGHTLY_TOOLCHAIN }}
run: |
cmake -S vortex-ffi -B "${RUNNER_TEMP}/ffi-c-build" \
-DCMAKE_LINKER_TYPE=MOLD \
-DBUILD_SHARED_LIBS=ON \
-DVORTEX_CARGO_PROFILE=ci \
-DVORTEX_BUILD_TESTS=ON
cmake --build "${RUNNER_TEMP}/ffi-c-build" --parallel "$(nproc)"
ctest --test-dir "${RUNNER_TEMP}/ffi-c-build" --parallel "$(nproc)" --output-on-failure
# Populate the registry cache for the lockfile-selected offline fixtures.
- name: Fetch Cargo dependencies for offline fixtures
run: cargo fetch --locked
- name: "regenerate the edition records"
run: |
cargo run --profile ci -p xtask -- generate-editions
- name: "check frozen edition records never change"
# Independent of the regeneration above: a stale record must not mask a frozen one
# being edited, nor the other way round.
if: "!cancelled()"
run: |
BASE="${{ github.event.pull_request.base.sha || 'HEAD^' }}"
cargo run --profile ci -p xtask -- check-editions --base "$BASE"
- name: "Make sure no files changed after regenerating"
run: |
git status --porcelain
test -z "$(git status --porcelain)"
# Schemas live under `<crate>/flatbuffers/` but resolve includes against those dirs
# collectively, so flatten each revision into one tree for flatc to compare.
- name: "Collect flatbuffer schemas from this revision and from develop"
if: "!cancelled()"
run: |
git fetch origin develop --depth 1
collect() {
git ls-tree -r --name-only "$1" \
| grep -E '(^|/)flatbuffers/.*\.fbs$' \
| while read -r path; do
rel="${path#*/flatbuffers/}"
mkdir -p "$2/$(dirname "$rel")"
git show "$1:$path" > "$2/$rel"
done
}
collect HEAD "$RUNNER_TEMP/fbs.head"
collect origin/develop "$RUNNER_TEMP/fbs.develop"
- name: "Verify flatbuffer back-compat"
if: "!cancelled()"
working-directory: ${{ runner.temp }}
run: |
find fbs.develop/ -type f -name "*.fbs" | sed 's|^fbs.develop/||' \
| xargs -I{} -n1 flatc -I fbs.head --conform-includes fbs.develop --conform fbs.develop/{} fbs.head/{}
- parallel:
- name: Build C++ tests and examples (asan)
if: "!cancelled()"
env:
RUSTUP_TOOLCHAIN: ${{ env.NIGHTLY_TOOLCHAIN }}
# symbolizing error backtrace on debug .so is slow
RUST_LIB_BACKTRACE: 0
run: |
cmake -S lang/cpp -B build \
-DCMAKE_BUILD_TYPE=Debug \
-DCMAKE_LINKER_TYPE=MOLD \
-DVORTEX_CARGO_PROFILE=ci \
-DBUILD_SHARED_LIBS=ON \
-DCMAKE_C_COMPILER=clang \
-DCMAKE_CXX_COMPILER=clang++ \
-DVORTEX_SANITIZER=asan \
-DVORTEX_SANITIZE_RUST_STD=ON \
-DVORTEX_BUILD_TESTS=ON \
-DVORTEX_BUILD_EXAMPLES=ON
cmake --build build --parallel "$(nproc)"
- name: Test CMake integration
if: "!cancelled()"
run: |
# These tests assert on the compiler wrappers they observe.
unset RUSTC_WRAPPER CMAKE_C_COMPILER_LAUNCHER CMAKE_CXX_COMPILER_LAUNCHER
python3 -m unittest discover -s vortex-ffi/cmake/tests -v
- name: Show C++ API compiler cache statistics
if: "!cancelled() && github.repository == 'vortex-data/vortex'"
run: sccache --show-stats
- name: Run C++ tests
if: "!cancelled()"
env:
RUST_LIB_BACKTRACE: 0
run: ctest --test-dir build --output-on-failure
- name: Run C++ examples
if: "!cancelled()"
env:
RUST_LIB_BACKTRACE: 0
run: |
cd build/examples
./writer people0.vortex
./writer people1.vortex
./writer me.vortex
./scan '*.vortex'
./dtype me.vortex
./reader
integration-smoke:
name: "Integration smoke tests (btrblocks golden, WASM, DuckDB S3)"
needs: duckdb-mirror
if: ${{ !cancelled() }}
runs-on: >-
${{ github.repository == 'vortex-data/vortex'
&& format('runs-on={0}/region=us-east-1/runner=amd64-ci-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=integration-smoke', github.run_id)
|| 'ubuntu-latest' }}
timeout-minutes: 30
steps:
- parallel:
- uses: runs-on/action@v2
if: github.repository == 'vortex-data/vortex'
with:
sccache: s3
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
- parallel:
- uses: ./.github/actions/setup-prebuild
with:
enable-sccache: "true"
targets: "wasm32-wasip1,wasm32-unknown-unknown"
- name: Setup Wasmer
uses: wasmerio/setup-wasmer@ba7631cb2d521612aa7ac5b13cb76fb77e824a1a # v3
with:
version: "v7.4.0"
- name: Verify Wasmer installation
run: wasmer --version
- parallel:
- name: Build DuckDB tests
if: (needs.duckdb-mirror.result == 'success' || needs.duckdb-mirror.result == 'skipped')
shell: bash
run: |
cargo nextest run --cargo-profile ci --locked -p vortex-duckdb --no-run
- name: Start RustFS
if: (needs.duckdb-mirror.result == 'success' || needs.duckdb-mirror.result == 'skipped')
shell: bash
run: |
curl -sSL https://dl.rustfs.com/artifacts/rustfs/release/rustfs-linux-x86_64-musl-latest.zip \
-o "${RUNNER_TEMP}/rustfs.zip"
unzip -o -q "${RUNNER_TEMP}/rustfs.zip" rustfs -d "${RUNNER_TEMP}"
chmod +x "${RUNNER_TEMP}/rustfs"
mkdir -p "${RUNNER_TEMP}/rustfs-data"
RUSTFS_ACCESS_KEY=rustfsadmin RUSTFS_SECRET_KEY=rustfsadmin \
"${RUNNER_TEMP}/rustfs" "${RUNNER_TEMP}/rustfs-data" --address 127.0.0.1:9000 &
until curl -sf http://127.0.0.1:9000/health; do sleep 1; done
curl -sf -X PUT "http://127.0.0.1:9000/vortex-test" \
--user "rustfsadmin:rustfsadmin" --aws-sigv4 "aws:amz:us-east-1:s3"
# The golden suite pins compressor decisions for the core-only and explicitly enabled
# edition configurations. Compact compression additionally needs its optional schemes.
- name: Golden corpus (edition configurations)
run: |
cargo nextest run --cargo-profile ci --locked --no-fail-fast -p vortex-btrblocks --test golden
- run: cargo build --profile ci --target wasm32-wasip1
working-directory: ./wasm-test
- parallel:
- name: Golden corpus (compact, zstd + pco)
if: "!cancelled()"
run: |
cargo nextest run --cargo-profile ci --locked --no-fail-fast -p vortex-btrblocks --test golden \
--features zstd,pco -E 'test(golden_compact)'
- name: Run tests
if: "!cancelled() && (needs.duckdb-mirror.result == 'success' || needs.duckdb-mirror.result == 'skipped')"
shell: bash
env:
AWS_ACCESS_KEY_ID: rustfsadmin
AWS_SECRET_ACCESS_KEY: rustfsadmin
AWS_ENDPOINT: "http://127.0.0.1:9000"
AWS_REGION: "us-east-1"
AWS_ALLOW_HTTP: "true"
VORTEX_DUCKDB_S3_URI: "s3://vortex-test/duckdb"
run: |
cargo nextest run --cargo-profile ci --locked -p vortex-duckdb --run-ignored all s3_
- run: wasmer run ./target/wasm32-wasip1/ci/wasm-test.wasm
if: "!cancelled()"
working-directory: ./wasm-test
- run: RUSTFLAGS='--cfg getrandom_backend="unsupported"' cargo build --profile ci --target wasm32-unknown-unknown
if: "!cancelled()"
working-directory: ./wasm-test
- name: "Check wasm32-unknown-unknown does not need wasm-bindgen"
# Pure wasm hosts (e.g. wasmtime) cannot provide wasm-bindgen's JS imports.
if: "!cancelled()"
shell: bash
working-directory: ./wasm-test
run: |
! cargo tree --target wasm32-unknown-unknown -e normal --prefix none \
| grep -E '^(wasm-bindgen|js-sys) '
# Checks each publishable crate on its own, to catch features that only unify in wasm-test.
- name: "Check publishable crates for wasm32-unknown-unknown"
if: "!cancelled()"
uses: ./.github/actions/check-rebuild
with:
command: >-
RUSTFLAGS='-A warnings --cfg getrandom_backend="unsupported"'
cargo hack check --profile ci --locked --ignore-private --target wasm32-unknown-unknown
--exclude vortex --exclude vortex-cuda --exclude vortex-cub --exclude vortex-nvcomp
--exclude vortex-datafusion --exclude vortex-duckdb --exclude vortex-tui --exclude vortex-zstd
--exclude vortex-test-e2e-cuda --exclude vortex-python-cuda --exclude vortex-sqllogictest
--exclude vortex-parquet-variant
- name: "Make sure no files changed after build"
if: "!cancelled()"
run: |
git status --porcelain
test -z "$(git status --porcelain)"