Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions .github/workflows/release-java.yml
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,11 @@ concurrency:
cancel-in-progress: false

jobs:
preflight:
uses: ./.github/workflows/release-preflight.yml

build-native:
needs: [preflight]
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
Expand Down
35 changes: 35 additions & 0 deletions .github/workflows/release-preflight.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

name: Release preflight

on:
workflow_call:

permissions:
contents: read

jobs:
versions:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6
- uses: actions/setup-python@v5
with:
python-version: '3.12'
- name: Check tag and all source versions
run: python3 tools/check_release_versions.py
4 changes: 4 additions & 0 deletions .github/workflows/release-python-publish.yml
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,11 @@ permissions:
contents: read

jobs:
preflight:
uses: ./.github/workflows/release-preflight.yml

publish:
needs: [preflight]
name: Publish to PyPI
if: github.repository == 'apache/paimon-vector-index' && startsWith(github.ref, 'refs/tags/')
runs-on: ubuntu-latest
Expand Down
15 changes: 15 additions & 0 deletions .github/workflows/release-python.yml
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,11 @@ permissions:
contents: read

jobs:
preflight:
uses: ./.github/workflows/release-preflight.yml

wheels-linux:
needs: [preflight]
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
Expand Down Expand Up @@ -78,13 +82,17 @@ jobs:
- name: Verify wheel legal and native contents
run: python3 tools/verify_binary_artifact.py --wheel "wheelhouse/*.whl"

- name: Install and exercise the final manylinux wheel
run: python3 tools/smoke_wheel.py --wheel "wheelhouse/*.whl"

- name: Upload wheels
uses: actions/upload-artifact@v5
with:
name: wheels-linux-${{ matrix.arch }}
path: wheelhouse/*.whl

wheels-macos:
needs: [preflight]
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
Expand Down Expand Up @@ -139,13 +147,17 @@ jobs:
- name: Verify wheel legal and native contents
run: python3 tools/verify_binary_artifact.py --wheel "python/dist/*.whl"

- name: Install and exercise the final repaired wheel
run: python3 tools/smoke_wheel.py --wheel "python/dist/*.whl"

- name: Upload wheel
uses: actions/upload-artifact@v5
with:
name: wheels-${{ matrix.os }}-${{ matrix.target }}
path: python/dist/*.whl

wheels-windows:
needs: [preflight]
runs-on: windows-latest
steps:
- uses: actions/checkout@v6
Expand Down Expand Up @@ -185,6 +197,9 @@ jobs:
- name: Verify wheel legal and native contents
run: python tools/verify_binary_artifact.py --wheel "python/dist/*.whl"

- name: Install and exercise the final Windows wheel
run: python tools/smoke_wheel.py --wheel "python/dist/*.whl"

- name: Upload wheel
uses: actions/upload-artifact@v5
with:
Expand Down
4 changes: 4 additions & 0 deletions .github/workflows/release-rust.yml
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,11 @@ concurrency:
cancel-in-progress: false

jobs:
preflight:
uses: ./.github/workflows/release-preflight.yml

publish:
needs: [preflight]
runs-on: ubuntu-latest
permissions:
contents: read
Expand Down
6 changes: 6 additions & 0 deletions .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -36,17 +36,23 @@ permissions:
contents: read

jobs:
preflight:
uses: ./.github/workflows/release-preflight.yml

rust:
name: Rust release
needs: [preflight]
uses: ./.github/workflows/release-rust.yml
secrets: inherit

java:
name: Java release
needs: [preflight]
uses: ./.github/workflows/release-java.yml

python-wheels:
name: Python wheels
needs: [preflight]
uses: ./.github/workflows/release-python.yml
secrets: inherit

Expand Down
54 changes: 50 additions & 4 deletions core/src/index_io_util.rs
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,48 @@ use std::io;
use std::mem::size_of;

pub(crate) const MAX_IVF_BATCH_READ_BYTES: usize = 64 * 1024 * 1024;
// Bound resident metadata derived from untrusted headers, including decoded
// arrays and auxiliary quantizer/rotation state, before constructing them.
pub(crate) const MAX_READER_METADATA_BYTES: usize = 1 << 30;
const IVF_STREAM_ALLOCATION_SLACK: usize = 64;

pub(crate) fn validate_reader_metadata_bytes(bytes: Option<usize>) -> io::Result<usize> {
bytes
.filter(|&bytes| bytes <= MAX_READER_METADATA_BYTES)
.ok_or_else(|| {
io::Error::new(
io::ErrorKind::InvalidData,
"index metadata exceeds the allocation limit",
)
})
}

pub(crate) fn try_vec_filled<T: Clone>(len: usize, value: T) -> io::Result<Vec<T>> {
let mut values = Vec::new();
values.try_reserve_exact(len).map_err(io::Error::other)?;
values.resize(len, value);
Ok(values)
}

pub(crate) fn read_metadata<R: SeekRead>(
reader: &mut R,
offset: u64,
len: usize,
) -> io::Result<Vec<u8>> {
// SeekRead does not expose file length. Verify the end of a large section
// with a one-byte read before reserving its buffer (notably for header-only
// inputs). Small metadata retains the single coalesced read path.
if len > MAX_IVF_BATCH_READ_BYTES {
let end = offset.checked_add(len as u64 - 1).ok_or_else(|| {
io::Error::new(io::ErrorKind::InvalidData, "metadata end offset overflow")
})?;
reader.pread(&mut [ReadRequest::new(end, &mut [0u8; 1])])?;
}
let mut bytes = try_vec_filled(len, 0u8)?;
reader.pread(&mut [ReadRequest::new(offset, &mut bytes)])?;
Ok(bytes)
}

pub(crate) fn ivf_payload_is_oversized(payload_len: usize) -> bool {
payload_len > MAX_IVF_BATCH_READ_BYTES
}
Expand Down Expand Up @@ -398,10 +438,16 @@ pub(crate) fn bytes_to_f32_vec(bytes: &[u8]) -> io::Result<Vec<f32>> {
"f32 byte section is not 4-byte aligned",
));
}
Ok(bytes
.chunks_exact(4)
.map(|c| f32::from_le_bytes([c[0], c[1], c[2], c[3]]))
.collect())
let mut values = Vec::new();
values
.try_reserve_exact(bytes.len() / 4)
.map_err(io::Error::other)?;
values.extend(
bytes
.chunks_exact(4)
.map(|c| f32::from_le_bytes(c.try_into().unwrap())),
);
Ok(values)
}

pub(crate) fn validate_positive_i32(val: i32, field: &str) -> io::Result<i32> {
Expand Down
Loading
Loading