diff --git a/.github/workflows/release-java.yml b/.github/workflows/release-java.yml index ac8ca3e..9758fc1 100644 --- a/.github/workflows/release-java.yml +++ b/.github/workflows/release-java.yml @@ -30,7 +30,11 @@ concurrency: cancel-in-progress: false jobs: + preflight: + uses: ./.github/workflows/release-preflight.yml + build-native: + needs: [preflight] runs-on: ${{ matrix.os }} strategy: fail-fast: false diff --git a/.github/workflows/release-preflight.yml b/.github/workflows/release-preflight.yml new file mode 100644 index 0000000..c69a4a2 --- /dev/null +++ b/.github/workflows/release-preflight.yml @@ -0,0 +1,35 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Release preflight + +on: + workflow_call: + +permissions: + contents: read + +jobs: + versions: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Check tag and all source versions + run: python3 tools/check_release_versions.py diff --git a/.github/workflows/release-python-publish.yml b/.github/workflows/release-python-publish.yml index 78bcd07..d2c7442 100644 --- a/.github/workflows/release-python-publish.yml +++ b/.github/workflows/release-python-publish.yml @@ -32,7 +32,11 @@ permissions: contents: read jobs: + preflight: + uses: ./.github/workflows/release-preflight.yml + publish: + needs: [preflight] name: Publish to PyPI if: github.repository == 'apache/paimon-vector-index' && startsWith(github.ref, 'refs/tags/') runs-on: ubuntu-latest diff --git a/.github/workflows/release-python.yml b/.github/workflows/release-python.yml index 41c2e27..6ea8917 100644 --- a/.github/workflows/release-python.yml +++ b/.github/workflows/release-python.yml @@ -34,7 +34,11 @@ permissions: contents: read jobs: + preflight: + uses: ./.github/workflows/release-preflight.yml + wheels-linux: + needs: [preflight] runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -78,6 +82,9 @@ jobs: - name: Verify wheel legal and native contents run: python3 tools/verify_binary_artifact.py --wheel "wheelhouse/*.whl" + - name: Install and exercise the final manylinux wheel + run: python3 tools/smoke_wheel.py --wheel "wheelhouse/*.whl" + - name: Upload wheels uses: actions/upload-artifact@v5 with: @@ -85,6 +92,7 @@ jobs: path: wheelhouse/*.whl wheels-macos: + needs: [preflight] runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -139,6 +147,9 @@ jobs: - name: Verify wheel legal and native contents run: python3 tools/verify_binary_artifact.py --wheel "python/dist/*.whl" + - name: Install and exercise the final repaired wheel + run: python3 tools/smoke_wheel.py --wheel "python/dist/*.whl" + - name: Upload wheel uses: actions/upload-artifact@v5 with: @@ -146,6 +157,7 @@ jobs: path: python/dist/*.whl wheels-windows: + needs: [preflight] runs-on: windows-latest steps: - uses: actions/checkout@v6 @@ -185,6 +197,9 @@ jobs: - name: Verify wheel legal and native contents run: python tools/verify_binary_artifact.py --wheel "python/dist/*.whl" + - name: Install and exercise the final Windows wheel + run: python tools/smoke_wheel.py --wheel "python/dist/*.whl" + - name: Upload wheel uses: actions/upload-artifact@v5 with: diff --git a/.github/workflows/release-rust.yml b/.github/workflows/release-rust.yml index 2fd540c..e51259b 100644 --- a/.github/workflows/release-rust.yml +++ b/.github/workflows/release-rust.yml @@ -33,7 +33,11 @@ concurrency: cancel-in-progress: false jobs: + preflight: + uses: ./.github/workflows/release-preflight.yml + publish: + needs: [preflight] runs-on: ubuntu-latest permissions: contents: read diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9339be0..129eda9 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -36,17 +36,23 @@ permissions: contents: read jobs: + preflight: + uses: ./.github/workflows/release-preflight.yml + rust: name: Rust release + needs: [preflight] uses: ./.github/workflows/release-rust.yml secrets: inherit java: name: Java release + needs: [preflight] uses: ./.github/workflows/release-java.yml python-wheels: name: Python wheels + needs: [preflight] uses: ./.github/workflows/release-python.yml secrets: inherit diff --git a/core/src/index_io_util.rs b/core/src/index_io_util.rs index 6d486e9..cba301a 100644 --- a/core/src/index_io_util.rs +++ b/core/src/index_io_util.rs @@ -21,8 +21,48 @@ use std::io; use std::mem::size_of; pub(crate) const MAX_IVF_BATCH_READ_BYTES: usize = 64 * 1024 * 1024; +// Bound resident metadata derived from untrusted headers, including decoded +// arrays and auxiliary quantizer/rotation state, before constructing them. +pub(crate) const MAX_READER_METADATA_BYTES: usize = 1 << 30; const IVF_STREAM_ALLOCATION_SLACK: usize = 64; +pub(crate) fn validate_reader_metadata_bytes(bytes: Option) -> io::Result { + bytes + .filter(|&bytes| bytes <= MAX_READER_METADATA_BYTES) + .ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidData, + "index metadata exceeds the allocation limit", + ) + }) +} + +pub(crate) fn try_vec_filled(len: usize, value: T) -> io::Result> { + let mut values = Vec::new(); + values.try_reserve_exact(len).map_err(io::Error::other)?; + values.resize(len, value); + Ok(values) +} + +pub(crate) fn read_metadata( + reader: &mut R, + offset: u64, + len: usize, +) -> io::Result> { + // SeekRead does not expose file length. Verify the end of a large section + // with a one-byte read before reserving its buffer (notably for header-only + // inputs). Small metadata retains the single coalesced read path. + if len > MAX_IVF_BATCH_READ_BYTES { + let end = offset.checked_add(len as u64 - 1).ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidData, "metadata end offset overflow") + })?; + reader.pread(&mut [ReadRequest::new(end, &mut [0u8; 1])])?; + } + let mut bytes = try_vec_filled(len, 0u8)?; + reader.pread(&mut [ReadRequest::new(offset, &mut bytes)])?; + Ok(bytes) +} + pub(crate) fn ivf_payload_is_oversized(payload_len: usize) -> bool { payload_len > MAX_IVF_BATCH_READ_BYTES } @@ -398,10 +438,16 @@ pub(crate) fn bytes_to_f32_vec(bytes: &[u8]) -> io::Result> { "f32 byte section is not 4-byte aligned", )); } - Ok(bytes - .chunks_exact(4) - .map(|c| f32::from_le_bytes([c[0], c[1], c[2], c[3]])) - .collect()) + let mut values = Vec::new(); + values + .try_reserve_exact(bytes.len() / 4) + .map_err(io::Error::other)?; + values.extend( + bytes + .chunks_exact(4) + .map(|c| f32::from_le_bytes(c.try_into().unwrap())), + ); + Ok(values) } pub(crate) fn validate_positive_i32(val: i32, field: &str) -> io::Result { diff --git a/core/src/io.rs b/core/src/io.rs index 95c8c76..8d7d6fa 100644 --- a/core/src/io.rs +++ b/core/src/io.rs @@ -19,7 +19,8 @@ use crate::distance::MetricType; use crate::index_io_util::{ bounded_ivf_payload_batch_end, bounded_ivf_stream_chunk_rows, bytes_to_f32_vec, decode_delta_varint_ids, encode_delta_varint_ids, pread_batched_slices, - read_delta_varint_ids_at, validate_reserved_zero, + read_delta_varint_ids_at, read_metadata, try_vec_filled, validate_reader_metadata_bytes, + validate_reserved_zero, }; use crate::ivfpq::IVFPQIndex; use crate::opq::OPQMatrix; @@ -506,6 +507,33 @@ fn u64_to_i64(value: u64, field: &str) -> io::Result { // --- Reader --- +fn pq_metadata_bytes( + d: usize, + nlist: usize, + m: usize, + ksub: usize, + has_opq: bool, +) -> io::Result { + let bytes = (|| { + let rotation = if has_opq { + d.checked_mul(d)?.checked_mul(4)? + } else { + 0 + }; + let centroids = nlist.checked_mul(d)?.checked_mul(4)?; + let codebook = d.checked_mul(ksub)?.checked_mul(4)?; + let table = nlist.checked_mul(16)?; + let wire = rotation + .checked_add(centroids)? + .checked_add(codebook)? + .checked_add(table)?; + wire.checked_mul(2)? + .checked_add(m.checked_mul(ksub)?.checked_mul(4)?)? + .checked_add(m.checked_add(1)?.checked_mul(size_of::())?) + })(); + validate_reader_metadata_bytes(bytes) +} + pub struct IVFPQIndexReader { reader: MeasuredSeekRead, pub d: usize, @@ -575,7 +603,7 @@ impl IVFPQIndexReader { format!("unsupported ksub {} (must be 16 or 256)", ksub), )); } - if d != m * dsub { + if Some(d) != m.checked_mul(dsub) { return Err(io::Error::new( io::ErrorKind::InvalidData, format!( @@ -583,7 +611,7 @@ impl IVFPQIndexReader { d, m, dsub, - m * dsub + m.saturating_mul(dsub) ), )); } @@ -628,6 +656,8 @@ impl IVFPQIndexReader { let transposed_codes = flags & FLAG_TRANSPOSED_CODES != 0; let has_opq = flags & FLAG_HAS_OPQ != 0; + pq_metadata_bytes(d, nlist, m, ksub, has_opq)?; + Ok(IVFPQIndexReader { reader: MeasuredSeekRead::new(reader), d, @@ -640,7 +670,7 @@ impl IVFPQIndexReader { total_vectors, opq: None, quantizer_centroids: Vec::new(), - pq: ProductQuantizer::with_nbits(d, m, ksub.trailing_zeros() as usize), + pq: ProductQuantizer::try_with_nbits(d, m, ksub.trailing_zeros() as usize)?, list_offsets: Vec::new(), list_counts: Vec::new(), list_id_bytes_lens: Vec::new(), @@ -670,9 +700,14 @@ impl IVFPQIndexReader { let m = self.m; let ksub = self.ksub; let dsub = self.dsub; + pq_metadata_bytes(d, nlist, m, ksub, self.has_opq)?; // Validate section sizes before allocating - let rotation_count = checked_section_size(d, d)?; + let rotation_count = if self.has_opq { + checked_section_size(d, d)? + } else { + 0 + }; let centroids_count = checked_section_size(nlist, d)?; let mk = m .checked_mul(ksub) @@ -708,15 +743,13 @@ impl IVFPQIndexReader { .ok_or_else(|| { io::Error::new(io::ErrorKind::InvalidData, "IVFPQ metadata size overflow") })?; - let mut metadata = vec![0u8; metadata_bytes]; - self.reader - .pread(&mut [ReadRequest::new(HEADER_SIZE as u64, &mut metadata)])?; + let metadata = read_metadata(&mut self.reader, HEADER_SIZE as u64, metadata_bytes)?; let mut position = 0usize; - if self.has_opq { + let opq = if self.has_opq { let rotation = bytes_to_f32_vec(&metadata[position..position + rotation_bytes])?; position += rotation_bytes; - self.opq = Some(OPQMatrix { + Some(OPQMatrix { d, m, rotation, @@ -725,24 +758,23 @@ impl IVFPQIndexReader { niter_pq: 0, niter_pq_0: 0, max_train_points: 0, - }); - } + }) + } else { + None + }; - self.quantizer_centroids = - bytes_to_f32_vec(&metadata[position..position + centroid_bytes])?; + let quantizer_centroids = bytes_to_f32_vec(&metadata[position..position + centroid_bytes])?; position += centroid_bytes; let pq_centroids = bytes_to_f32_vec(&metadata[position..position + pq_centroid_bytes])?; position += pq_centroid_bytes; - self.pq.set_centroids(pq_centroids); - - self.list_offsets = vec![0i64; nlist]; - self.list_counts = vec![0i32; nlist]; - self.list_id_bytes_lens = vec![0i32; nlist]; + let mut list_offsets = try_vec_filled(nlist, 0i64)?; + let mut list_counts = try_vec_filled(nlist, 0i32)?; + let mut list_id_bytes_lens = try_vec_filled(nlist, 0i32)?; let offset_table = &metadata[position..]; let mut actual_total = 0i64; for (i, entry) in offset_table.chunks_exact(16).enumerate() { - self.list_offsets[i] = i64::from_le_bytes(entry[0..8].try_into().unwrap()); + list_offsets[i] = i64::from_le_bytes(entry[0..8].try_into().unwrap()); let count = i32::from_le_bytes(entry[8..12].try_into().unwrap()); if count < 0 { return Err(io::Error::new( @@ -750,7 +782,7 @@ impl IVFPQIndexReader { format!("negative list count {} at list {}", count, i), )); } - self.list_counts[i] = count; + list_counts[i] = count; actual_total = actual_total.checked_add(count as i64).ok_or_else(|| { io::Error::new(io::ErrorKind::InvalidData, "IVFPQ vector count overflow") })?; @@ -761,7 +793,7 @@ impl IVFPQIndexReader { format!("negative id_bytes_len {} at list {}", id_bytes_len, i), )); } - self.list_id_bytes_lens[i] = id_bytes_len; + list_id_bytes_lens[i] = id_bytes_len; } if actual_total != self.total_vectors { return Err(io::Error::new( @@ -773,6 +805,16 @@ impl IVFPQIndexReader { )); } + // This is the final fallible step and updates the PQ only on success. + // Reuse its existing chunk plan to avoid allocating a second copy. + self.pq + .try_set_centroids(pq_centroids) + .map_err(io::Error::other)?; + self.opq = opq; + self.quantizer_centroids = quantizer_centroids; + self.list_offsets = list_offsets; + self.list_counts = list_counts; + self.list_id_bytes_lens = list_id_bytes_lens; self.loaded = true; Ok(()) } @@ -1259,6 +1301,7 @@ mod tests { #[derive(Default)] struct ReadStats { pread_calls: usize, + max_request_bytes: usize, } struct CountingPreadCursor { @@ -1278,7 +1321,10 @@ mod tests { impl SeekRead for CountingPreadCursor { fn pread(&mut self, ranges: &mut [ReadRequest<'_>]) -> io::Result<()> { for range in ranges { - self.stats.lock().unwrap().pread_calls += 1; + let mut stats = self.stats.lock().unwrap(); + stats.pread_calls += 1; + stats.max_request_bytes = stats.max_request_bytes.max(range.buf.len()); + drop(stats); let old_pos = io::Seek::stream_position(&mut self.inner)?; io::Seek::seek(&mut self.inner, io::SeekFrom::Start(range.pos))?; let result = io::Read::read_exact(&mut self.inner, range.buf); @@ -1336,6 +1382,97 @@ mod tests { assert!(decode_delta_varint_ids(0, &bytes, 1).is_err()); } + #[test] + fn ivfpq_header_rejects_extreme_derived_allocations() { + for (d, nlist, m, ksub, opq) in [ + (i32::MAX, 1, i32::MAX, 256, false), + (1 << 28, 1, 1, 256, false), + (1, i32::MAX, 1, 256, false), + (100_000, 1, 1, 256, true), + ] { + let mut header = [0u8; HEADER_SIZE]; + for (offset, value) in [ + (0, MAGIC), + (4, VERSION), + (8, d as u32), + (12, nlist as u32), + (16, m as u32), + (20, ksub), + (24, (d / m) as u32), + (28, MetricType::L2 as u32), + (40, REQUIRED_FLAGS | if opq { FLAG_HAS_OPQ } else { 0 }), + ] { + header[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); + } + let error = IVFPQIndexReader::open(std::io::Cursor::new(header)) + .err() + .unwrap(); + assert_eq!(error.kind(), io::ErrorKind::InvalidData); + assert!(error.to_string().contains("metadata")); + } + } + + #[test] + fn ivfpq_header_only_large_metadata_probes_end_before_allocation() { + let mut header = [0u8; HEADER_SIZE]; + for (offset, value) in [ + (0, MAGIC), + (4, VERSION), + (8, 65536), + (12, 1), + (16, 1), + (20, 256), + (24, 65536), + (28, MetricType::L2 as u32), + (40, REQUIRED_FLAGS), + ] { + header[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); + } + let stats = Arc::new(Mutex::new(ReadStats::default())); + let mut reader = + IVFPQIndexReader::open(CountingPreadCursor::new(header.to_vec(), stats.clone())) + .unwrap(); + assert_eq!( + reader.ensure_loaded().unwrap_err().kind(), + io::ErrorKind::UnexpectedEof + ); + assert_eq!(stats.lock().unwrap().pread_calls, 2); + assert_eq!(stats.lock().unwrap().max_request_bytes, HEADER_SIZE); + assert!(reader.pq.centroids().is_empty()); + } + + #[test] + fn ivfpq_failed_metadata_load_does_not_retain_decoded_arrays_on_retry() { + let mut index = IVFPQIndex::new(4, 1, 1, MetricType::L2, true); + index.set_quantizer_centroids(vec![0.0; 4]); + index.pq.set_centroids(vec![0.0; 4 * 256]); + index.opq = Some(OPQMatrix { + d: 4, + m: 1, + rotation: vec![0.0; 16], + is_trained: true, + niter: 0, + niter_pq: 0, + niter_pq_0: 0, + max_train_points: 0, + }); + let mut bytes = Vec::new(); + write_index(&index, &mut PosWriter::new(&mut bytes)).unwrap(); + bytes[32..40].copy_from_slice(&1i64.to_le_bytes()); + let mut reader = IVFPQIndexReader::open(Cursor::new(bytes)).unwrap(); + for _ in 0..2 { + let error = reader.ensure_loaded().unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::InvalidData); + assert!(error.to_string().contains("list total")); + assert!(reader.opq.is_none()); + assert!(reader.quantizer_centroids.is_empty()); + assert!(reader.pq.centroids().is_empty()); + assert!(reader.list_offsets.is_empty()); + assert!(reader.list_counts.is_empty()); + assert!(reader.list_id_bytes_lens.is_empty()); + } + } + #[test] fn test_delta_varint_ids_roundtrip() { let ids = vec![3i64, 7, 12, 15, 23, 100, 200]; @@ -1821,11 +1958,10 @@ mod tests { buf.extend_from_slice(&[0u8; 20]); let mut cursor = Cursor::new(&buf); - let mut reader = IVFPQIndexReader::open(&mut cursor).unwrap(); - let result = reader.ensure_loaded(); + let result = IVFPQIndexReader::open(&mut cursor); assert!( result.is_err(), - "huge m*ksub*dsub should return error, not panic" + "huge m*ksub*dsub should fail while opening" ); } diff --git a/core/src/ivfrq_io.rs b/core/src/ivfrq_io.rs index c6c7379..b35135d 100644 --- a/core/src/ivfrq_io.rs +++ b/core/src/ivfrq_io.rs @@ -19,8 +19,8 @@ use crate::collect::{Collector, RangeCollector}; use crate::distance::{fvec_norm_l2sqr, preprocess_vectors, MetricType}; use crate::index::validate_queries; use crate::index_io_util::{ - decode_delta_varint_ids, encode_delta_varint_ids, pread_batched_payloads, - validate_search_inputs, + bytes_to_f32_vec, decode_delta_varint_ids, encode_delta_varint_ids, pread_batched_payloads, + read_metadata, try_vec_filled, validate_reader_metadata_bytes, validate_search_inputs, }; use crate::io::{PreadCursor, ReadRequest, SeekRead, SeekWrite}; use crate::ivfpq::RowIdFilter; @@ -53,6 +53,8 @@ const REQUIRED_FLAGS: u32 = FLAG_DELTA_IDS | FLAG_BLOCK_TRANSPOSED_CODES | FLAG_ const SUPPORTED_FLAGS: u32 = REQUIRED_FLAGS; const FACTOR_BYTES: usize = 4; const MAX_RQ_BATCH_READ_BYTES: usize = 64 * 1024 * 1024; +const MAX_RQ_STREAM_BYTES: usize = 1024 * 1024; +const RQ_ID_READ_BYTES: usize = 64 * 1024; const PARALLEL_RQ_SCAN_MIN_CANDIDATES: usize = 8 * 1024; const PARALLEL_RQ_SEED_LISTS: usize = 1; const PARALLEL_RQ_SEED_VECTORS: usize = RQ_SCAN_BLOCK_SIZE; @@ -277,7 +279,9 @@ pub struct IVFRQIndexReader { pub list_counts: Vec, pub list_id_bytes_lens: Vec, quantizer: RaBitQuantizer, - rotation: RQRotation, + rotation: Option, + batch_read_limit: usize, + stream_chunk_bytes: usize, last_search_stats: IVFRQSearchStats, loaded: bool, } @@ -293,6 +297,115 @@ struct RQListPayloadMeta { payload_len: usize, } +fn rq_metadata_bytes(d: usize, nlist: usize, rounds: u32) -> io::Result { + let bytes = (|| { + let wire = nlist + .checked_mul(d)? + .checked_mul(4)? + .checked_add(nlist.checked_mul(16)?)?; + wire.checked_mul(2)? + .checked_add(nlist.checked_mul(4)?)? + .checked_add(RQRotation::allocation_bytes(d, rounds)?) + })(); + validate_reader_metadata_bytes(bytes) +} + +fn rq_stream_chunk_rows( + code_size: usize, + fields: usize, + target_bytes: usize, + id_window_bytes: usize, +) -> io::Result { + let available = MAX_RQ_BATCH_READ_BYTES + .checked_sub(id_window_bytes) + .ok_or_else(|| invalid_data("IVF-RQ ID window exceeds the streaming allocation limit"))?; + let block_bytes = fields + .checked_mul(8) + .and_then(|bytes| bytes.checked_add(8)) + .and_then(|bytes| bytes.checked_add(code_size)) + .and_then(|bytes| bytes.checked_mul(RQ_SCAN_BLOCK_SIZE)) + .filter(|&bytes| bytes != 0 && bytes <= available) + .ok_or_else(|| invalid_data("one IVF-RQ block exceeds the streaming allocation limit"))?; + // Prefer 1 MiB chunks, but support high-dimensional indexes whose single + // block is larger. Even those blocks must fit the 64 MiB allocation bound. + // The ID decoder keeps its window allocation across every scan chunk, + // including after the last refill truncates its length. + let budget = target_bytes + .max(block_bytes + id_window_bytes) + .min(MAX_RQ_BATCH_READ_BYTES); + Ok(((budget - id_window_bytes) / block_bytes) * RQ_SCAN_BLOCK_SIZE) +} + +// IDs precede all codes and factors on disk. Retain only a bounded encoded +// window and carry the delta across both I/O windows and scan chunks. +struct RQIdStream { + next_offset: u64, + unread: usize, + buffer: Vec, + position: usize, + current: i64, +} + +impl RQIdStream { + fn new(offset: u64, bytes: usize, base: i64) -> Self { + Self { + next_offset: offset, + unread: bytes, + buffer: Vec::new(), + position: 0, + current: base, + } + } + + fn next(&mut self, reader: &mut impl SeekRead) -> io::Result { + let mut delta = 0u64; + for part in 0..10 { + if self.position == self.buffer.len() { + if self.unread == 0 { + return Err(invalid_data("truncated delta-varint value")); + } + let len = self.unread.min(RQ_ID_READ_BYTES); + if self.buffer.is_empty() { + self.buffer = try_vec_filled(len, 0)?; + } else { + self.buffer.truncate(len); + } + reader.pread(&mut [ReadRequest::new(self.next_offset, &mut self.buffer)])?; + self.next_offset = self + .next_offset + .checked_add(self.buffer.len() as u64) + .ok_or_else(|| invalid_data("IVF-RQ ID offset overflow"))?; + self.unread -= self.buffer.len(); + self.position = 0; + } + let byte = self.buffer[self.position]; + self.position += 1; + if part == 9 && byte & 0xfe != 0 { + return Err(invalid_data("delta-varint value exceeds u64 limit")); + } + delta |= ((byte & 0x7f) as u64) << (part * 7); + if byte & 0x80 == 0 { + let next = (self.current as u64).wrapping_add(delta) as i64; + if next < self.current { + return Err(invalid_data( + "decoded ID sequence is not monotonically non-decreasing", + )); + } + self.current = next; + return Ok(next); + } + } + Err(invalid_data("delta-varint value exceeds u64 limit")) + } + + fn finish(&self) -> io::Result<()> { + if self.unread != 0 || self.position != self.buffer.len() { + return Err(invalid_data("trailing bytes in delta-varint ID section")); + } + Ok(()) + } +} + impl IVFRQIndexReader { pub fn open(mut reader: R) -> io::Result { let mut header = [0u8; IVF_RQ_HEADER_SIZE]; @@ -372,6 +485,7 @@ impl IVFRQIndexReader { ))); } + rq_metadata_bytes(d, nlist, rotation_rounds)?; Ok(Self { reader, d, @@ -391,7 +505,9 @@ impl IVFRQIndexReader { list_counts: Vec::new(), list_id_bytes_lens: Vec::new(), quantizer: RaBitQuantizer::new(d, num_bits), - rotation: RQRotation::new(d, rotation_seed, rotation_rounds), + rotation: None, + batch_read_limit: MAX_RQ_BATCH_READ_BYTES, + stream_chunk_bytes: MAX_RQ_STREAM_BYTES, last_search_stats: IVFRQSearchStats::default(), loaded: false, }) @@ -409,32 +525,33 @@ impl IVFRQIndexReader { if self.loaded { return Ok(()); } + rq_metadata_bytes(self.d, self.nlist, self.rotation_rounds)?; let centroid_values = checked_section_size(self.nlist, self.d)?; let centroid_bytes = checked_list_bytes(centroid_values, FACTOR_BYTES)?; let offset_table_bytes = checked_list_bytes(self.nlist, 16)?; let metadata_bytes = centroid_bytes .checked_add(offset_table_bytes) .ok_or_else(|| invalid_data("IVF-RQ resident metadata size overflow"))?; - let mut metadata = vec![0u8; metadata_bytes]; - self.reader - .pread(&mut [ReadRequest::new(IVF_RQ_HEADER_SIZE as u64, &mut metadata)])?; - - self.quantizer_centroids = bytes_to_f32_vec(&metadata[..centroid_bytes])?; - self.list_offsets = vec![0; self.nlist]; - self.list_counts = vec![0; self.nlist]; - self.list_id_bytes_lens = vec![0; self.nlist]; + let metadata = read_metadata(&mut self.reader, IVF_RQ_HEADER_SIZE as u64, metadata_bytes)?; + + // Keep failed loads empty so a retry cannot retain the previous + // decoded arrays alongside a new metadata read and decode. + let quantizer_centroids = bytes_to_f32_vec(&metadata[..centroid_bytes])?; + let mut list_offsets = try_vec_filled(self.nlist, 0)?; + let mut list_counts = try_vec_filled(self.nlist, 0)?; + let mut list_id_bytes_lens = try_vec_filled(self.nlist, 0)?; for (list_id, entry) in metadata[centroid_bytes..].chunks_exact(16).enumerate() { - self.list_offsets[list_id] = i64::from_le_bytes(entry[0..8].try_into().unwrap()); - self.list_counts[list_id] = validate_non_negative_i32( + list_offsets[list_id] = i64::from_le_bytes(entry[0..8].try_into().unwrap()); + list_counts[list_id] = validate_non_negative_i32( i32::from_le_bytes(entry[8..12].try_into().unwrap()), "list count", )?; - self.list_id_bytes_lens[list_id] = validate_non_negative_i32( + list_id_bytes_lens[list_id] = validate_non_negative_i32( i32::from_le_bytes(entry[12..16].try_into().unwrap()), "delta ID bytes", )?; } - let actual_total: i64 = self.list_counts.iter().map(|&count| count as i64).sum(); + let actual_total: i64 = list_counts.iter().map(|&count| count as i64).sum(); if actual_total != self.total_vectors { return Err(invalid_data(format!( "IVF-RQ list counts total {actual_total} does not match header {}", @@ -442,11 +559,20 @@ impl IVFRQIndexReader { ))); } - self.quantizer_centroid_norms = self - .quantizer_centroids - .chunks_exact(self.d) - .map(fvec_norm_l2sqr) - .collect(); + let mut quantizer_centroid_norms = try_vec_filled(self.nlist, 0.0)?; + for (norm, centroid) in quantizer_centroid_norms + .iter_mut() + .zip(quantizer_centroids.chunks_exact(self.d)) + { + *norm = fvec_norm_l2sqr(centroid); + } + let rotation = RQRotation::try_new(self.d, self.rotation_seed, self.rotation_rounds)?; + self.quantizer_centroids = quantizer_centroids; + self.quantizer_centroid_norms = quantizer_centroid_norms; + self.list_offsets = list_offsets; + self.list_counts = list_counts; + self.list_id_bytes_lens = list_id_bytes_lens; + self.rotation = Some(rotation); self.loaded = true; Ok(()) } @@ -508,7 +634,7 @@ impl IVFRQIndexReader { let next = payload_bytes .checked_add(meta.payload_len) .ok_or_else(|| invalid_data("IVF-RQ batch payload overflow"))?; - if index > 0 && next > MAX_RQ_BATCH_READ_BYTES { + if index > 0 && next > self.batch_read_limit { return Ok(index); } payload_bytes = next; @@ -517,6 +643,79 @@ impl IVFRQIndexReader { Ok(list_ids.len()) } + fn list_is_oversized(&self, list_id: usize) -> io::Result { + Ok(self + .list_payload_meta(list_id)? + .is_some_and(|meta| meta.payload_len > self.batch_read_limit)) + } + + fn for_each_streamed_list_chunk( + &mut self, + list_id: usize, + mut consume: impl FnMut(&RaBitQuantizer, &RQReadList) -> io::Result<()>, + ) -> io::Result<()> { + self.ensure_loaded()?; + let Some(meta) = self.list_payload_meta(list_id)? else { + return Ok(()); + }; + let mut header = [0u8; 16]; + self.reader + .pread(&mut [ReadRequest::new(meta.offset, &mut header)])?; + let base = i64::from_le_bytes(header[..8].try_into().unwrap()); + if i32::from_le_bytes(header[8..12].try_into().unwrap()) as usize != meta.id_bytes_len { + return Err(invalid_data("IVF-RQ delta ID length mismatch")); + } + if i32::from_le_bytes(header[12..16].try_into().unwrap()) as usize != meta.code_bytes_len { + return Err(invalid_data("IVF-RQ code length mismatch")); + } + let id_offset = meta + .offset + .checked_add(16) + .ok_or_else(|| invalid_data("IVF-RQ ID offset overflow"))?; + let code_offset = id_offset + .checked_add(meta.id_bytes_len as u64) + .ok_or_else(|| invalid_data("IVF-RQ code offset overflow"))?; + let factor_offset = code_offset + .checked_add(meta.code_bytes_len as u64) + .ok_or_else(|| invalid_data("IVF-RQ factor offset overflow"))?; + let code_size = self.quantizer.code_size(); + let fields = self.quantizer.factor_fields(); + // Account for codes, decoded IDs, raw/decoded factors, and the retained + // ID window, which is separate from the current chunk's decoded IDs. + let rows = rq_stream_chunk_rows( + code_size, + fields, + self.stream_chunk_bytes, + meta.id_bytes_len.min(RQ_ID_READ_BYTES), + )?; + let mut ids = RQIdStream::new(id_offset, meta.id_bytes_len, base); + for start in (0..meta.count).step_by(rows) { + let count = (meta.count - start).min(rows); + let mut list = RQReadList::empty(list_id); + list.ids = try_vec_filled(count, 0)?; + for id in &mut list.ids { + *id = ids.next(&mut self.reader)?; + } + list.payload = try_vec_filled(count * code_size, 0)?; + list.blocked_code_end = list.payload.len(); + let mut factor_bytes = try_vec_filled(count * fields * FACTOR_BYTES, 0)?; + // Each section stores consecutive 32-row blocks; chunk boundaries + // must coincide with block boundaries to preserve the SoA layout. + self.reader.pread(&mut [ReadRequest::new( + code_offset + (start * code_size) as u64, + &mut list.payload, + )])?; + self.reader.pread(&mut [ReadRequest::new( + factor_offset + (start * fields * FACTOR_BYTES) as u64, + &mut factor_bytes, + )])?; + list.blocked_factors = bytes_to_f32_vec(&factor_bytes)?; + drop(factor_bytes); + consume(&self.quantizer, &list)?; + } + ids.finish() + } + fn list_payload_meta(&self, list_id: usize) -> io::Result> { if list_id >= self.nlist { return Err(invalid_input(format!( @@ -534,12 +733,16 @@ impl IVFRQIndexReader { let payload_len = 16usize .checked_add(id_bytes_len) .and_then(|size| size.checked_add(code_bytes_len)) - .and_then(|size| size.checked_add(factor_values_len * FACTOR_BYTES)) + .and_then(|size| size.checked_add(factor_values_len.checked_mul(FACTOR_BYTES)?)) .ok_or_else(|| invalid_data("IVF-RQ list payload overflow"))?; + let offset = checked_list_offset(self.list_offsets[list_id], list_id)?; + offset + .checked_add(payload_len as u64) + .ok_or_else(|| invalid_data("IVF-RQ list end overflow"))?; Ok(Some(RQListPayloadMeta { list_id, count, - offset: checked_list_offset(self.list_offsets[list_id], list_id)?, + offset, id_bytes_len, code_bytes_len, factor_values_len, @@ -739,7 +942,11 @@ impl IVFRQIndexReader { let mut scratch = vec![0.0; self.padded_d]; for query in queries.chunks_exact(self.d) { let mut rotated = vec![0.0; self.padded_d]; - self.rotation.rotate(query, &mut rotated, &mut scratch); + self.rotation.as_ref().expect("loaded rotation").rotate( + query, + &mut rotated, + &mut scratch, + ); if rotated.iter().any(|value| !value.is_finite()) { return Err(invalid_data("non-finite IVF-RQ rotated query")); } @@ -767,6 +974,38 @@ impl IVFRQIndexReader { .collect::>(); let mut batch_start = 0; while batch_start < unique_lists.len() { + let first_list = unique_lists[batch_start]; + if self.list_is_oversized(first_list)? { + let metric = self.metric; + let centroid_norm = self.quantizer_centroid_norms[first_list]; + self.for_each_streamed_list_chunk(first_list, |quantizer, list| { + for &(query_index, distance) in &list_to_queries[first_list] { + let terms = range_query_terms( + quantizer, + distance, + query_norms.get(query_index).copied().unwrap_or(0.0), + centroid_norm, + metric, + )?; + let mut collector = RangeCollector::new(params.band()); + scan_range_blocked_list( + list, + quantizer, + &query_contexts[query_index], + terms, + filter, + &mut collector, + )?; + let mut output = query_merges[query_index].lock().expect("output lock"); + output.scanned += collector.scanned(); + output.rows.extend(collector.into_rows()); + } + Ok(()) + })?; + builder.record_list_read(); + batch_start += 1; + continue; + } let batch_end = batch_start + self.batch_read_end(&unique_lists[batch_start..])?; let loaded_lists = self.read_inverted_lists(&unique_lists[batch_start..batch_end])?; for list in &loaded_lists { @@ -776,22 +1015,13 @@ impl IVFRQIndexReader { } let scan_one = |list: &RQReadList, query_index: usize, distance: f32| -> io::Result<()> { - let terms = if self.metric == MetricType::L2 { - RQQueryTerms { - g_add: distance, - g_error: distance.sqrt(), - } - } else { - self.quantizer.query_terms_from_coarse_distance( - distance, - query_norms.get(query_index).copied().unwrap_or(0.0), - self.quantizer_centroid_norms[list.list_id], - self.metric, - ) - }; - if !terms.g_add.is_finite() || !terms.g_error.is_finite() { - return Err(invalid_data("non-finite IVF-RQ query terms")); - } + let terms = range_query_terms( + &self.quantizer, + distance, + query_norms.get(query_index).copied().unwrap_or(0.0), + self.quantizer_centroid_norms[list.list_id], + self.metric, + )?; let mut collector = RangeCollector::new(params.band()); scan_range_blocked_list( list, @@ -943,7 +1173,11 @@ pub(crate) fn search_batch_ivfrq_reader_filter_range( let mut rotated = vec![0.0; reader.padded_d]; let mut scratch = vec![0.0; reader.padded_d]; for query in processed.chunks_exact(reader.d) { - reader.rotation.rotate(query, &mut rotated, &mut scratch); + reader.rotation.as_ref().expect("loaded rotation").rotate( + query, + &mut rotated, + &mut scratch, + ); query_contexts.push(reader.quantizer.prepare_query(rotated.clone())); } @@ -967,6 +1201,38 @@ pub(crate) fn search_batch_ivfrq_reader_filter_range( }; let mut batch_start = 0; while batch_start < unique_lists.len() { + let first_list = unique_lists[batch_start]; + if reader.list_is_oversized(first_list)? { + let active = all_probe_indices + .iter() + .enumerate() + .filter_map(|(query_index, probes)| { + probes + .iter() + .enumerate() + .skip(probe_start) + .find(|(_, &id)| id == first_list) + .map(|(position, _)| (query_index, all_query_terms[query_index][position])) + }) + .collect::>(); + reader.for_each_streamed_list_chunk(first_list, |quantizer, list| { + for &(query_index, terms) in &active { + scan_blocked_list( + list, + quantizer, + &query_contexts[query_index], + terms, + filter, + &mut heaps[query_index], + &mut query_stats[query_index], + usize::MAX, + ); + } + Ok(()) + })?; + batch_start += 1; + continue; + } let count = reader.batch_read_end(&unique_lists[batch_start..])?; let batch_end = batch_start + count; let loaded_lists = reader.read_inverted_lists(&unique_lists[batch_start..batch_end])?; @@ -1403,6 +1669,27 @@ fn scan_blocked_list( } } +fn range_query_terms( + quantizer: &RaBitQuantizer, + distance: f32, + query_norm: f32, + centroid_norm: f32, + metric: MetricType, +) -> io::Result { + let terms = if metric == MetricType::L2 { + RQQueryTerms { + g_add: distance, + g_error: distance.sqrt(), + } + } else { + quantizer.query_terms_from_coarse_distance(distance, query_norm, centroid_norm, metric) + }; + if !terms.g_add.is_finite() || !terms.g_error.is_finite() { + return Err(invalid_data("non-finite IVF-RQ query terms")); + } + Ok(terms) +} + fn scan_range_blocked_list( list: &RQReadList, quantizer: &RaBitQuantizer, @@ -1605,16 +1892,6 @@ fn read_u64_le(reader: &mut PreadCursor<'_, R>) -> io::Res Ok(u64::from_le_bytes(bytes)) } -fn bytes_to_f32_vec(bytes: &[u8]) -> io::Result> { - if !bytes.len().is_multiple_of(4) { - return Err(invalid_data("IVF-RQ f32 section is not aligned")); - } - Ok(bytes - .chunks_exact(4) - .map(|chunk| f32::from_le_bytes(chunk.try_into().unwrap())) - .collect()) -} - fn validate_positive_i32(value: i32, field: &str) -> io::Result { if value <= 0 { Err(invalid_data(format!( @@ -1698,6 +1975,8 @@ mod tests { struct ReaderStats { calls: usize, max_ranges_per_batch: usize, + max_request_bytes: usize, + max_call_bytes: usize, } struct CountingReader { @@ -1711,6 +1990,16 @@ mod tests { let mut stats = self.stats.lock().unwrap(); stats.calls += 1; stats.max_ranges_per_batch = stats.max_ranges_per_batch.max(ranges.len()); + stats.max_request_bytes = stats.max_request_bytes.max( + ranges + .iter() + .map(|range| range.buf.len()) + .max() + .unwrap_or(0), + ); + stats.max_call_bytes = stats + .max_call_bytes + .max(ranges.iter().map(|range| range.buf.len()).sum()); drop(stats); for range in ranges { self.inner.seek(SeekFrom::Start(range.pos))?; @@ -1727,6 +2016,350 @@ mod tests { } } + fn stream_fixture(bits: usize, metric: MetricType) -> (IVFRQIndex, Vec) { + let d = 256; + let n = 99; + let mut index = IVFRQIndex::with_bits(d, 1, bits, metric); + index.set_quantizer_centroids(vec![0.0; d]); + let vectors = (0..n * d) + .map(|i| ((i * 17 + i / d * 31) % 503) as f32 * 0.001 - 0.25) + .collect::>(); + let ids = (0..n) + .map(|row| -50_000 + row as i64 * 10_000) + .collect::>(); + index.add(&vectors, &ids, n); + let mut bytes = Vec::new(); + write_ivfrq_index(&index, &mut PosWriter::new(&mut bytes)).unwrap(); + (index, bytes) + } + + #[test] + fn ivfrq_streamed_chunks_match_full_layout_and_bound_reads() { + for bits in [1, 4, 8] { + let (_, bytes) = stream_fixture(bits, MetricType::L2); + let mut full = IVFRQIndexReader::open(Cursor::new(bytes.clone())).unwrap(); + let list = full.read_inverted_list(0).unwrap(); + let stats = Arc::new(Mutex::new(ReaderStats::default())); + let mut streamed = IVFRQIndexReader::open(CountingReader { + inner: Cursor::new(bytes), + stats: stats.clone(), + max_ranges_per_pread: 1, + }) + .unwrap(); + streamed.ensure_loaded().unwrap(); + *stats.lock().unwrap() = ReaderStats::default(); + let id_window_bytes = streamed.list_id_bytes_lens[0] as usize; + let block_bytes = (full.quantizer.code_size() + 8 + full.quantizer.factor_fields() * 8) + * RQ_SCAN_BLOCK_SIZE; + let chunk_limit = 2 * block_bytes; + streamed.stream_chunk_bytes = chunk_limit; + let mut start = 0; + let mut chunks = 0; + streamed + .for_each_streamed_list_chunk(0, |quantizer, chunk| { + let end = start + chunk.ids.len(); + assert_eq!(chunk.ids, list.ids[start..end]); + assert_eq!( + chunk.blocked_codes(), + &list.blocked_codes() + [start * quantizer.code_size()..end * quantizer.code_size()] + ); + assert_eq!( + chunk.blocked_factors, + list.blocked_factors + [start * quantizer.factor_fields()..end * quantizer.factor_fields()] + ); + // Includes all retained payloads and the temporary factor decode. + let peak = chunk.ids.capacity() * 8 + + chunk.payload.capacity() + + chunk.blocked_factors.capacity() * 4 + + chunk.ids.len() * quantizer.factor_fields() * 4 + + id_window_bytes; + assert!(peak <= chunk_limit); + if end != list.ids.len() { + assert_eq!(chunk.ids.len() % RQ_SCAN_BLOCK_SIZE, 0); + } + start = end; + chunks += 1; + Ok(()) + }) + .unwrap(); + assert_eq!(start, list.ids.len()); + assert!(chunks > 1); + let stats = stats.lock().unwrap(); + assert!(stats.max_request_bytes <= chunk_limit); + assert!(stats.max_call_bytes <= chunk_limit); + } + } + + #[test] + fn ivfrq_streamed_search_matches_full_for_metrics_filters_and_batches() { + for bits in [1, 4, 8] { + for metric in [MetricType::L2, MetricType::Cosine, MetricType::InnerProduct] { + let (index, bytes) = stream_fixture(bits, metric); + let queries = (0..index.d * 2) + .map(|i| ((i * 7) % 29) as f32 * 0.01) + .collect::>(); + let filter = index.ids[0] + .iter() + .copied() + .filter(|id| id % 30_000 == 0) + .collect::>(); + for filter in [None, Some(&filter as &dyn RowIdFilter)] { + let mut full = IVFRQIndexReader::open(Cursor::new(bytes.clone())).unwrap(); + let mut streamed = IVFRQIndexReader::open(Cursor::new(bytes.clone())).unwrap(); + streamed.batch_read_limit = 1; + streamed.stream_chunk_bytes = + (index.code_size() + 8 + index_factors_fields(bits) * 8) + * RQ_SCAN_BLOCK_SIZE; + for nq in [1, 2] { + let queries = &queries[..index.d * nq]; + let expected = + search_batch_ivfrq_reader_filter(&mut full, queries, nq, 15, 1, filter) + .unwrap(); + let actual = search_batch_ivfrq_reader_filter( + &mut streamed, + queries, + nq, + 15, + 1, + filter, + ) + .unwrap(); + assert_eq!(actual, expected, "bits={bits}, metric={metric:?}"); + let params = VectorRangeSearchParams::new( + crate::range::DistanceBand::from_raw( + crate::range::Bound::Unbounded, + crate::range::Bound::Unbounded, + metric, + ) + .unwrap(), + 1, + ); + let expected = full + .range_search_batch_with_filter(queries, nq, params, filter) + .unwrap(); + let actual = streamed + .range_search_batch_with_filter(queries, nq, params, filter) + .unwrap(); + for query in 0..nq { + assert_eq!(range_pairs(&actual, query), range_pairs(&expected, query)); + } + assert_eq!( + actual.call_stats().list_reads(), + expected.call_stats().list_reads() + ); + for query in 0..nq { + assert_eq!( + actual.query(query).stats.rows_scanned(), + expected.query(query).stats.rows_scanned() + ); + assert_eq!( + actual.query(query).stats.lists_probed(), + expected.query(query).stats.lists_probed() + ); + } + } + } + } + } + } + + #[test] + fn ivfrq_stream_propagates_consumer_and_io_errors() { + let (_, bytes) = stream_fixture(4, MetricType::L2); + let mut reader = IVFRQIndexReader::open(Cursor::new(bytes.clone())).unwrap(); + reader.stream_chunk_bytes = 12 * 1024; + let mut calls = 0; + let error = reader + .for_each_streamed_list_chunk(0, |_, _| { + calls += 1; + Err(io::Error::other("consumer failed")) + }) + .unwrap_err(); + assert_eq!(calls, 1); + assert_eq!(error.to_string(), "consumer failed"); + let mut truncated = bytes; + truncated.pop(); + let mut reader = IVFRQIndexReader::open(Cursor::new(truncated)).unwrap(); + assert_eq!( + reader + .for_each_streamed_list_chunk(0, |_, _| Ok(())) + .unwrap_err() + .kind(), + io::ErrorKind::UnexpectedEof + ); + } + + #[test] + fn ivfrq_id_stream_carries_deltas_across_windows_and_rejects_corruption() { + let mut bytes = vec![0; RQ_ID_READ_BYTES - 1]; + bytes.extend_from_slice(&[0x80, 1, 3]); + let mut stream = RQIdStream::new(0, bytes.len(), -1000); + let mut reader = Cursor::new(bytes); + for _ in 0..RQ_ID_READ_BYTES - 1 { + assert_eq!(stream.next(&mut reader).unwrap(), -1000); + } + assert_eq!(stream.next(&mut reader).unwrap(), -872); + assert_eq!(stream.next(&mut reader).unwrap(), -869); + stream.finish().unwrap(); + assert_eq!(stream.buffer.capacity(), RQ_ID_READ_BYTES); + assert!(stream.buffer.len() < stream.buffer.capacity()); + for bytes in [vec![0x80], vec![0xff; 10], vec![1]] { + let mut stream = RQIdStream::new(0, bytes.len(), i64::MAX); + assert_eq!( + stream.next(&mut Cursor::new(bytes)).unwrap_err().kind(), + io::ErrorKind::InvalidData + ); + } + let mut stream = RQIdStream::new(0, 2, 0); + stream.next(&mut Cursor::new(vec![0, 0])).unwrap(); + assert!(stream.finish().is_err()); + } + + #[test] + fn ivfrq_extreme_header_fails_before_rotation_allocation() { + let (_, bytes) = stream_fixture(4, MetricType::L2); + let mut header = bytes[..IVF_RQ_HEADER_SIZE].to_vec(); + let d: i32 = 1 << 30; + header[8..12].copy_from_slice(&d.to_le_bytes()); + header[12..16].copy_from_slice(&d.to_le_bytes()); + header[52..56].copy_from_slice(&(d / 8).to_le_bytes()); + let error = IVFRQIndexReader::open(Cursor::new(header)).err().unwrap(); + assert_eq!(error.kind(), io::ErrorKind::InvalidData); + assert!(error.to_string().contains("metadata")); + let reader = + IVFRQIndexReader::open(Cursor::new(bytes[..IVF_RQ_HEADER_SIZE].to_vec())).unwrap(); + assert!( + reader.rotation.is_none(), + "header open must defer rotation construction" + ); + } + + #[test] + fn ivfrq_header_only_large_metadata_probes_end_before_allocation() { + let (_, bytes) = stream_fixture(4, MetricType::L2); + let mut header = bytes[..IVF_RQ_HEADER_SIZE].to_vec(); + header[16..20].copy_from_slice(&65536i32.to_le_bytes()); + let stats = Arc::new(Mutex::new(ReaderStats::default())); + let mut reader = IVFRQIndexReader::open(CountingReader { + inner: Cursor::new(header), + stats: stats.clone(), + max_ranges_per_pread: 1, + }) + .unwrap(); + assert_eq!( + reader.ensure_loaded().unwrap_err().kind(), + io::ErrorKind::UnexpectedEof + ); + assert_eq!(stats.lock().unwrap().calls, 2); + assert_eq!(stats.lock().unwrap().max_request_bytes, IVF_RQ_HEADER_SIZE); + assert!(reader.rotation.is_none()); + } + + #[test] + fn ivfrq_failed_metadata_load_does_not_retain_decoded_arrays_on_retry() { + let (_, mut bytes) = stream_fixture(4, MetricType::L2); + bytes[32..40].copy_from_slice(&100i64.to_le_bytes()); + let mut reader = IVFRQIndexReader::open(Cursor::new(bytes)).unwrap(); + for _ in 0..2 { + let error = reader.ensure_loaded().unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::InvalidData); + assert!(error.to_string().contains("counts total")); + assert!(reader.quantizer_centroids.is_empty()); + assert!(reader.quantizer_centroid_norms.is_empty()); + assert!(reader.list_offsets.is_empty()); + assert!(reader.list_counts.is_empty()); + assert!(reader.list_id_bytes_lens.is_empty()); + assert!(reader.rotation.is_none()); + } + } + + #[test] + fn ivfrq_high_dimension_streaming_retains_one_complete_block() { + assert_eq!( + rq_stream_chunk_rows(32768, 5, MAX_RQ_STREAM_BYTES, 1024).unwrap(), + RQ_SCAN_BLOCK_SIZE + ); + assert!(rq_stream_chunk_rows( + MAX_RQ_BATCH_READ_BYTES / RQ_SCAN_BLOCK_SIZE, + 5, + MAX_RQ_STREAM_BYTES, + 1024 + ) + .is_err()); + assert!(rq_stream_chunk_rows(usize::MAX, 5, MAX_RQ_STREAM_BYTES, 1024).is_err()); + // This encodable 8-bit shape fits before adding its retained ID window. + assert!(rq_stream_chunk_rows(2_097_088, 5, MAX_RQ_STREAM_BYTES, 1024).is_err()); + let rows = rq_stream_chunk_rows(64, 5, MAX_RQ_STREAM_BYTES, RQ_ID_READ_BYTES).unwrap(); + assert!(rows * (64 + 8 + 5 * 8) + RQ_ID_READ_BYTES <= MAX_RQ_STREAM_BYTES); + assert!( + (rows + RQ_SCAN_BLOCK_SIZE) * (64 + 8 + 5 * 8) + RQ_ID_READ_BYTES > MAX_RQ_STREAM_BYTES + ); + } + + #[test] + fn ivfrq_streamed_mixed_lists_and_probe_continuation_match_full_search() { + let (mut index, _) = range_fixture(3, 67); + let code_size = index.code_size(); + for (list_id, rows) in [(0, 3), (2, 2)] { + index.ids[list_id].truncate(rows); + index.codes[list_id].truncate(rows * code_size); + index.factors[list_id].truncate(rows); + } + let mut bytes = Vec::new(); + write_ivfrq_index(&index, &mut PosWriter::new(&mut bytes)).unwrap(); + let queries = [vec![0.2; index.d], vec![32.1; index.d]].concat(); + let mut full = IVFRQIndexReader::open(Cursor::new(bytes.clone())).unwrap(); + let mut streamed = IVFRQIndexReader::open(Cursor::new(bytes)).unwrap(); + streamed.batch_read_limit = 512; + streamed.stream_chunk_bytes = 2560; + let expected = search_batch_ivfrq_reader(&mut full, &queries, 2, 15, 3).unwrap(); + let actual = search_batch_ivfrq_reader(&mut streamed, &queries, 2, 15, 3).unwrap(); + // Equal-distance rows have no ordering guarantee when list/chunk + // visitation changes. Compare the selected IDs and exact distances. + let canonical = |result: &(Vec, Vec)| { + (0..2) + .map(|query| { + let mut pairs = result.0[query * 15..(query + 1) * 15] + .iter() + .copied() + .zip( + result.1[query * 15..(query + 1) * 15] + .iter() + .map(|distance| distance.to_bits()), + ) + .collect::>(); + pairs.sort_unstable(); + pairs + }) + .collect::>() + }; + assert_eq!(canonical(&actual), canonical(&expected)); + let (seed_ids, seed_distances) = + search_batch_ivfrq_reader(&mut streamed, &queries, 2, 15, 1).unwrap(); + let resumed = search_batch_ivfrq_reader_filter_range( + &mut streamed, + &queries, + 2, + 15, + 1, + 3, + &seed_ids, + &seed_distances, + None, + ) + .unwrap(); + assert_eq!(canonical(&resumed), canonical(&expected)); + let params = all_range_params(3); + let expected = full.range_search_batch(&queries, 2, params).unwrap(); + let actual = streamed.range_search_batch(&queries, 2, params).unwrap(); + for query in 0..2 { + assert_eq!(range_pairs(&actual, query), range_pairs(&expected, query)); + } + assert_eq!(actual.call_stats().list_reads(), 3); + } + fn range_fixture(nlist: usize, per_list: usize) -> (IVFRQIndex, Vec) { let dimension = 16; let mut index = IVFRQIndex::with_bits(dimension, nlist, 4, MetricType::L2); @@ -2254,7 +2887,10 @@ mod tests { let processed = preprocess_vectors(&queries, nq, d, metric); for query in processed.chunks_exact(d) { let mut rotated = vec![0.0; d]; - fast.rotation.rotate(query, &mut rotated, &mut vec![0.0; d]); + fast.rotation + .as_ref() + .unwrap() + .rotate(query, &mut rotated, &mut vec![0.0; d]); let context = fast.quantizer.prepare_query(rotated); let terms = fast.quantizer.query_terms(&context, &vec![0.0; d], metric); for stride in [1, 2, 16] { diff --git a/core/src/pq.rs b/core/src/pq.rs index 062416c..64d50e9 100644 --- a/core/src/pq.rs +++ b/core/src/pq.rs @@ -105,6 +105,17 @@ impl ProductQuantizer { Self::with_validated_chunk_offsets(d, nbits, chunk_offsets) } + /// Header readers validate the derived sizes before using this fallible + /// constructor. Codebooks and their caches remain unloaded. + pub(crate) fn try_with_nbits(d: usize, m: usize, nbits: usize) -> std::io::Result { + let mut offsets = Vec::new(); + offsets + .try_reserve_exact(m + 1) + .map_err(std::io::Error::other)?; + offsets.extend((0..=m).map(|chunk| chunk * (d / m))); + Ok(Self::with_validated_chunk_offsets(d, nbits, offsets)) + } + /// Create a quantizer whose chunks differ in width by at most one. /// /// The first `d % m` chunks contain one additional component. This is the diff --git a/core/src/rq.rs b/core/src/rq.rs index eebf2c4..674dcd0 100644 --- a/core/src/rq.rs +++ b/core/src/rq.rs @@ -69,19 +69,42 @@ pub struct RQRotation { impl RQRotation { pub fn new(d: usize, seed: u64, rounds: u32) -> Self { + Self::try_new(d, seed, rounds).expect("RQ rotation allocation failed") + } + + pub(crate) fn allocation_bytes(d: usize, rounds: u32) -> Option { + let padded = padded_dimension(d); + padded + .checked_mul(std::mem::size_of::())? + .checked_add(padded.div_ceil(8))? + .checked_add(2 * std::mem::size_of::>())? + .checked_mul(rounds as usize) + } + + pub(crate) fn try_new(d: usize, seed: u64, rounds: u32) -> std::io::Result { let padded_d = padded_dimension(d); let mut rng = SplitMix64::new(seed ^ (d as u64).rotate_left(17)); - let mut sign_masks = Vec::with_capacity(rounds as usize); - let mut permutations = Vec::with_capacity(rounds as usize); + let mut sign_masks = Vec::new(); + let mut permutations = Vec::new(); + sign_masks + .try_reserve_exact(rounds as usize) + .map_err(std::io::Error::other)?; + permutations + .try_reserve_exact(rounds as usize) + .map_err(std::io::Error::other)?; for _ in 0..rounds { - let mut signs = vec![0u8; padded_d.div_ceil(8)]; + let mut signs = crate::index_io_util::try_vec_filled(padded_d.div_ceil(8), 0u8)?; for value in &mut signs { *value = rng.next_u64() as u8; } sign_masks.push(signs); - let mut permutation: Vec = (0..padded_d).collect(); + let mut permutation = Vec::new(); + permutation + .try_reserve_exact(padded_d) + .map_err(std::io::Error::other)?; + permutation.extend(0..padded_d); for i in (1..padded_d).rev() { let j = rng.next_usize(i + 1); permutation.swap(i, j); @@ -89,14 +112,14 @@ impl RQRotation { permutations.push(permutation); } - Self { + Ok(Self { d, padded_d, seed, rounds, sign_masks, permutations, - } + }) } pub fn dimension(&self) -> usize { diff --git a/java/src/test/java/org/apache/paimon/index/vector/VectorIndexNativePanicBoundaryTest.java b/java/src/test/java/org/apache/paimon/index/vector/VectorIndexNativePanicBoundaryTest.java index a7b6d19..b266667 100644 --- a/java/src/test/java/org/apache/paimon/index/vector/VectorIndexNativePanicBoundaryTest.java +++ b/java/src/test/java/org/apache/paimon/index/vector/VectorIndexNativePanicBoundaryTest.java @@ -28,6 +28,7 @@ public static void main(String[] args) { testVoidEntrypointErrorBecomesRuntimeException(); testObjectEntrypointSkipsNanScores(); + testChunkedOutputReusesBuffersAndPreservesJavaException(); VectorIndexTrainer survivor = VectorIndexTrainer.create(ivfFlatOptions()); survivor.close(); @@ -98,6 +99,88 @@ private static void assertNanSkipped(long[] ids, float[] distances) { assertEquals(Float.floatToRawIntBits(Float.MAX_VALUE), Float.floatToRawIntBits(distances[1])); } + private static void testChunkedOutputReusesBuffersAndPreservesJavaException() { + Map options = ivfFlatOptions(); + options.put("index.type", "ivf_sq"); + options.put("dimension", "16"); + final int rows = 150001; + float[] data = new float[rows * 16]; + long[] ids = new long[rows]; + for (int row = 0; row < rows; row++) { + ids[row] = row + 1; + for (int dim = 0; dim < 16; dim++) { + data[row * 16 + dim] = (row * 7 + dim * 13) % 256; + } + } + float[] training = java.util.Arrays.copyOf(data, 256 * 16); + VectorIndexWriter writer = + new VectorIndexWriter(VectorIndexTrainer.train(options, training, 256)); + try { + writer.addVectors(ids, data, rows); + ChunkCheckingOutput output = new ChunkCheckingOutput(false); + writer.writeIndex(output); + if (output.fullChunks < 2) { + throw new AssertionError("SQ list did not exercise repeated full chunks"); + } + byte[] bytes = output.out.toByteArray(); + // Global SQ bounds, list SQ bounds, then the coarse centroid. + int table = 64 + 5 * 16 * Float.BYTES; + int codesOffset = (int) readLongLe(bytes, table); + // Check the complete blocked SQ byte order, including signed bytes + // and the short final 32-row block, across both JNI boundaries. + int offset = codesOffset; + for (int start = 0; start < rows; start += 32) { + int lanes = Math.min(32, rows - start); + for (int dim = 0; dim < 16; dim++) { + for (int lane = 0; lane < lanes; lane++) { + assertEquals((int) data[(start + lane) * 16 + dim], bytes[offset++] & 255); + } + } + } + ChunkCheckingOutput failing = new ChunkCheckingOutput(true); + try { + writer.writeIndex(failing); + throw new AssertionError("expected Java output exception"); + } catch (RuntimeException error) { + if (error != failing.failure) { + throw new AssertionError("JNI replaced the original Java exception", error); + } + assertEquals(2, failing.fullChunks); + } + } finally { + writer.close(); + } + } + + public static final class ChunkCheckingOutput { + private final ByteArrayOutputStream out = new ByteArrayOutputStream(); + private final RuntimeException failure = new RuntimeException("chunk write failed"); + private final boolean fail; + private byte[] fullBuffer; + private int fullChunks; + + ChunkCheckingOutput(boolean fail) { + this.fail = fail; + } + + public void write(byte[] bytes) { + if (bytes.length > 1024 * 1024) { + throw new AssertionError("JNI write exceeds the chunk limit"); + } + if (bytes.length == 1024 * 1024) { + fullChunks++; + if (fullBuffer != null && fullBuffer != bytes) { + throw new AssertionError("JNI did not reuse the full-chunk Java buffer"); + } + fullBuffer = bytes; + if (fail && fullChunks == 2) { + throw failure; + } + } + out.write(bytes, 0, bytes.length); + } + } + private static Map ivfFlatOptions() { Map options = new HashMap(); options.put("index.type", "ivf_flat"); diff --git a/jni/src/lib.rs b/jni/src/lib.rs index be5ddca..ec047f8 100644 --- a/jni/src/lib.rs +++ b/jni/src/lib.rs @@ -705,7 +705,9 @@ pub extern "system" fn Java_org_apache_paimon_index_vector_VectorIndexNative_wri let mut output = JniOutputStream::new(jvm, global_ref); if let Err(e) = writer.writer.write(&mut output) { - throw_and_return::<()>(env, &format!("write index: {}", e)); + if !env.exception_check().unwrap_or(false) { + throw_and_return::<()>(env, &format!("write index: {}", e)); + } } }) } diff --git a/jni/src/stream.rs b/jni/src/stream.rs index a10bdc9..1297717 100644 --- a/jni/src/stream.rs +++ b/jni/src/stream.rs @@ -40,6 +40,7 @@ impl JniSeekableStream { } const MAX_JNI_RANGES_PER_CALL: usize = 256; +const JNI_WRITE_CHUNK_BYTES: usize = 1024 * 1024; impl SeekRead for JniSeekableStream { /// Positional reads via VectorIndexInput.pread(long[] positions, byte[][] buffers). @@ -194,6 +195,9 @@ impl JniOutputStream { impl paimon_vindex_core::io::SeekWrite for JniOutputStream { fn write_all(&mut self, buf: &[u8]) -> io::Result<()> { + if buf.is_empty() { + return Ok(()); + } let mut env = self .jvm .attach_current_thread() @@ -201,12 +205,31 @@ impl paimon_vindex_core::io::SeekWrite for JniOutputStream { let result = env .with_local_frame(8, |env| { - Ok::<_, jni::errors::Error>(write_chunk(env, self.stream_ref.as_obj(), buf)) + let capacity = buf.len().min(JNI_WRITE_CHUNK_BYTES); + let jbuf = env.new_byte_array(capacity as i32)?; + Ok::<_, jni::errors::Error>(write_chunks( + buf, + &mut self.pos, + JNI_WRITE_CHUNK_BYTES, + |chunk| { + if chunk.len() == capacity { + write_chunk(env, self.stream_ref.as_obj(), &jbuf, chunk) + } else { + // The Java contract is write(byte[]), so a short final + // chunk needs its exact length to avoid trailing bytes. + let tail = env + .new_byte_array(chunk.len() as i32) + .map_err(|e| io::Error::other(format!("JNI alloc: {e}")))?; + let result = write_chunk(env, self.stream_ref.as_obj(), &tail, chunk); + // Local-frame cleanup also works with a pending Java exception. + result + } + }, + )) }) .map_err(|e| io::Error::other(format!("JNI output local frame: {e}")))?; result?; - self.pos += buf.len() as u64; Ok(()) } @@ -215,14 +238,39 @@ impl paimon_vindex_core::io::SeekWrite for JniOutputStream { } } -fn write_chunk(env: &mut jni::JNIEnv<'_>, stream: &JObject<'_>, buf: &[u8]) -> io::Result<()> { - let jbuf = env - .new_byte_array(buf.len() as i32) - .map_err(|e| io::Error::other(format!("JNI alloc: {e}")))?; - let signed = buf.iter().map(|&byte| byte as i8).collect::>(); - env.set_byte_array_region(&jbuf, 0, &signed) +fn write_chunks( + buf: &[u8], + pos: &mut u64, + chunk_bytes: usize, + mut write: impl FnMut(&[u8]) -> io::Result<()>, +) -> io::Result<()> { + if chunk_bytes == 0 || chunk_bytes > i32::MAX as usize { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "invalid JNI write chunk size", + )); + } + pos.checked_add(buf.len() as u64) + .ok_or_else(|| io::Error::other("JNI output position overflow"))?; + for chunk in buf.chunks(chunk_bytes) { + write(chunk)?; + *pos += chunk.len() as u64; + } + Ok(()) +} + +fn write_chunk( + env: &mut jni::JNIEnv<'_>, + stream: &JObject<'_>, + jbuf: &JByteArray<'_>, + buf: &[u8], +) -> io::Result<()> { + // u8 and JNI jbyte (i8) have identical size/alignment and all bit patterns + // are valid. Borrow the bytes rather than copying into a second Rust Vec. + let signed = unsafe { std::slice::from_raw_parts(buf.as_ptr().cast::(), buf.len()) }; + env.set_byte_array_region(jbuf, 0, signed) .map_err(|e| io::Error::other(format!("JNI set_region: {e}")))?; - env.call_method(stream, "write", "([B)V", &[JValue::Object(&jbuf)]) + env.call_method(stream, "write", "([B)V", &[JValue::Object(jbuf)]) .map_err(|e| io::Error::other(format!("JNI write: {e}")))?; Ok(()) } @@ -231,6 +279,55 @@ fn write_chunk(env: &mut jni::JNIEnv<'_>, stream: &JObject<'_>, buf: &[u8]) -> i mod tests { use super::*; + #[test] + fn jni_chunked_output_preserves_bytes_and_position() { + let input = [0, 127, 128, 255, 4, 5, 6, 7]; + for limit in [1, 3, 4, 8, 20] { + let mut pos = 17; + let mut output = Vec::new(); + let mut sizes = Vec::new(); + write_chunks(&input, &mut pos, limit, |chunk| { + sizes.push(chunk.len()); + output.extend_from_slice(chunk); + Ok(()) + }) + .unwrap(); + assert_eq!(output, input); + assert_eq!(pos, 25); + assert_eq!(sizes.len(), input.len().div_ceil(limit)); + assert!(sizes.iter().all(|&len| len <= limit)); + } + } + + #[test] + fn jni_chunked_output_stops_on_error_and_counts_only_successful_chunks() { + let mut pos = 17; + let mut calls = 0; + let mut output = Vec::new(); + let error = write_chunks(&[0, 1, 2, 128, 255, 5, 6, 7], &mut pos, 3, |chunk| { + calls += 1; + if calls == 2 { + return Err(io::Error::new( + io::ErrorKind::BrokenPipe, + "Java write failed", + )); + } + output.extend_from_slice(chunk); + Ok(()) + }) + .unwrap_err(); + assert_eq!(calls, 2); + assert_eq!(output, [0, 1, 2]); + assert_eq!(pos, 20); + assert_eq!(error.kind(), io::ErrorKind::BrokenPipe); + assert_eq!(error.to_string(), "Java write failed"); + write_chunks(&[], &mut pos, 3, |_| panic!("empty write")).unwrap(); + assert_eq!(pos, 20); + assert!(write_chunks(&[0], &mut pos, 0, |_| panic!("invalid limit")).is_err()); + let mut full_pos = u64::MAX; + assert!(write_chunks(&[0], &mut full_pos, 1, |_| panic!("position overflow")).is_err()); + } + #[test] fn jni_seekable_stream_is_cloneable_for_parallel_diskann_batch() { fn assert_clone() {} diff --git a/tools/check_release_versions.py b/tools/check_release_versions.py new file mode 100644 index 0000000..05b4d7b --- /dev/null +++ b/tools/check_release_versions.py @@ -0,0 +1,84 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Fail release jobs before publishing if their source versions do not match.""" + +import argparse +import json +import os +from pathlib import Path +import re +import subprocess +import sys +import xml.etree.ElementTree as ET + + +def check_versions(root, ref): + match = re.fullmatch( + r"refs/tags/v((?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*))(?:-rc([1-9][0-9]*))?", + ref, + ) + if not match: + raise ValueError("release requires a vX.Y.Z or vX.Y.Z-rcN tag") + version = match.group(1) + # RC tags vote on the exact source later tagged as final. Cargo, Maven, + # and the source pyproject retain X.Y.Z; wheel builds inject X.Y.ZrcN. + result = subprocess.run( + ["cargo", "metadata", "--offline", "--no-deps", "--format-version", "1"], + cwd=root, text=True, capture_output=True, check=True, + ) + metadata = json.loads(result.stdout) + members = set(metadata["workspace_members"]) + packages = [p for p in metadata["packages"] if p["id"] in members] + if not packages: + raise ValueError("Cargo workspace has no packages") + names = {p["name"] for p in packages} + versions = {"Cargo " + p["name"]: p["version"] for p in packages} + for package in packages: + for dependency in package["dependencies"]: + if dependency["name"] in names and dependency.get("path"): + versions[package["name"] + " -> " + dependency["name"]] = dependency["req"].removeprefix("^") + text = (root / "python/pyproject.toml").read_text() + project = re.search(r"(?ms)^\[project\]\s*\n(.*?)(?=^\[|\Z)", text) + python_versions = re.findall(r'^version\s*=\s*"([^"]+)"\s*$', project.group(1) if project else "", re.M) + if len(python_versions) != 1: + raise ValueError("expected one [project].version in python/pyproject.toml") + versions["Python"] = python_versions[0] + pom = ET.parse(root / "java/pom.xml").getroot() + versions["Maven"] = pom.findtext("{*}version") + mismatches = [f"{name}: {actual!r} (expected {version})" for name, actual in versions.items() if actual != version] + if mismatches: + raise ValueError("release version mismatch:\n" + "\n".join(mismatches)) + return version + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--root", type=Path, default=Path(__file__).resolve().parents[1]) + parser.add_argument("--ref", default=os.environ.get("GITHUB_REF", "")) + args = parser.parse_args() + try: + version = check_versions(args.root.resolve(), args.ref) + except (ValueError, OSError, subprocess.CalledProcessError, ET.ParseError) as error: + print(f"Release preflight failed: {error}", file=sys.stderr) + return 1 + print(f"Release preflight passed: {args.ref}, source version {version}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/smoke_wheel.py b/tools/smoke_wheel.py new file mode 100644 index 0000000..33bcea8 --- /dev/null +++ b/tools/smoke_wheel.py @@ -0,0 +1,90 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Install a final wheel in isolation and exercise its bundled native library.""" + +import argparse +import glob +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import venv + + +def smoke_installed(): + import io + import numpy as np + import paimon_vindex + from paimon_vindex import _ffi, SearchParams, VectorIndexReader, VectorIndexTrainer, VectorIndexWriter + + package = Path(paimon_vindex.__file__).resolve().parent + assert sys.prefix != sys.base_prefix, "smoke test requires an isolated environment" + assert Path(sys.prefix).resolve() in package.parents, package + native = Path(_ffi.lib._name).resolve() + assert native.parent == package, f"native library was not loaded from the installed wheel: {native}" + + class Input: + def __init__(self, data): + self.data = data + + def pread_many(self, ranges): + return [self.data[pos:pos + length] for pos, length in ranges] + + data = np.array([[0, 0], [0.1, 0], [10, 10], [10.1, 10]], dtype=np.float32) + ids = np.array([101, 102, 103, 104], dtype=np.int64) + options = {"index.type": "ivf_flat", "dimension": "2", "nlist": "2", "metric": "l2"} + output = io.BytesIO() + with VectorIndexWriter(VectorIndexTrainer.train(options, data)) as writer: + writer.add_vectors(ids, data) + writer.write(output) + assert output.tell() == len(output.getvalue()) > 64 + with VectorIndexReader(Input(output.getvalue())) as reader: + reader.optimize_for_search() + actual_ids, distances = reader.search(data[0], SearchParams.ivf(top_k=4, nprobe=2)) + np.testing.assert_array_equal(actual_ids, ids) + np.testing.assert_allclose(distances, np.sum(data * data, axis=1), rtol=1e-5, atol=1e-6) + print(f"Final wheel smoke passed: {package}, native {native.name}") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--wheel", help="one final wheel path or glob") + parser.add_argument("--installed", action="store_true", help=argparse.SUPPRESS) + args = parser.parse_args() + if args.installed: + smoke_installed() + return + wheels = sorted(glob.glob(args.wheel or "")) + if len(wheels) != 1: + parser.error(f"expected exactly one final wheel, found {len(wheels)}") + wheel = str(Path(wheels[0]).resolve()) + script = str(Path(__file__).resolve()) + env = os.environ.copy() + for key in ("PAIMON_VINDEX_LIB_PATH", "PYTHONPATH", "PYTHONHOME", "LD_LIBRARY_PATH", "DYLD_LIBRARY_PATH"): + env.pop(key, None) + with tempfile.TemporaryDirectory(prefix="paimon-wheel-smoke-") as directory: + root = Path(directory) + venv.create(root / "venv", with_pip=True) + python = root / "venv" / ("Scripts/python.exe" if os.name == "nt" else "bin/python") + subprocess.run([str(python), "-I", "-m", "pip", "install", "--only-binary=:all:", wheel], cwd=root, env=env, check=True) + subprocess.run([str(python), "-I", script, "--installed"], cwd=root, env=env, check=True) + + +if __name__ == "__main__": + main() diff --git a/tools/tests/test_release_scripts.py b/tools/tests/test_release_scripts.py index aa650ca..1fae081 100644 --- a/tools/tests/test_release_scripts.py +++ b/tools/tests/test_release_scripts.py @@ -17,6 +17,7 @@ import os import shutil import subprocess +import sys import tempfile import unittest from pathlib import Path @@ -197,6 +198,53 @@ def test_create_release_branch_uses_minor_release_line(self): ).stdout.strip() self.assertEqual("release-0.4", branch) + def _preflight(self, ref): + return self._run( + [sys.executable, str(REPO_ROOT / "tools/check_release_versions.py"), + "--root", str(self.repo), "--ref", ref], + check=False, capture_output=True, + ) + + def _release_pom(self): + path = self.repo / "java/pom.xml" + path.write_text(path.read_text().replace("0.4.0-SNAPSHOT", "0.4.0")) + + def test_release_preflight_accepts_final_and_rc_tags_with_clean_source_versions(self): + self._release_pom() + for ref in ["refs/tags/v0.4.0", "refs/tags/v0.4.0-rc1", "refs/tags/v0.4.0-rc12"]: + with self.subTest(ref=ref): + result = self._preflight(ref) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_release_preflight_rejects_each_version_mismatch(self): + self._release_pom() + for relative in ["core/Cargo.toml", "ffi/Cargo.toml", "jni/Cargo.toml", "python/pyproject.toml", "java/pom.xml"]: + path = self.repo / relative + original = path.read_text() + for tag in ["refs/tags/v0.4.0", "refs/tags/v0.4.0-rc2"]: + with self.subTest(file=relative, tag=tag): + path.write_text(original.replace("0.4.0", "0.5.0", 1)) + result = self._preflight(tag) + self.assertNotEqual(0, result.returncode) + self.assertIn("preflight failed", result.stderr) + path.write_text(original) + + def test_release_preflight_rejects_snapshot_dependencies_and_non_tags(self): + result = self._preflight("refs/tags/v0.4.0") + self.assertNotEqual(0, result.returncode) + self.assertIn("Maven", result.stderr) + self._release_pom() + path = self.repo / "ffi/Cargo.toml" + path.write_text(path.read_text().replace('version = "0.4.0" }', 'version = "0.3.0" }')) + result = self._preflight("refs/tags/v0.4.0") + self.assertNotEqual(0, result.returncode) + self.assertIn("paimon-vindex-ffi -> paimon-vindex-core", result.stderr) + for ref in ["", "refs/heads/main", "refs/pull/123/merge", "refs/tags/v0.4.0-beta1", "refs/tags/v0.4.0-rc0"]: + with self.subTest(ref=ref): + result = self._preflight(ref) + self.assertNotEqual(0, result.returncode) + self.assertIn("requires a vX.Y.Z", result.stderr) + def test_update_branch_version_reports_missing_old_version(self): env = os.environ.copy() env.pop("OLD_VERSION", None)