Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions .github/workflows/release-java.yml
Original file line number Diff line number Diff line change
Expand Up @@ -239,7 +239,10 @@ jobs:
)"
test -n "$jar_file"
mkdir -p smoke-classes
javac -cp "$jar_file" -d smoke-classes \
java/src/test/java/org/apache/paimon/index/vector/VectorIndexNativeLoaderSmokeTest.java
javac -cp "$jar_file" -sourcepath java/src/test/java -d smoke-classes \
java/src/test/java/org/apache/paimon/index/vector/VectorIndexNativeLoaderSmokeTest.java \
java/src/test/java/org/apache/paimon/index/vector/VectorIndexCentroidShardingTest.java
java -cp "$jar_file${{ matrix.classpath_separator }}smoke-classes" \
org.apache.paimon.index.vector.VectorIndexNativeLoaderSmokeTest
java -cp "$jar_file${{ matrix.classpath_separator }}smoke-classes" \
org.apache.paimon.index.vector.VectorIndexCentroidShardingTest
19 changes: 19 additions & 0 deletions core/STORAGE_FORMAT.md
Original file line number Diff line number Diff line change
Expand Up @@ -456,3 +456,22 @@ normally use their own per-dimension residual bounds. A reader validates all
offsets, counts, encoded-ID sizes, and SQ bounds before exposing the index.
Putting blocked codes first lets the reader retain the list payload allocation
as the scan buffer after decoding and truncating the trailing ID section.

## Centroid routing model

A router can select files containing any supported local index type. The routing
payload is independent of the standard index file inside each populated shard.
The file mapping and the global row IDs belong to the caller.

| Offset | Size | Value |
| ---: | ---: | --- |
| 0 | 8 | ASCII `PVROUT01` |
| 8 | 4 | Vector dimension, little-endian `u32` |
| 12 | 4 | Number of routing centers, little-endian `u32` |
| 16 | 4 | Metric code, as in the index header |
| 20 | `4 * dimension * centers` | Center coordinates, little-endian `f32`, grouped by center ID |

Dimension and center count are positive. The reader checks the metadata size bound,
the exact payload length, the metric code and finite center coordinates. Row
assignment and query routing use squared L2 distances to these centers, with query
and training vectors normalized for cosine. Ties prefer the lower center ID.
88 changes: 82 additions & 6 deletions core/src/index.rs
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,9 @@ use roaring::RoaringTreemap;
use std::collections::{HashMap, HashSet};
use std::io::{self, Cursor};

mod router;
pub use router::CentroidRouter;

/// Default ratio between the serialized PQ code and the raw `f32` vector.
///
/// At 8 bits per subquantizer this resolves to one PQ subquantizer per four
Expand Down Expand Up @@ -1369,6 +1372,30 @@ impl VectorIndexTrainer {
self.writer.dimension()
}

/// Input ordinals retained by streaming this many vectors, in final reservoir slot order.
/// Call on a fresh trainer. Planning uses no vectors and leaves the trainer untouched.
pub fn plan_training_sample(&self, vector_count: usize) -> io::Result<Vec<usize>> {
if self.training_vectors_seen != 0 {
return Err(invalid_input(
"plan the training sample before adding vectors",
));
}
let limit = self.training_sample_limit;
let mut ordinals: Vec<_> = (0..vector_count.min(limit)).collect();
let mut rng = self.training_rng.clone();
for ordinal in limit..vector_count {
if let Some(slot) = Self::replacement_slot(&mut rng, ordinal, limit) {
ordinals[slot] = ordinal;
}
}
Ok(ordinals)
}

fn replacement_slot(rng: &mut StdRng, ordinal: usize, limit: usize) -> Option<usize> {
let slot = rng.gen_range(0..=ordinal);
(slot < limit).then_some(slot)
}

pub fn add_training_vectors(mut self, data: &[f32], n: usize) -> io::Result<Self> {
self.add_training_vectors_mut(data, n)?;
Ok(self)
Expand All @@ -1387,12 +1414,13 @@ impl VectorIndexTrainer {
let stream_index = self.training_vectors_seen + batch_index;
if stream_index < self.training_sample_limit {
self.training_data.extend_from_slice(vector);
} else {
let replacement = self.training_rng.gen_range(0..=stream_index);
if replacement < self.training_sample_limit {
let start = replacement * dimension;
self.training_data[start..start + dimension].copy_from_slice(vector);
}
} else if let Some(slot) = Self::replacement_slot(
&mut self.training_rng,
stream_index,
self.training_sample_limit,
) {
let start = slot * dimension;
self.training_data[start..start + dimension].copy_from_slice(vector);
}
}
self.training_vector_count = training_vectors_seen.min(self.training_sample_limit);
Expand Down Expand Up @@ -1451,6 +1479,8 @@ impl VectorIndexWriter {
use_approximate_coarse_assignment,
..
} => {
// Check the on-disk metadata bound before allocating per-list state.
crate::ivfflat_io::flat_metadata_bytes(dimension, nlist)?;
let mut index = IVFFlatIndex::new(dimension, nlist, metric);
index.set_approximate_coarse_assignment(use_approximate_coarse_assignment);
Self::IvfFlat(index)
Expand Down Expand Up @@ -3264,6 +3294,43 @@ mod tests {
}
}

#[test]
fn planned_samples_match_streaming_and_can_be_replayed_for_each_index_type() {
for kind in ["ivf_flat", "ivf_sq", "ivf_pq", "ivf_rq", "diskann"] {
let mut options = HashMap::from([
("index.type".to_string(), kind.to_string()),
("dimension".into(), "4".into()),
("nlist".into(), "4".into()),
("metric".into(), "l2".into()),
]);
if kind == "diskann" {
options.remove("nlist");
options.insert("diskann.seed".into(), "93".into());
}
let config = VectorIndexConfig::from_options(&options).unwrap();
let mut trainer = VectorIndexTrainer::new(config.clone()).unwrap();
let n = trainer.training_sample_limit * 2 + 37;
let data: Vec<_> = (0..n * 4).map(|i| (i % 2017) as f32).collect();
let ordinals = trainer.plan_training_sample(n).unwrap();
let sample: Vec<_> = ordinals
.iter()
.flat_map(|&i| &data[i * 4..i * 4 + 4])
.copied()
.collect();
for chunk in data.chunks(67 * 4) {
trainer
.add_training_vectors_mut(chunk, chunk.len() / 4)
.unwrap();
}
assert_eq!(sample, trainer.training_data, "{kind}");
let replay = VectorIndexTrainer::new(config)
.unwrap()
.add_training_vectors(&sample, ordinals.len())
.unwrap();
assert_eq!(replay.training_data, trainer.training_data, "{kind}");
}
}

#[test]
fn unified_search_stats_cover_all_families_calls_filters_and_storage() {
let pool = rayon::ThreadPoolBuilder::new()
Expand Down Expand Up @@ -4862,7 +4929,16 @@ mod tests {
};

let mut whole = VectorIndexTrainer::new(config()).unwrap();
let positions = whole.plan_training_sample(count).unwrap();
assert_eq!(positions, whole.plan_training_sample(count).unwrap());
let selected: Vec<_> = positions
.iter()
.flat_map(|&i| &data[i * dimension..(i + 1) * dimension])
.copied()
.collect();
whole.add_training_vectors_mut(&data, count).unwrap();
assert_eq!(selected, whole.training_data);
assert!(whole.plan_training_sample(count).is_err());
let mut batched = VectorIndexTrainer::new(config()).unwrap();
for vectors in data.chunks(137 * dimension) {
batched
Expand Down
219 changes: 219 additions & 0 deletions core/src/index/router.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,219 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

use super::*;
use crate::distance::{fvec_l2sqr, fvec_normalize};
use crate::kmeans::find_topk_checked;
use std::borrow::Cow;

const MAGIC: &[u8; 8] = b"PVROUT01";
const HEADER: usize = 20;

/// Exact coarse routing shared by all shard index types. Contains no shard training state.
pub struct CentroidRouter {
dimension: usize,
metric: MetricType,
centers: Vec<f32>,
}

impl CentroidRouter {
fn payload_len(d: usize, nlist: usize) -> io::Result<usize> {
if d == 0 || nlist == 0 || d > i32::MAX as usize || nlist > i32::MAX as usize {
return Err(invalid_input("invalid centroid router dimensions"));
}
// Bound both center storage and the per-list allocations of an IVF-FLAT trainer.
crate::ivfflat_io::flat_metadata_bytes(d, nlist)?;
Ok(d * nlist * 4)
}

pub fn dimension(&self) -> usize {
self.dimension
}

pub fn nlist(&self) -> usize {
self.centers.len() / self.dimension
}

pub fn serialize(&self) -> Vec<u8> {
let mut bytes = Vec::with_capacity(HEADER + self.centers.len() * 4);
bytes.extend_from_slice(MAGIC);
bytes.extend_from_slice(&(self.dimension as u32).to_le_bytes());
bytes.extend_from_slice(&(self.nlist() as u32).to_le_bytes());
bytes.extend_from_slice(&(self.metric as u32).to_le_bytes());
for v in &self.centers {
bytes.extend_from_slice(&v.to_le_bytes());
}
bytes
}

/// Largest supported serialized router, including its fixed header.
pub const MAX_SERIALIZED_BYTES: usize =
HEADER + crate::index_io_util::MAX_READER_METADATA_BYTES / 2;
pub const HEADER_BYTES: usize = HEADER;

/// Check the fixed header and return the exact serialized size before allocating its body.
pub fn serialized_size(header: &[u8]) -> io::Result<usize> {
let (dimension, nlist, _) = Self::header(header)?;
Ok(HEADER + Self::payload_len(dimension, nlist)?)
}

fn header(bytes: &[u8]) -> io::Result<(usize, usize, MetricType)> {
if bytes.len() < HEADER || &bytes[..8] != MAGIC {
return Err(invalid_input("invalid centroid router header"));
}
let number = |offset| u32::from_le_bytes(bytes[offset..offset + 4].try_into().unwrap());
let dimension = number(8) as usize;
let nlist = number(12) as usize;
let metric = MetricType::from_code(number(16))
.ok_or_else(|| invalid_input("invalid routing metric"))?;
Ok((dimension, nlist, metric))
}

pub fn deserialize(bytes: &[u8]) -> io::Result<Self> {
let (dimension, nlist, metric) = Self::header(bytes)?;
if Self::payload_len(dimension, nlist)? != bytes.len() - HEADER {
return Err(invalid_input("invalid centroid router length"));
}
let centers = bytes[HEADER..]
.chunks_exact(4)
.map(|b| f32::from_le_bytes(b.try_into().unwrap()))
.collect();
Self::from_centers(dimension, metric, centers)
}

fn from_centers(dimension: usize, metric: MetricType, centers: Vec<f32>) -> io::Result<Self> {
if centers.iter().any(|v| !v.is_finite()) {
return Err(invalid_input("non-finite routing centroid"));
}
Ok(Self {
dimension,
metric,
centers,
})
}

/// Resolve outer routing with the same automatic width policy as ordinary IVF search.
pub fn resolve_nprobe(
&self,
params: VectorSearchParams,
total_vectors: usize,
matching_vectors: Option<usize>,
) -> io::Result<usize> {
params.validate()?;
params.resolve_ivf_nprobe(self.nlist(), total_vectors, matching_vectors)
}

fn routing_vector<'a>(&self, query: &'a [f32]) -> io::Result<Cow<'a, [f32]>> {
let d = self.dimension();
validate_query(query, d)?;
let mut processed = Cow::Borrowed(query);
if self.metric == MetricType::Cosine {
fvec_normalize(processed.to_mut());
}
if let Cow::Owned(ref values) = processed {
validate_query(values, d)?;
}
Ok(processed)
}

/// Exact coarse routing, for a query in the same metric space as training.
pub fn find_nearest_centroids(&self, query: &[f32], nprobe: usize) -> io::Result<Vec<usize>> {
validate_positive(nprobe, "nprobe")?;
let processed = self.routing_vector(query)?;
find_topk_checked(
&processed,
&self.centers,
self.nlist(),
self.dimension(),
nprobe,
)
.map(|centroids| {
centroids
.into_iter()
.map(|(_, centroid)| centroid)
.collect()
})
.map_err(|_| invalid_input("non-finite coarse distance"))
}

pub fn assign_centroid(&self, vector: &[f32]) -> io::Result<usize> {
let point = self.routing_vector(vector)?;
let mut nearest = 0;
let mut best = f32::INFINITY;
for (id, center) in self.centers.chunks_exact(self.dimension()).enumerate() {
let distance = fvec_l2sqr(&point, center);
if !distance.is_finite() {
return Err(invalid_input("non-finite coarse distance"));
}
if distance < best {
best = distance;
nearest = id;
}
}
Ok(nearest)
}

/// Take the coarse centers from a trained IVF-FLAT model.
pub fn from_training(training: VectorIndexTraining) -> io::Result<Self> {
let VectorIndexWriter::IvfFlat(flat) = training.inner else {
return Err(invalid_input(
"a centroid router requires IVF-FLAT training",
));
};
Self::payload_len(flat.d, flat.nlist)?;
Self::from_centers(flat.d, flat.metric, flat.into_quantizer_centroids())
}
}

#[cfg(test)]
mod tests {
use super::*;

#[test]
fn router_payload_is_independent_of_shard_type_and_rejects_corruption() {
// Frozen routing format: two 2D cosine centers, including a tie on [1, 1].
let mut bytes = b"PVROUT01".to_vec();
for n in [2u32, 2, 2] {
bytes.extend(n.to_le_bytes());
}
for f in [1.0f32, 0.0, 0.0, 1.0] {
bytes.extend(f.to_le_bytes());
}
let router = CentroidRouter::deserialize(&bytes).unwrap();
assert_eq!(router.serialize(), bytes);
assert_eq!(router.assign_centroid(&[0., 30.]).unwrap(), 1);
assert_eq!(router.assign_centroid(&[1., 1.]).unwrap(), 0);
assert_eq!(router.find_nearest_centroids(&[1., 1.], 9).unwrap(), [0, 1]);
assert!(router.assign_centroid(&[f32::NAN, 0.]).is_err());
assert!(router.find_nearest_centroids(&[0.], 1).is_err());
assert!(router.find_nearest_centroids(&[1., 0.], 0).is_err());
for (offset, word) in [
(0, 0u32),
(8, 0),
(12, u32::MAX),
(16, 7),
(20, f32::INFINITY.to_bits()),
] {
let mut bad = bytes.clone();
bad[offset..offset + 4].copy_from_slice(&word.to_le_bytes());
assert!(CentroidRouter::deserialize(&bad).is_err());
}
assert!(CentroidRouter::deserialize(&bytes[..bytes.len() - 1]).is_err());
bytes.push(0);
assert!(CentroidRouter::deserialize(&bytes).is_err());
}
}
4 changes: 4 additions & 0 deletions core/src/ivfflat.rs
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,10 @@ impl IVFFlatIndex {
&self.quantizer_centroids
}

pub(crate) fn into_quantizer_centroids(self) -> Vec<f32> {
self.quantizer_centroids
}

/// Enables automatic Vamana coarse assignment for large centroid matrices.
/// Disable it to keep vector assignment exact.
pub(crate) fn set_approximate_coarse_assignment(&mut self, enabled: bool) {
Expand Down
Loading
Loading