This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/paimon-vector-index.git
The following commit(s) were added to refs/heads/main by this push:
new d184608 feat: add configurable IVF and PQ training sample limits (#98)
d184608 is described below
commit d184608ca527dfe2b25e5d76769e777f16be1f89
Author: jerry <[email protected]>
AuthorDate: Thu Sep 10 15:09:12 2026 +0800
feat: add configurable IVF and PQ training sample limits (#98)
---
core/benches/ann_bench.rs | 6 ++
core/src/diskann.rs | 11 +-
core/src/index.rs | 253 ++++++++++++++++++++++++++++++++++++++++++++--
core/src/ivfflat.rs | 7 +-
core/src/ivfpq.rs | 17 +++-
core/src/ivfrq.rs | 7 +-
core/src/ivfsq.rs | 7 +-
core/src/kmeans.rs | 2 +
core/src/opq.rs | 47 ++++++++-
docs/api.html | 10 ++
docs/diskann.html | 2 +
docs/ivf-flat.html | 4 +-
docs/ivf-pq.html | 3 +-
docs/ivf-rq.html | 4 +-
docs/ivf-sq.html | 4 +-
docs/releases.html | 1 +
16 files changed, 360 insertions(+), 25 deletions(-)
diff --git a/core/benches/ann_bench.rs b/core/benches/ann_bench.rs
index 0d79609..16c15c6 100644
--- a/core/benches/ann_bench.rs
+++ b/core/benches/ann_bench.rs
@@ -645,6 +645,7 @@ fn index_specs(config: &Config) -> Vec<IndexSpec> {
nlist: config.nlist,
metric: MetricType::L2,
use_approximate_coarse_assignment: true,
+ ivf_train_max_points_per_centroid: 256,
},
searches: vec![ivf_search],
},
@@ -655,6 +656,7 @@ fn index_specs(config: &Config) -> Vec<IndexSpec> {
nlist: config.nlist,
metric: MetricType::L2,
use_approximate_coarse_assignment: true,
+ ivf_train_max_points_per_centroid: 256,
},
searches: vec![ivf_search],
},
@@ -668,6 +670,8 @@ fn index_specs(config: &Config) -> Vec<IndexSpec> {
use_opq: false,
use_approximate_coarse_assignment: true,
canonical_pq_encoding: false,
+ ivf_train_max_points_per_centroid: 256,
+ pq_train_max_points_per_centroid: 256,
},
searches: vec![ivf_search],
},
@@ -679,6 +683,7 @@ fn index_specs(config: &Config) -> Vec<IndexSpec> {
bits: config.rq_bits,
metric: MetricType::L2,
use_approximate_coarse_assignment: true,
+ ivf_train_max_points_per_centroid: 256,
},
searches: vec![ivf_search],
},
@@ -695,6 +700,7 @@ fn index_specs(config: &Config) -> Vec<IndexSpec> {
raw_vector_encoding: config.diskann_raw_vector_encoding,
..DiskAnnBuildParams::default()
},
+ pq_train_max_points_per_centroid: 256,
},
searches: config
.diskann_l_searches
diff --git a/core/src/diskann.rs b/core/src/diskann.rs
index 30ae782..b668b59 100644
--- a/core/src/diskann.rs
+++ b/core/src/diskann.rs
@@ -282,6 +282,15 @@ impl DiskAnnIndex {
}
pub fn train(&mut self, data: &[f32], n: usize) -> io::Result<()> {
+ self.train_with_config(data, n, &KMeansConfig::default())
+ }
+
+ pub fn train_with_config(
+ &mut self,
+ data: &[f32],
+ n: usize,
+ config: &KMeansConfig,
+ ) -> io::Result<()> {
if n == 0 {
return Err(invalid_input(
"DiskANN training vector count must be greater than zero",
@@ -307,7 +316,7 @@ impl DiskAnnIndex {
self.pq.train_hot_start_with_parallelism(
&processed,
plan.sample_count,
- &KMeansConfig::default(),
+ config,
false,
plan.parallelism,
);
diff --git a/core/src/index.rs b/core/src/index.rs
index 9acdee5..6b3e491 100644
--- a/core/src/index.rs
+++ b/core/src/index.rs
@@ -50,6 +50,7 @@ use crate::ivfsq_io::{
search_batch_ivfsq_reader_filter_range,
search_batch_ivfsq_reader_roaring_filter_range,
write_ivfsq_index, IVFSQIndexReader, IVF_SQ_MAGIC,
};
+use crate::kmeans::KMeansConfig;
pub use crate::read_options::{DeploymentProfile, VectorIndexReadPlan,
VectorIndexReaderOptions};
use crate::rq::{is_supported_rq_bits, padded_dimension, DEFAULT_RQ_BITS};
use rand::rngs::StdRng;
@@ -147,6 +148,7 @@ pub enum VectorIndexConfig {
nlist: usize,
metric: MetricType,
use_approximate_coarse_assignment: bool,
+ ivf_train_max_points_per_centroid: usize,
},
IvfPq {
dimension: usize,
@@ -158,6 +160,8 @@ pub enum VectorIndexConfig {
/// Use canonical expanded-form PQ encoding instead of the default
/// transposed direct-L2 encoder.
canonical_pq_encoding: bool,
+ ivf_train_max_points_per_centroid: usize,
+ pq_train_max_points_per_centroid: usize,
},
IvfRq {
dimension: usize,
@@ -165,12 +169,14 @@ pub enum VectorIndexConfig {
bits: usize,
metric: MetricType,
use_approximate_coarse_assignment: bool,
+ ivf_train_max_points_per_centroid: usize,
},
IvfSq {
dimension: usize,
nlist: usize,
metric: MetricType,
use_approximate_coarse_assignment: bool,
+ ivf_train_max_points_per_centroid: usize,
},
DiskAnn {
dimension: usize,
@@ -178,6 +184,7 @@ pub enum VectorIndexConfig {
pq_m: usize,
pq_bits: usize,
build: DiskAnnBuildParams,
+ pq_train_max_points_per_centroid: usize,
},
}
@@ -211,6 +218,8 @@ impl VectorIndexConfig {
use_opq,
use_approximate_coarse_assignment: true,
canonical_pq_encoding: false,
+ ivf_train_max_points_per_centroid: 256,
+ pq_train_max_points_per_centroid: 256,
};
validate_config(&config)?;
Ok(config)
@@ -228,6 +237,7 @@ impl VectorIndexConfig {
pq_m: infer_pq_m(dimension, pq_bits, DEFAULT_PQ_CODE_RATIO)?,
pq_bits,
build,
+ pq_train_max_points_per_centroid: 256,
};
validate_config(&config)?;
Ok(config)
@@ -240,6 +250,7 @@ impl VectorIndexConfig {
bits: DEFAULT_RQ_BITS,
metric,
use_approximate_coarse_assignment: true,
+ ivf_train_max_points_per_centroid: 256,
};
validate_config(&config)?;
Ok(config)
@@ -290,6 +301,8 @@ pub struct ResolvedVectorIndexConfig {
/// Only meaningful for IVF-PQ.
pub canonical_pq_encoding: bool,
pub diskann_build: Option<DiskAnnBuildParams>,
+ pub ivf_train_max_points_per_centroid: Option<usize>,
+ pub pq_train_max_points_per_centroid: Option<usize>,
}
impl From<&VectorIndexConfig> for ResolvedVectorIndexConfig {
@@ -300,12 +313,14 @@ impl From<&VectorIndexConfig> for
ResolvedVectorIndexConfig {
nlist,
metric,
use_approximate_coarse_assignment,
+ ivf_train_max_points_per_centroid,
}
| VectorIndexConfig::IvfSq {
dimension,
nlist,
metric,
use_approximate_coarse_assignment,
+ ivf_train_max_points_per_centroid,
} => Self {
index_type: config.index_type(),
dimension: *dimension,
@@ -318,6 +333,8 @@ impl From<&VectorIndexConfig> for ResolvedVectorIndexConfig
{
use_approximate_coarse_assignment:
*use_approximate_coarse_assignment,
canonical_pq_encoding: false,
diskann_build: None,
+ ivf_train_max_points_per_centroid:
Some(*ivf_train_max_points_per_centroid),
+ pq_train_max_points_per_centroid: None,
},
VectorIndexConfig::IvfPq {
dimension,
@@ -327,6 +344,8 @@ impl From<&VectorIndexConfig> for ResolvedVectorIndexConfig
{
use_opq,
use_approximate_coarse_assignment,
canonical_pq_encoding,
+ ivf_train_max_points_per_centroid,
+ pq_train_max_points_per_centroid,
} => Self {
index_type: IndexType::IvfPq,
dimension: *dimension,
@@ -339,6 +358,8 @@ impl From<&VectorIndexConfig> for ResolvedVectorIndexConfig
{
use_approximate_coarse_assignment:
*use_approximate_coarse_assignment,
canonical_pq_encoding: *canonical_pq_encoding,
diskann_build: None,
+ ivf_train_max_points_per_centroid:
Some(*ivf_train_max_points_per_centroid),
+ pq_train_max_points_per_centroid:
Some(*pq_train_max_points_per_centroid),
},
VectorIndexConfig::IvfRq {
dimension,
@@ -346,6 +367,7 @@ impl From<&VectorIndexConfig> for ResolvedVectorIndexConfig
{
bits,
metric,
use_approximate_coarse_assignment,
+ ivf_train_max_points_per_centroid,
} => Self {
index_type: IndexType::IvfRq,
dimension: *dimension,
@@ -358,6 +380,8 @@ impl From<&VectorIndexConfig> for ResolvedVectorIndexConfig
{
use_approximate_coarse_assignment:
*use_approximate_coarse_assignment,
canonical_pq_encoding: false,
diskann_build: None,
+ ivf_train_max_points_per_centroid:
Some(*ivf_train_max_points_per_centroid),
+ pq_train_max_points_per_centroid: None,
},
VectorIndexConfig::DiskAnn {
dimension,
@@ -365,6 +389,7 @@ impl From<&VectorIndexConfig> for ResolvedVectorIndexConfig
{
pq_m,
pq_bits,
build,
+ pq_train_max_points_per_centroid,
} => Self {
index_type: IndexType::DiskAnn,
dimension: *dimension,
@@ -377,6 +402,8 @@ impl From<&VectorIndexConfig> for ResolvedVectorIndexConfig
{
use_approximate_coarse_assignment: false,
canonical_pq_encoding: false,
diskann_build: Some(*build),
+ ivf_train_max_points_per_centroid: None,
+ pq_train_max_points_per_centroid:
Some(*pq_train_max_points_per_centroid),
},
}
}
@@ -450,6 +477,10 @@ impl VectorIndexBuildPlan {
nlist: parse_nlist_options(&mut options,
expected_vector_count)?,
metric,
use_approximate_coarse_assignment,
+ ivf_train_max_points_per_centroid:
parse_training_max_points_per_centroid(
+ &mut options,
+ "ivf.train.max-points-per-centroid",
+ )?,
},
IndexType::IvfPq => VectorIndexConfig::IvfPq {
dimension,
@@ -473,6 +504,14 @@ impl VectorIndexBuildPlan {
},
use_approximate_coarse_assignment,
canonical_pq_encoding,
+ ivf_train_max_points_per_centroid:
parse_training_max_points_per_centroid(
+ &mut options,
+ "ivf.train.max-points-per-centroid",
+ )?,
+ pq_train_max_points_per_centroid:
parse_training_max_points_per_centroid(
+ &mut options,
+ "pq.train.max-points-per-centroid",
+ )?,
},
IndexType::IvfRq => {
let explicit_bits = options
@@ -494,6 +533,10 @@ impl VectorIndexBuildPlan {
bits,
metric,
use_approximate_coarse_assignment,
+ ivf_train_max_points_per_centroid:
parse_training_max_points_per_centroid(
+ &mut options,
+ "ivf.train.max-points-per-centroid",
+ )?,
}
}
IndexType::IvfSq => VectorIndexConfig::IvfSq {
@@ -501,6 +544,10 @@ impl VectorIndexBuildPlan {
nlist: parse_nlist_options(&mut options,
expected_vector_count)?,
metric,
use_approximate_coarse_assignment,
+ ivf_train_max_points_per_centroid:
parse_training_max_points_per_centroid(
+ &mut options,
+ "ivf.train.max-points-per-centroid",
+ )?,
},
IndexType::DiskAnn => {
let pq_bits = match options.optional("pq.bits") {
@@ -547,6 +594,10 @@ impl VectorIndexBuildPlan {
)?,
pq_bits,
build,
+ pq_train_max_points_per_centroid:
parse_training_max_points_per_centroid(
+ &mut options,
+ "pq.train.max-points-per-centroid",
+ )?,
}
}
};
@@ -620,6 +671,17 @@ impl ConfigOptions {
}
}
+fn parse_training_max_points_per_centroid(
+ options: &mut ConfigOptions,
+ key: &str,
+) -> io::Result<usize> {
+ options
+ .optional(key)
+ .map(|value| parse_usize_option(key, &value))
+ .transpose()
+ .map(|value|
value.unwrap_or(KMeansConfig::default().max_points_per_centroid))
+}
+
fn parse_nlist_options(
options: &mut ConfigOptions,
expected_vector_count: Option<usize>,
@@ -1224,6 +1286,8 @@ pub struct DiskAnnMetadata {
pub struct VectorIndexTrainer {
writer: VectorIndexWriter,
+ ivf_training: KMeansConfig,
+ pq_training: KMeansConfig,
training_data: Vec<f32>,
training_vector_count: usize,
training_vectors_seen: usize,
@@ -1233,6 +1297,15 @@ pub struct VectorIndexTrainer {
impl VectorIndexTrainer {
pub fn new(config: VectorIndexConfig) -> io::Result<Self> {
+ let resolved = config.resolved();
+ let mut ivf_training = KMeansConfig::default();
+ let mut pq_training = KMeansConfig::default();
+ if let Some(max_points) = resolved.ivf_train_max_points_per_centroid {
+ ivf_training.max_points_per_centroid = max_points;
+ }
+ if let Some(max_points) = resolved.pq_train_max_points_per_centroid {
+ pq_training.max_points_per_centroid = max_points;
+ }
let training_sample_limit = match &config {
VectorIndexConfig::DiskAnn {
dimension,
@@ -1240,6 +1313,7 @@ impl VectorIndexTrainer {
pq_m,
pq_bits,
build,
+ ..
} => diskann_training_sample_limit(
*dimension,
*metric,
@@ -1256,6 +1330,8 @@ impl VectorIndexTrainer {
let writer = VectorIndexWriter::from_config(config)?;
Ok(Self {
writer,
+ ivf_training,
+ pq_training,
training_data: Vec::new(),
training_vector_count: 0,
training_vectors_seen: 0,
@@ -1311,8 +1387,12 @@ impl VectorIndexTrainer {
if self.training_vector_count == 0 || self.training_data.is_empty() {
return Err(invalid_input("no training vectors added"));
}
- self.writer
- .train_internal(&self.training_data, self.training_vector_count)?;
+ self.writer.train_internal(
+ &self.training_data,
+ self.training_vector_count,
+ &self.ivf_training,
+ &self.pq_training,
+ )?;
Ok(VectorIndexTraining { inner: self.writer })
}
}
@@ -1352,6 +1432,7 @@ impl VectorIndexWriter {
nlist,
metric,
use_approximate_coarse_assignment,
+ ..
} => {
let mut index = IVFFlatIndex::new(dimension, nlist, metric);
index.set_approximate_coarse_assignment(use_approximate_coarse_assignment);
@@ -1362,6 +1443,7 @@ impl VectorIndexWriter {
nlist,
metric,
use_approximate_coarse_assignment,
+ ..
} => {
let mut index = IVFSQIndex::new(dimension, nlist, metric);
index.set_approximate_coarse_assignment(use_approximate_coarse_assignment);
@@ -1375,6 +1457,7 @@ impl VectorIndexWriter {
use_opq,
use_approximate_coarse_assignment,
canonical_pq_encoding,
+ ..
} => {
let mut index = IVFPQIndex::new(dimension, nlist, m, metric,
use_opq);
index.set_approximate_coarse_assignment(use_approximate_coarse_assignment);
@@ -1387,6 +1470,7 @@ impl VectorIndexWriter {
bits,
metric,
use_approximate_coarse_assignment,
+ ..
} => {
let mut index = IVFRQIndex::with_bits(dimension, nlist, bits,
metric);
index.set_approximate_coarse_assignment(use_approximate_coarse_assignment);
@@ -1398,6 +1482,7 @@ impl VectorIndexWriter {
pq_m,
pq_bits,
build,
+ ..
} => Self::DiskAnn(DiskAnnIndex::with_pq_bits(
dimension, metric, pq_m, pq_bits, build,
)),
@@ -1424,14 +1509,20 @@ impl VectorIndexWriter {
}
}
- fn train_internal(&mut self, data: &[f32], n: usize) -> io::Result<()> {
+ fn train_internal(
+ &mut self,
+ data: &[f32],
+ n: usize,
+ ivf_training: &KMeansConfig,
+ pq_training: &KMeansConfig,
+ ) -> io::Result<()> {
debug_assert_eq!(Some(data.len()), n.checked_mul(self.dimension()));
match self {
- Self::IvfFlat(index) => index.train(data, n),
- Self::IvfSq(index) => index.train(data, n),
- Self::IvfPq(index) => index.train(data, n),
- Self::IvfRq(index) => index.train(data, n),
- Self::DiskAnn(index) => return index.train(data, n),
+ Self::IvfFlat(index) => index.train_with_config(data, n,
ivf_training),
+ Self::IvfSq(index) => index.train_with_config(data, n,
ivf_training),
+ Self::IvfPq(index) => index.train_with_config(data, n,
ivf_training, pq_training),
+ Self::IvfRq(index) => index.train_with_config(data, n,
ivf_training),
+ Self::DiskAnn(index) => return index.train_with_config(data, n,
pq_training),
}
Ok(())
}
@@ -2375,11 +2466,32 @@ fn validate_config(config: &VectorIndexConfig) ->
io::Result<()> {
pq_m,
pq_bits,
build,
+ ..
} => {
validate_diskann_config(*dimension, *metric, *pq_m, *pq_bits,
*build)?;
}
_ => {}
}
+ let resolved = config.resolved();
+ for (key, max_points, centroids) in [
+ (
+ "ivf.train.max-points-per-centroid",
+ resolved.ivf_train_max_points_per_centroid,
+ config.nlist(),
+ ),
+ (
+ "pq.train.max-points-per-centroid",
+ resolved.pq_train_max_points_per_centroid,
+ 1usize << resolved.pq_bits.unwrap_or(8),
+ ),
+ ] {
+ if let Some(max_points) = max_points {
+ validate_positive(max_points, key)?;
+ centroids.checked_mul(max_points).ok_or_else(|| {
+ invalid_input(format!("{key} times centroid count overflows
usize"))
+ })?;
+ }
+ }
Ok(())
}
@@ -2691,6 +2803,7 @@ mod tests {
nlist: 1,
metric: MetricType::L2,
use_approximate_coarse_assignment: true,
+ ivf_train_max_points_per_centroid: 256,
},
&[0.0, 1.0],
2,
@@ -2720,6 +2833,7 @@ mod tests {
build_search_list_size: 16,
..DiskAnnBuildParams::default()
},
+ pq_train_max_points_per_centroid: 256,
});
reader
@@ -2744,6 +2858,7 @@ mod tests {
metric: MetricType::L2,
bits: 4,
use_approximate_coarse_assignment: true,
+ ivf_train_max_points_per_centroid: 256,
});
reader
@@ -2768,6 +2883,7 @@ mod tests {
metric: MetricType::L2,
bits: 4,
use_approximate_coarse_assignment: true,
+ ivf_train_max_points_per_centroid: 256,
});
let queries = [0, nlist - 1]
.into_iter()
@@ -2813,6 +2929,7 @@ mod tests {
build_search_list_size: 16,
..DiskAnnBuildParams::default()
},
+ pq_train_max_points_per_centroid: 256,
},
&data,
count,
@@ -2922,6 +3039,7 @@ mod tests {
nlist: 4,
metric: MetricType::L2,
use_approximate_coarse_assignment: true,
+ ivf_train_max_points_per_centroid: 256,
});
roundtrip(VectorIndexConfig::ivf_pq(16, 4, MetricType::L2,
false).unwrap());
roundtrip(VectorIndexConfig::IvfRq {
@@ -2930,12 +3048,14 @@ mod tests {
bits: DEFAULT_RQ_BITS,
metric: MetricType::L2,
use_approximate_coarse_assignment: true,
+ ivf_train_max_points_per_centroid: 256,
});
roundtrip(VectorIndexConfig::IvfSq {
dimension: 8,
nlist: 4,
metric: MetricType::L2,
use_approximate_coarse_assignment: true,
+ ivf_train_max_points_per_centroid: 256,
});
roundtrip(
VectorIndexConfig::disk_ann(
@@ -2975,6 +3095,7 @@ mod tests {
nlist: 4,
metric: MetricType::L2,
use_approximate_coarse_assignment: true,
+ ivf_train_max_points_per_centroid: 256,
},
VectorIndexConfig::IvfPq {
dimension: 16,
@@ -2984,6 +3105,8 @@ mod tests {
use_opq: false,
use_approximate_coarse_assignment: true,
canonical_pq_encoding: false,
+ ivf_train_max_points_per_centroid: 256,
+ pq_train_max_points_per_centroid: 256,
},
VectorIndexConfig::IvfRq {
dimension: 8,
@@ -2991,12 +3114,14 @@ mod tests {
bits: DEFAULT_RQ_BITS,
metric: MetricType::L2,
use_approximate_coarse_assignment: true,
+ ivf_train_max_points_per_centroid: 256,
},
VectorIndexConfig::IvfSq {
dimension: 8,
nlist: 4,
metric: MetricType::L2,
use_approximate_coarse_assignment: true,
+ ivf_train_max_points_per_centroid: 256,
},
] {
let d = config.dimension();
@@ -3085,6 +3210,8 @@ mod tests {
use_opq: false,
use_approximate_coarse_assignment: true,
canonical_pq_encoding: false,
+ ivf_train_max_points_per_centroid: 256,
+ pq_train_max_points_per_centroid: 256,
}) {
Ok(_) => panic!("invalid PQ config should be rejected"),
Err(err) => err,
@@ -3100,6 +3227,7 @@ mod tests {
bits: DEFAULT_RQ_BITS,
metric: MetricType::L2,
use_approximate_coarse_assignment: true,
+ ivf_train_max_points_per_centroid: 256,
})
.unwrap();
let err = match VectorIndexTrainer::new(VectorIndexConfig::IvfRq {
@@ -3108,6 +3236,7 @@ mod tests {
bits: 9,
metric: MetricType::L2,
use_approximate_coarse_assignment: true,
+ ivf_train_max_points_per_centroid: 256,
}) {
Ok(_) => panic!("invalid RQ config should be rejected"),
Err(err) => err,
@@ -3824,6 +3953,7 @@ mod tests {
build_distance: DiskAnnBuildDistance::ProductQuantized,
..DiskAnnBuildParams::default()
},
+ pq_train_max_points_per_centroid: 256,
};
let mut writer = build_writer(config, data, count);
writer.add_vectors(ids, data, count).unwrap();
@@ -3929,6 +4059,7 @@ mod tests {
raw_vector_encoding: DiskAnnRawVectorEncoding::F32,
..DiskAnnBuildParams::default()
},
+ pq_train_max_points_per_centroid: 256,
},
&data,
count,
@@ -4053,6 +4184,7 @@ mod tests {
pq_m: 2,
pq_bits: 8,
build: DiskAnnBuildParams::default(),
+ pq_train_max_points_per_centroid: 256,
})
.expect("DiskANN trainer should open");
@@ -4075,6 +4207,7 @@ mod tests {
seed: 73,
..DiskAnnBuildParams::default()
},
+ pq_train_max_points_per_centroid: 256,
};
let mut whole = VectorIndexTrainer::new(config()).unwrap();
@@ -4113,6 +4246,7 @@ mod tests {
memory_budget_bytes,
..DiskAnnBuildParams::default()
},
+ pq_train_max_points_per_centroid: 256,
};
let trainer = VectorIndexTrainer::new(config).unwrap();
assert!(trainer.training_sample_limit <
DISKANN_MAX_PQ_TRAINING_VECTORS);
@@ -4135,6 +4269,7 @@ mod tests {
pq_m: 2,
pq_bits: 8,
build: DiskAnnBuildParams::default(),
+ pq_train_max_points_per_centroid: 256,
},
&data,
count,
@@ -4157,6 +4292,7 @@ mod tests {
pq_m: 2,
pq_bits: 8,
build: DiskAnnBuildParams::default(),
+ pq_train_max_points_per_centroid: 256,
},
&training_data,
training_count,
@@ -4276,6 +4412,7 @@ mod tests {
nlist: 1,
metric: MetricType::L2,
use_approximate_coarse_assignment: true,
+ ivf_train_max_points_per_centroid: 256,
},
&[value, 1.0],
2,
@@ -4396,6 +4533,105 @@ mod tests {
assert!(error.to_string().contains("only valid for IVF-PQ"));
}
+ #[test]
+ fn training_max_points_per_centroid_defaults_and_validation() {
+ let base = options(&[
+ ("index.type", "ivf_pq"),
+ ("dimension", "4"),
+ ("nlist", "2"),
+ ("metric", "l2"),
+ ]);
+ let resolved =
VectorIndexConfig::from_options(&base).unwrap().resolved();
+ assert_eq!(resolved.ivf_train_max_points_per_centroid, Some(256));
+ assert_eq!(resolved.pq_train_max_points_per_centroid, Some(256));
+
+ for key in [
+ "ivf.train.max-points-per-centroid",
+ "pq.train.max-points-per-centroid",
+ ] {
+ for value in ["0", "-1", "1.5", "abc", "",
&usize::MAX.to_string()] {
+ let mut opts = base.clone();
+ opts.insert(key.into(), value.into());
+ let error =
VectorIndexConfig::from_options(&opts).unwrap_err();
+ assert_eq!(error.kind(), io::ErrorKind::InvalidInput);
+ assert!(error.to_string().contains(key), "{error}");
+ }
+ }
+ for (index_type, key) in [
+ ("ivf_flat", "pq.train.max-points-per-centroid"),
+ ("ivf_sq", "pq.train.max-points-per-centroid"),
+ ("ivf_rq", "pq.train.max-points-per-centroid"),
+ ("diskann", "ivf.train.max-points-per-centroid"),
+ ("ivf_pq", "fields.vector.ivf.train.max-points-per-centroid"),
+ ("ivf_pq", "fields.vector.pq.train.max-points-per-centroid"),
+ ] {
+ let mut opts = base.clone();
+ opts.insert("index.type".into(), index_type.into());
+ if index_type == "diskann" {
+ opts.remove("nlist");
+ }
+ opts.insert(key.into(), "32".into());
+ let error = VectorIndexConfig::from_options(&opts).unwrap_err();
+ assert!(error.to_string().contains("unknown vector index option"));
+ assert!(error.to_string().contains(key));
+ }
+ }
+
+ #[test]
+ fn training_max_points_per_centroid_controls_ivf_and_pq() {
+ use crate::kmeans::{kmeans_train, KMeansConfig};
+ use crate::pq::ProductQuantizer;
+
+ let n = 600;
+ let d = 4;
+ let data = (0..n * d)
+ .map(|i| ((i * 37 % 997) as f32).sin())
+ .collect::<Vec<_>>();
+ let ivf_config = KMeansConfig {
+ max_points_per_centroid: 1,
+ ..KMeansConfig::default()
+ };
+ let pq_config = KMeansConfig {
+ max_points_per_centroid: 2,
+ ..KMeansConfig::default()
+ };
+ let expected_centroids = kmeans_train(&ivf_config, &data, n, d, 2);
+ let mut expected_pq = ProductQuantizer::new(d, 1);
+ expected_pq.train_with_config(&data, n, &pq_config);
+
+ for index_type in ["ivf_flat", "ivf_sq", "ivf_rq", "ivf_pq",
"diskann"] {
+ let mut opts = options(&[
+ ("index.type", index_type),
+ ("dimension", "4"),
+ ("metric", "inner_product"),
+ ]);
+ if index_type != "diskann" {
+ opts.insert("nlist".into(), "2".into());
+ opts.insert("ivf.train.max-points-per-centroid".into(),
"1".into());
+ }
+ if matches!(index_type, "ivf_pq" | "diskann") {
+ opts.insert("pq.m".into(), "1".into());
+ opts.insert("pq.train.max-points-per-centroid".into(),
"2".into());
+ }
+ let config = VectorIndexConfig::from_options(&opts).unwrap();
+ let training = VectorIndexTrainer::train(config, &data,
n).unwrap();
+ let centroids = match VectorIndexWriter::new(training) {
+ VectorIndexWriter::IvfFlat(index) =>
index.quantizer_centroids().to_vec(),
+ VectorIndexWriter::IvfSq(index) =>
index.quantizer_centroids().to_vec(),
+ VectorIndexWriter::IvfRq(index) =>
index.quantizer_centroids().to_vec(),
+ VectorIndexWriter::IvfPq(index) => {
+ assert_eq!(index.pq.centroids(), expected_pq.centroids());
+ index.quantizer_centroids().to_vec()
+ }
+ VectorIndexWriter::DiskAnn(index) => {
+ assert_eq!(index.pq.centroids(), expected_pq.centroids());
+ continue;
+ }
+ };
+ assert_eq!(centroids, expected_centroids, "{index_type}");
+ }
+ }
+
#[test]
fn config_from_options_rejects_unknown_options() {
let err = VectorIndexConfig::from_options(&options(&[
@@ -4470,6 +4706,7 @@ mod tests {
nlist: 1,
metric: MetricType::L2,
use_approximate_coarse_assignment: true,
+ ivf_train_max_points_per_centroid: 256,
},
&[0.0, 1.0],
2,
diff --git a/core/src/ivfflat.rs b/core/src/ivfflat.rs
index ab1d622..67251d2 100644
--- a/core/src/ivfflat.rs
+++ b/core/src/ivfflat.rs
@@ -73,9 +73,12 @@ impl IVFFlatIndex {
}
pub fn train(&mut self, data: &[f32], n: usize) {
+ self.train_with_config(data, n, &KMeansConfig::default())
+ }
+
+ pub fn train_with_config(&mut self, data: &[f32], n: usize, config:
&KMeansConfig) {
let train_data = self.preprocess_vectors(data, n);
- self.quantizer_centroids =
- kmeans::kmeans_train(&KMeansConfig::default(), &train_data, n,
self.d, self.nlist);
+ self.quantizer_centroids = kmeans::kmeans_train(config, &train_data,
n, self.d, self.nlist);
self.coarse_assignment.reset();
}
diff --git a/core/src/ivfpq.rs b/core/src/ivfpq.rs
index a7b1082..60644e8 100644
--- a/core/src/ivfpq.rs
+++ b/core/src/ivfpq.rs
@@ -209,6 +209,16 @@ impl IVFPQIndex {
}
pub fn train(&mut self, data: &[f32], n: usize) {
+ self.train_with_config(data, n, &KMeansConfig::default(),
&KMeansConfig::default());
+ }
+
+ pub fn train_with_config(
+ &mut self,
+ data: &[f32],
+ n: usize,
+ ivf_config: &KMeansConfig,
+ pq_config: &KMeansConfig,
+ ) {
let d = self.d;
let train_data = if self.metric == MetricType::Cosine {
@@ -225,7 +235,7 @@ impl IVFPQIndex {
// IVF centroids must be trained on projected (rotated) data since
// add() and search() assign rotated vectors via preprocess_queries().
let effective_data = if let Some(ref mut opq) = self.opq {
- opq.train(&train_data, n, &mut self.pq);
+ opq.train_with_config(&train_data, n, &mut self.pq, pq_config);
let mut projected = vec![0.0f32; n * d];
opq.apply_batch(&train_data, &mut projected, n);
projected
@@ -233,9 +243,8 @@ impl IVFPQIndex {
train_data
};
- let km_config = KMeansConfig::default();
self.quantizer_centroids =
- kmeans::kmeans_train(&km_config, &effective_data, n, d,
self.nlist);
+ kmeans::kmeans_train(ivf_config, &effective_data, n, d,
self.nlist);
self.coarse_assignment.reset();
// Retrain PQ on the same assignment distribution that add/search will
encode.
@@ -259,7 +268,7 @@ impl IVFPQIndex {
} else {
effective_data
};
- self.pq.train(&pq_train_data, n);
+ self.pq.train_with_config(&pq_train_data, n, pq_config);
}
/// Add vectors in batches (Faiss-style: batch assign → batch residual →
batch encode).
diff --git a/core/src/ivfrq.rs b/core/src/ivfrq.rs
index 86596ae..036caea 100644
--- a/core/src/ivfrq.rs
+++ b/core/src/ivfrq.rs
@@ -152,6 +152,10 @@ impl IVFRQIndex {
}
pub fn train(&mut self, data: &[f32], n: usize) {
+ self.train_with_config(data, n, &KMeansConfig::default())
+ }
+
+ pub fn train_with_config(&mut self, data: &[f32], n: usize, config:
&KMeansConfig) {
let timing = build_timing_enabled();
let total_started = Instant::now();
let phase_started = Instant::now();
@@ -159,8 +163,7 @@ impl IVFRQIndex {
log_build_timing(timing, "train.preprocess", phase_started);
let phase_started = Instant::now();
- self.quantizer_centroids =
- kmeans::kmeans_train(&KMeansConfig::default(), &processed, n,
self.d, self.nlist);
+ self.quantizer_centroids = kmeans::kmeans_train(config, &processed, n,
self.d, self.nlist);
log_build_timing(timing, "train.kmeans", phase_started);
let phase_started = Instant::now();
diff --git a/core/src/ivfsq.rs b/core/src/ivfsq.rs
index af69ed9..58e68be 100644
--- a/core/src/ivfsq.rs
+++ b/core/src/ivfsq.rs
@@ -82,9 +82,12 @@ impl IVFSQIndex {
}
pub fn train(&mut self, data: &[f32], n: usize) {
+ self.train_with_config(data, n, &KMeansConfig::default())
+ }
+
+ pub fn train_with_config(&mut self, data: &[f32], n: usize, config:
&KMeansConfig) {
let processed = self.preprocess_vectors(data, n);
- self.quantizer_centroids =
- kmeans::kmeans_train(&KMeansConfig::default(), &processed, n,
self.d, self.nlist);
+ self.quantizer_centroids = kmeans::kmeans_train(config, &processed, n,
self.d, self.nlist);
self.coarse_assignment.reset();
let list_ids = self.coarse_assignment.assign(
&processed,
diff --git a/core/src/kmeans.rs b/core/src/kmeans.rs
index 55fd814..0eac266 100644
--- a/core/src/kmeans.rs
+++ b/core/src/kmeans.rs
@@ -150,6 +150,7 @@ fn kmeans_train_hierarchical(
let initial_config = KMeansConfig {
niter: config.niter,
seed: config.seed,
+ max_points_per_centroid: config.max_points_per_centroid,
..KMeansConfig::default()
};
let initial_centroids =
@@ -200,6 +201,7 @@ fn kmeans_train_hierarchical(
let sub_config = KMeansConfig {
niter: 10,
seed: config.seed + finalized.len() as u64,
+ max_points_per_centroid: config.max_points_per_centroid,
..KMeansConfig::default()
};
let sub_centroids = kmeans_train_with_init(&sub_config, &sub_data,
sub_n, d, split_k, None);
diff --git a/core/src/opq.rs b/core/src/opq.rs
index 0ebf46b..29acd57 100644
--- a/core/src/opq.rs
+++ b/core/src/opq.rs
@@ -57,6 +57,16 @@ impl OPQMatrix {
/// Train the OPQ rotation matrix.
/// data: flat [n * d].
pub fn train(&mut self, data: &[f32], n: usize, pq: &mut ProductQuantizer)
{
+ self.train_with_config(data, n, pq, &KMeansConfig::default());
+ }
+
+ pub fn train_with_config(
+ &mut self,
+ data: &[f32],
+ n: usize,
+ pq: &mut ProductQuantizer,
+ config: &KMeansConfig,
+ ) {
let d = self.d;
let mut rng = StdRng::seed_from_u64(12345);
@@ -122,7 +132,7 @@ impl OPQMatrix {
};
let km_config = KMeansConfig {
niter: pq_niter,
- ..KMeansConfig::default()
+ ..*config
};
let hot_start = iter > 0;
pq.train_hot_start(&projected, train_n, &km_config, hot_start);
@@ -156,7 +166,7 @@ impl OPQMatrix {
// Final PQ training with the learned rotation
self.apply_batch(&train_data, &mut projected, train_n);
- pq.train_with_config(&projected, train_n, &KMeansConfig::default());
+ pq.train_with_config(&projected, train_n, config);
self.is_trained = true;
}
@@ -196,6 +206,39 @@ impl OPQMatrix {
mod tests {
use super::*;
+ #[test]
+ fn
training_max_points_per_centroid_applies_to_opq_iterations_and_final_pq() {
+ let d = 4;
+ let n = 64;
+ let mut rng = StdRng::seed_from_u64(42);
+ let mut data = Vec::new();
+ for _ in 0..n / 2 {
+ let vector = (0..d).map(|_| rng.gen::<f32>()).collect::<Vec<_>>();
+ data.extend_from_slice(&vector);
+ data.extend(vector.iter().map(|value| -value));
+ }
+ let config = KMeansConfig {
+ max_points_per_centroid: 1,
+ ..KMeansConfig::default()
+ };
+ let mut opq = OPQMatrix::new(d, 2);
+ opq.niter = 2;
+ let mut pq = ProductQuantizer::with_nbits(d, 2, 4);
+ opq.train_with_config(&data, n, &mut pq, &config);
+
+ // Paired vectors have zero mean, so OPQ centering leaves this data
unchanged.
+ let mut projected = vec![0.0; n * d];
+ opq.apply_batch(&data, &mut projected, n);
+ let mut expected = ProductQuantizer::with_nbits(d, 2, 4);
+ expected.train_with_config(&projected, n, &config);
+ assert_eq!(pq.centroids(), expected.centroids());
+
+ let mut default_opq = OPQMatrix::new(d, 2);
+ default_opq.niter = 2;
+ default_opq.train(&data, n, &mut expected);
+ assert_ne!(opq.rotation, default_opq.rotation);
+ }
+
#[test]
fn test_rotation_orthogonality() {
let d = 8;
diff --git a/docs/api.html b/docs/api.html
index 79bedf8..3d3bf35 100644
--- a/docs/api.html
+++ b/docs/api.html
@@ -49,6 +49,12 @@
<h2>Shared lifecycle</h2>
<div class="flow" aria-label="Unified API lifecycle"><div
class="flow-step"><small>01</small><strong>Create a Trainer<br>Parse and
validate options</strong></div><div
class="flow-step"><small>02</small><strong>Submit one or more<br>training
batches</strong></div><div class="flow-step"><small>03</small><strong>Finish
training and<br>create a one-shot Writer</strong></div><div
class="flow-step"><small>04</small><strong>Add row IDs / vectors<br>and write
the file</strong></div><di [...]
<ul><li>Vectors are contiguous <code>f32</code> values; length must
equal <code>vector_count × dimension</code>.</li><li>Training data may arrive
in batches. Every IVF trainer keeps a deterministic reservoir of at most
<code>max(65,536, 64 × resolved nlist)</code> vectors. DiskANN starts from a
50,000-row cap and lowers it when necessary so the retained sample, optional
cosine-normalized copy, codebook, and parallel PQ-training scratch fit
<code>diskann.memory-budget-bytes</cod [...]
+ <p>Training options are parsed by the shared Rust core in every
option-map API. Both limits default to <code>256</code> and must be positive
integers.</p>
+ <div class="table-wrap"><table><thead><tr><th>Option</th><th>Applies
to</th><th>Training limit</th></tr></thead><tbody>
+
<tr><td><code>ivf.train.max-points-per-centroid</code></td><td>IVF-FLAT,
IVF-SQ, IVF-RQ, IVF-PQ</td><td>At most <code>nlist × value</code> vectors for
coarse K-means; also applied to hierarchical clustering stages.</td></tr>
+
<tr><td><code>pq.train.max-points-per-centroid</code></td><td>IVF-PQ,
DiskANN</td><td>At most <code>2^pq_bits × value</code> vectors per PQ
subquantizer, including PQ training inside OPQ. IVF-PQ uses 8-bit
codebooks.</td></tr>
+ </tbody></table></div>
+ <p>These limits apply within the Trainer reservoir described above;
increasing them does not increase that reservoir. OPQ also retains its
65,536-row input cap, and DiskANN retains its memory budget. The limits affect
training only and are not stored in the index file. Pass the bare keys to this
library, without a <code>fields.<field-name>.</code> prefix. Paimon
integrations must also allow these keys through their option filter.</p>
<div class="callout warning"><strong>IVF coarse assignment is
approximate by default for large centroid matrices</strong>When <code>dimension
× nlist ≥ 1,000,000</code>, <code>ivf.coarse-assignment=auto</code> uses a
Vamana graph while training and adding vectors. Search still selects lists by
exact centroid distance, so graph assignment can lower recall at small
<code>nprobe</code> and does not guarantee that a vector is found by a
self-query with <code>nprobe=1</code>. Set <c [...]
<div class="callout warning"><strong>IVF-PQ encoding defaults to
auto</strong>For 8-bit PQ (<code>ksub=256</code>) where every subvector has at
least four dimensions, <code>ivf.pq-encoding=auto</code> uses the transposed
direct-L2 encoder on x86 with AVX2+FMA and on AArch64. Other CPUs use blocked
SGEMM for finite codebooks. Unsupported shapes, and non-finite codebooks on the
SGEMM fallback, use the canonical encoder. Codes can differ across these
backends; NaN and high-dynamic [...]
</section>
@@ -109,6 +115,7 @@ let config = VectorIndexConfig::IvfSq {
nlist: 1024,
metric: MetricType::L2,
use_approximate_coarse_assignment: true,
+ ivf_train_max_points_per_centroid: 256,
};
let training = VectorIndexTrainer::train(
@@ -132,6 +139,7 @@ let params = VectorSearchParams::automatic(10)
<div class="code-block"><span class="code-label">Rust · other
configurations</span><pre><code>VectorIndexConfig::IvfFlat {
dimension: 128, nlist: 1024, metric: MetricType::L2,
use_approximate_coarse_assignment: true,
+ ivf_train_max_points_per_centroid: 256,
};
VectorIndexConfig::ivf_pq(
128, 1024, MetricType::L2, false,
@@ -139,10 +147,12 @@ VectorIndexConfig::ivf_pq(
VectorIndexConfig::IvfRq {
dimension: 128, nlist: 1024, bits: 4, metric: MetricType::L2,
use_approximate_coarse_assignment: true,
+ ivf_train_max_points_per_centroid: 256,
};
VectorIndexConfig::IvfSq {
dimension: 128, nlist: 1024, metric: MetricType::L2,
use_approximate_coarse_assignment: true,
+ ivf_train_max_points_per_centroid: 256,
};</code></pre></div>
<p>The IVF-PQ constructor uses the default relative PQ-code budget
and resolves a concrete <code>m</code>. In every option-map API,
<code>pq.m</code> is optional: <code>pq.code-ratio=0.0625</code> is the
default, and an explicit <code>pq.m</code> takes precedence. Metadata and the
on-disk header expose the resolved value. Rust callers select the policy
through <code>VectorIndexConfig</code> before training; direct IVF indexes do
not expose a post-training policy switch.</p>
</section>
diff --git a/docs/diskann.html b/docs/diskann.html
index 168973c..1a9ed5f 100644
--- a/docs/diskann.html
+++ b/docs/diskann.html
@@ -162,6 +162,7 @@ ids, distances = reader.search(
<tr><td><code>pq.code-ratio</code></td><td>0.0625; finite and in
<code>(0, 0.25]</code> for 8-bit or <code>(0, 0.125]</code> for
4-bit</td><td>Target ratio between resident PQ-code bytes and raw
<code>f32</code>-vector bytes. The builder selects the nearest <code>m</code>
and distributes dimensions across balanced chunks.</td><td>Usually reduces PQ
error when increased, but grows resident memory and per-candidate lookup
work.</td></tr>
<tr><td><code>pq.m</code></td><td>Optional expert override;
<code>1..=dimension</code></td><td>Concrete PQ chunk count. Explicit values
take precedence over <code>pq.code-ratio</code>; exact chunk offsets are
persisted in the self-describing codebook.</td><td>Use only for a measured
override of automatic sizing. Non-divisible dimensions and odd 4-bit values are
valid.</td></tr>
<tr><td><code>pq.bits</code></td><td>8; must be 4 or
8</td><td>Centroids and stored bits per PQ chunk. Four-bit codes pack two
chunks per byte, use 16-entry query tables, and require a zero high padding
nibble when <code>m</code> is odd.</td><td>Eight bits generally improve
graph-navigation recall; four bits reduce codebook, resident codes, training
work, and lookup-table size. Rebuild and benchmark both.</td></tr>
+
<tr><td><code>pq.train.max-points-per-centroid</code></td><td>Positive integer;
default <code>256</code></td><td>Maximum training vectors per PQ
centroid</td><td>Caps input at <code>2^pq.bits × value</code> vectors per
subquantizer; the existing 50,000-vector cap and memory budget still
apply.</td></tr>
<tr><td><code>diskann.max-degree</code></td><td>64;
<code>1..=1023</code>, preserving page-contained raw fallback</td><td>Maximum
graph out-degree <code>R</code>.</td><td>May improve connectivity and recall;
increases graph bytes, build work, and page density cost.</td></tr>
<tr><td><code>diskann.build-search-list-size</code></td><td>Omitted:
<code>max(100, R)</code>; explicit values must be ≥
<code>R</code></td><td>Candidate width <code>Lbuild</code> during Vamana
construction.</td><td>Usually improves graph quality while increasing build CPU
and per-worker scratch.</td></tr>
<tr><td><code>diskann.alpha</code></td><td>1.2; finite and ≥
1</td><td>Second-pass robust-prune threshold.</td><td>Higher values prune
candidates less aggressively; validate degree, recall, and graph behavior
empirically.</td></tr>
@@ -171,6 +172,7 @@ ids, distances = reader.search(
<tr><td><code>diskann.raw-vector-encoding</code></td><td><code>auto</code> or
omitted; preset/budget resolves F32 or F16</td><td>Controls the persisted
rerank-vector element width for both layouts. Compact F32/F16 payloads are
exactly <code>4 × d × N</code> / <code>2 × d × N</code> bytes with no per-page
padding.</td><td>Explicit F32 preserves original rerank distances. Explicit F16
halves raw-vector I/O but must be recall-tested.</td></tr>
<tr><td><code>diskann.build-distance</code></td><td><code>auto</code> or
omitted; preset resolves PQ or full precision</td><td>Selects build-traversal
distance. Both modes use full precision for robust pruning and connectivity
repair.</td><td><code>high_recall</code> uses full precision; balanced/fast
presets use PQ guidance.</td></tr>
</tbody></table></div>
+ <p>Larger training samples increase training work and may improve
codebook quality. This option does not raise the existing training sample or
memory limits. See the <a href="api.html#lifecycle">shared training options</a>
for sampling and Paimon integration details.</p>
<h3>Starting values</h3>
<ul>
<li>Start with <code>diskann.build-preset=balanced</code>, the
automatic <code>pq.code-ratio=0.0625</code>, and the intended
<code>deployment-profile</code>. The balanced preset resolves to 8-bit PQ,
<code>R=64</code>, <code>Lbuild=100</code>, alpha 1.2, F16, and PQ-guided
construction unless an explicit option changes the representation.</li>
diff --git a/docs/ivf-flat.html b/docs/ivf-flat.html
index 5dc76b1..d225574 100644
--- a/docs/ivf-flat.html
+++ b/docs/ivf-flat.html
@@ -64,13 +64,15 @@ try (VectorIndexTraining training =
nlist: 1024,
metric: MetricType::L2,
use_approximate_coarse_assignment: true,
+ ivf_train_max_points_per_centroid: 256,
};
let params = VectorSearchParams::new(10, 16);</code></pre></div>
</section>
<section class="article-section" id="parameters">
<h2>Parameters</h2>
- <div
class="table-wrap"><table><thead><tr><th>Parameter</th><th>Requirement</th><th>Purpose</th><th>Effect
when
increased</th></tr></thead><tbody><tr><td><code>dimension</code></td><td>Inferred
by Java/Python one-shot training; otherwise required and > 0</td><td>Input
dimension</td><td>Linearly increases compute and vector
payload</td></tr><tr><td><code>nlist</code></td><td>Auto from
<code>expected-vector-count</code>, or explicit > 0</td><td>IVF partition
count</td><td>S [...]
+ <div
class="table-wrap"><table><thead><tr><th>Parameter</th><th>Requirement</th><th>Purpose</th><th>Effect
when
increased</th></tr></thead><tbody><tr><td><code>dimension</code></td><td>Inferred
by Java/Python one-shot training; otherwise required and > 0</td><td>Input
dimension</td><td>Linearly increases compute and vector
payload</td></tr><tr><td><code>nlist</code></td><td>Auto from
<code>expected-vector-count</code>, or explicit > 0</td><td>IVF partition
count</td><td>S [...]
+ <p>Larger training samples increase training work and may improve
centroid quality. These limits apply within the Trainer reservoir of
<code>max(65536, 64 × nlist)</code> vectors; increasing them does not enlarge
that reservoir. See the <a href="api.html#lifecycle">shared training
options</a> for sampling and Paimon integration details.</p>
</section>
<section class="article-section" id="storage">
diff --git a/docs/ivf-pq.html b/docs/ivf-pq.html
index d39de04..6aa3c22 100644
--- a/docs/ivf-pq.html
+++ b/docs/ivf-pq.html
@@ -73,7 +73,8 @@ try (VectorIndexTraining training =
<section class="article-section" id="parameters">
<h2>Parameters</h2>
- <div
class="table-wrap"><table><thead><tr><th>Parameter</th><th>Requirement /
default</th><th>Purpose</th><th>Tuning
meaning</th></tr></thead><tbody><tr><td><code>dimension</code></td><td>Inferred
by Java/Python one-shot training; otherwise > 0</td><td>Input dimension
<code>d</code></td><td>The inferred or explicit <code>pq.m</code> must divide
it</td></tr><tr><td><code>nlist</code></td><td>Auto from
<code>expected-vector-count</code>, or explicit > 0</td><td>IVF partitio
[...]
+ <div
class="table-wrap"><table><thead><tr><th>Parameter</th><th>Requirement /
default</th><th>Purpose</th><th>Tuning
meaning</th></tr></thead><tbody><tr><td><code>dimension</code></td><td>Inferred
by Java/Python one-shot training; otherwise > 0</td><td>Input dimension
<code>d</code></td><td>The inferred or explicit <code>pq.m</code> must divide
it</td></tr><tr><td><code>nlist</code></td><td>Auto from
<code>expected-vector-count</code>, or explicit > 0</td><td>IVF partitio
[...]
+ <p>Larger training samples increase training work and may improve
centroid quality. These limits apply within the Trainer reservoir of
<code>max(65536, 64 × nlist)</code> vectors; increasing them does not enlarge
that reservoir. OPQ also retains its 65,536-vector input cap. See the <a
href="api.html#lifecycle">shared training options</a> for sampling and Paimon
integration details.</p>
<p>All PQ encoding backends produce the same index format. The
accelerated automatic paths require 8-bit PQ (<code>ksub=256</code>) and at
least four dimensions in every subvector. Eligible shapes use transposed direct
squared L2 on x86 with AVX2+FMA and on AArch64; other CPUs use blocked SGEMM
when the codebook is finite. Unsupported shapes and non-finite codebooks on the
SGEMM fallback use canonical. SGEMM and canonical use the expanded form, so
codes can differ near ties; Na [...]
</section>
diff --git a/docs/ivf-rq.html b/docs/ivf-rq.html
index add8457..e33b440 100644
--- a/docs/ivf-rq.html
+++ b/docs/ivf-rq.html
@@ -56,13 +56,15 @@ try (VectorIndexReader reader = new
VectorIndexReader(vectorIndexInput)) {
bits: 4,
metric: MetricType::L2,
use_approximate_coarse_assignment: true,
+ ivf_train_max_points_per_centroid: 256,
};
let params = VectorSearchParams::new(10, 64);</code></pre></div>
</section>
<section class="article-section" id="parameters">
<h2>Parameters</h2>
- <div
class="table-wrap"><table><thead><tr><th>Parameter</th><th>Requirement /
default</th><th>Purpose</th><th>Guidance</th></tr></thead><tbody><tr><td><code>dimension</code></td><td>Inferred
by Java/Python one-shot training; otherwise > 0</td><td>Logical vector
dimension</td><td>Storage pads internally to a multiple of
64.</td></tr><tr><td><code>nlist</code></td><td>Auto from
<code>expected-vector-count</code>, or explicit > 0</td><td>IVF partition
count</td><td>Compare t [...]
+ <div
class="table-wrap"><table><thead><tr><th>Parameter</th><th>Requirement /
default</th><th>Purpose</th><th>Guidance</th></tr></thead><tbody><tr><td><code>dimension</code></td><td>Inferred
by Java/Python one-shot training; otherwise > 0</td><td>Logical vector
dimension</td><td>Storage pads internally to a multiple of
64.</td></tr><tr><td><code>nlist</code></td><td>Auto from
<code>expected-vector-count</code>, or explicit > 0</td><td>IVF partition
count</td><td>Compare t [...]
+ <p>Larger training samples increase training work and may improve
centroid quality. These limits apply within the Trainer reservoir of
<code>max(65536, 64 × nlist)</code> vectors; increasing them does not enlarge
that reservoir. See the <a href="api.html#lifecycle">shared training
options</a> for sampling and Paimon integration details.</p>
<div class="callout warning"><strong>No query-side bit
width</strong>The Reader always evaluates the representation stored in the
file. Changing <code>rq.bits</code> requires rebuilding the index; this keeps
one file's accuracy and cost contract stable.</div>
</section>
diff --git a/docs/ivf-sq.html b/docs/ivf-sq.html
index 34a52c9..718cefc 100644
--- a/docs/ivf-sq.html
+++ b/docs/ivf-sq.html
@@ -37,12 +37,14 @@ ivf.coarse-assignment = auto</code></pre></div>
nlist: 1024,
metric: MetricType::L2,
use_approximate_coarse_assignment: true,
+ ivf_train_max_points_per_centroid: 256,
};
let params = VectorSearchParams::new(10, 16);</code></pre></div>
</section>
<section class="article-section" id="parameters">
<h2>Parameters</h2>
- <div
class="table-wrap"><table><thead><tr><th>Parameter</th><th>Requirement</th><th>Effect</th></tr></thead><tbody><tr><td><code>dimension</code></td><td>Inferred
by Java/Python one-shot training; otherwise > 0</td><td>Each vector uses
exactly <code>d</code> SQ-code
bytes.</td></tr><tr><td><code>nlist</code></td><td>Auto from
<code>expected-vector-count</code>, or explicit > 0 and no larger than
training count</td><td>More lists shorten scans but enlarge centroid and per-
[...]
+ <div
class="table-wrap"><table><thead><tr><th>Parameter</th><th>Requirement</th><th>Effect</th></tr></thead><tbody><tr><td><code>dimension</code></td><td>Inferred
by Java/Python one-shot training; otherwise > 0</td><td>Each vector uses
exactly <code>d</code> SQ-code
bytes.</td></tr><tr><td><code>nlist</code></td><td>Auto from
<code>expected-vector-count</code>, or explicit > 0 and no larger than
training count</td><td>More lists shorten scans but enlarge centroid and per-
[...]
+ <p>Larger training samples increase training work and may improve
centroid quality. These limits apply within the Trainer reservoir of
<code>max(65536, 64 × nlist)</code> vectors; increasing them does not enlarge
that reservoir. See the <a href="api.html#lifecycle">shared training
options</a> for sampling and Paimon integration details.</p>
<p>The scalar code width is fixed at 8 bits in v1. There is
deliberately no <code>sq.bits</code>, graph-width, or search-width option.</p>
</section>
<section class="article-section" id="storage">
diff --git a/docs/releases.html b/docs/releases.html
index 65a2fb2..94b0bb0 100644
--- a/docs/releases.html
+++ b/docs/releases.html
@@ -52,6 +52,7 @@
<p>IVF-SQ now pools residual training bounds, fuses residual
encoding, transposes output in bounded parallel batches, and reuses query heaps
with conservative L2 block pruning. Unified readers and language bindings also
cache decoded partitions within the existing reader memory budget. The <a
href="ivf-sq.html#benchmarks">SIFT1M/GIST1M/GloVe benchmarks</a> record build,
native query, and recall results with reproducible commands.</p>
<p>IVSQ v1 files remain compatible. Existing indexes receive reader
optimizations without a rebuild; retrain and rebuild to use the new
quantization bounds. Set the reader memory budget to zero to disable the SQ
cache; direct Rust <code>IVFSQIndexReader::open</code> remains uncached. See <a
href="api.html#reader-options">reader options</a> and <a
href="ivf-sq.html#storage">upgrade details</a>.</p>
<div class="callout warning"><strong>Rust IVF API
migration</strong>Version 0.5.0 makes <code>quantizer_centroids</code> private
on <code>IVFFlatIndex</code>, <code>IVFPQIndex</code>, <code>IVFSQIndex</code>,
and <code>IVFRQIndex</code>. Replace direct reads with
<code>quantizer_centroids()</code> and direct assignments with
<code>set_quantizer_centroids(...)</code>. The setter validates the centroid
shape, rejects replacement after vectors are added, and refreshes cached deriv
[...]
+ <div class="callout warning"><strong>Rust training configuration
migration</strong>Direct <code>VectorIndexConfig</code> enum literals must now
include <code>ivf_train_max_points_per_centroid: 256</code> for
<code>IvfFlat</code>, <code>IvfSq</code>, <code>IvfRq</code>, and
<code>IvfPq</code>, and <code>pq_train_max_points_per_centroid: 256</code> for
<code>IvfPq</code> and <code>DiskAnn</code>. These values preserve the existing
training defaults; constructors and <code>train</ [...]
<div class="callout warning"><strong>IVF-PQ build encoding changes
by default</strong>For 8-bit PQ (<code>ksub=256</code>) where every subvector
has at least four dimensions, the default automatic encoder uses transposed
direct-L2 on x86 with AVX2+FMA and on AArch64. Other CPUs use blocked SGEMM
expanded-form encoding for finite codebooks. Unsupported shapes, and non-finite
codebooks on the SGEMM fallback, use canonical encoding. Codes can differ
across these backends at near t [...]
</section>
