JingsongLi commented on code in PR #82:
URL:
https://github.com/apache/paimon-vector-index/pull/82#discussion_r3854332900
##########
core/src/ivfpq.rs:
##########
@@ -200,17 +240,73 @@ impl IVFPQIndex {
// Retrain PQ on the exact distribution that add/search will encode.
// For OPQ: opq.train() trained PQ on centered data, but add/search
// encode uncentered vectors, so we must retrain here for all metrics.
+ let calibration = calibration_sample(&effective_data, n, d);
let pq_train_data = if self.by_residual {
compute_residuals(&effective_data, n, d,
&self.quantizer_centroids, self.nlist)
} else {
effective_data
};
self.pq.train(&pq_train_data, n);
+ self.rebuild_coarse_projection_with(&calibration, calibration.len() /
d.max(1));
+ }
+
+ /// Fit the build-only centroid projection used by `add`. Auto mode keeps
+ /// it only when the centroids are compressible enough for the projected
+ /// branch-and-bound to beat the exact scan; the result is exact either
way.
+ fn rebuild_coarse_projection(&mut self) {
+ self.rebuild_coarse_projection_with(&[], 0);
+ }
+
+ /// `calibration` (`rows × d`, centroid space) lets the projection pick its
+ /// width by measured cost; without it the variance rule applies.
+ fn rebuild_coarse_projection_with(&mut self, calibration: &[f32], rows:
usize) {
+ self.coarse_projection = None;
+ self.coarse_projection_centroids = None;
+ let force = match self.projected_assignment {
+ ProjectedAssignment::Disabled => return,
+ ProjectedAssignment::Enabled => true,
+ ProjectedAssignment::Auto => false,
+ };
+ let projection = CoarseProjection::train(
+ &self.quantizer_centroids,
+ self.nlist,
+ self.d,
+ force,
+ calibration,
+ rows,
+ );
+ match &projection {
+ Some(p) => emit_log(
+ LogLevel::Info,
+ &format!(
+ "IVF-PQ projected assignment enabled: d'={} of d={}
({:.1}% centroid variance)",
+ p.dimension(),
+ self.d,
+ p.explained_variance() * 100.0
+ ),
+ ),
+ None if force => emit_log(
+ LogLevel::Warn,
+ "IVF-PQ projected assignment requested but the centroids admit
no projection; using the exact scan",
+ ),
+ None => {}
+ }
+ self.coarse_projection = projection.map(Arc::new);
+ if self.coarse_projection.is_some() {
+ self.coarse_projection_centroids =
Some(Arc::from(self.quantizer_centroids.as_slice()));
+ }
}
/// Add vectors in batches (Faiss-style: batch assign → batch residual →
batch encode).
pub fn add(&mut self, data: &[f32], ids: &[i64], n: usize) {
const BATCH_SIZE: usize = 32768;
+ if self.coarse_projection.is_some()
Review Comment:
[P2] Avoid rescanning every centroid on each add
When a projection is active, this equality check walks all `nlist * d`
floats on every public `add`, even when the centroids have not changed. In an
arm64 release probe with `nlist=4096,d=768`, comparing the equal snapshot took
1.66 ms per call, while an exact `find_nearest` for one row took 0.423 ms;
repeated small adds therefore spend about 4x an entire exact coarse assignment
just validating the cache, erasing the projected path’s intended speedup.
Please make centroid mutation invalidate/version the projection in O(1) (for
example through an encapsulated mutation API) instead of validating the whole
matrix on each unchanged add, and cover repeated small batches.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
