jerry-024 commented on code in PR #75:
URL:
https://github.com/apache/paimon-vector-index/pull/75#discussion_r3763666889
##########
core/src/ivfpq.rs:
##########
@@ -2275,29 +2463,59 @@ fn
search_batch_reader_filter_with_reuse_mode_and_observer<R: SeekRead>(
heaps[qi].push(distance, row_id);
}
}
+ timing.scan += elapsed_since(scan_started);
batch_start = batch_end;
}
+ if timing_enabled {
+ let read_metrics = reader.end_read_metrics();
+ timing.io_read = read_metrics.elapsed;
+ // Both batch and streamed reads accumulated I/O plus decode above.
+ timing.decode = timing.decode.saturating_sub(timing.io_read);
+ timing.read_calls = read_metrics.calls;
+ timing.requested_bytes = read_metrics.requested_bytes;
+ }
+ let finalize_started = timing_enabled.then(Instant::now);
let mut result_ids = vec![-1i64; nq * k];
let mut result_dists = vec![f32::MAX; nq * k];
+ timing.min_hits_per_query = k;
for (qi, heap) in heaps.into_iter().enumerate() {
let sorted = heap.into_sorted();
+ if timing_enabled {
+ timing.queries_below_k = timing
+ .queries_below_k
+ .saturating_add(usize::from(sorted.len() < k));
+ timing.min_hits_per_query =
timing.min_hits_per_query.min(sorted.len());
+ }
let base = qi * k;
for (i, &(dist, id)) in sorted.iter().enumerate() {
result_ids[base + i] = id;
result_dists[base + i] = dist;
}
}
+ timing.finalize = elapsed_since(finalize_started);
- if !by_residual &&
std::env::var_os("PAIMON_VINDEX_LOG_IVFPQ_BATCH_REUSE").is_some() {
- use std::io::Write;
+ if timing_enabled {
+ let mut buf = Vec::with_capacity(256);
+ let _ = timing.write_to(
+ &mut buf,
+ elapsed_since(total_started),
+ nq,
+ nprobe,
+ reader.pq.nbits,
+ k,
+ unique_lists.len(),
+ filter.is_some(),
+ );
+ emit_log(LogLevel::Info, String::from_utf8_lossy(&buf).trim_end());
+ }
+ if !by_residual &&
std::env::var_os("PAIMON_VINDEX_LOG_IVFPQ_BATCH_REUSE").is_some() {
Review Comment:
Thanks. This extra forked-JVM case is not needed: the JNI bridge is a
process-wide, message-agnostic sink, and the existing timing smoke test already
exercises that transport end to end. The table-reuse behavior itself is covered
by the core batch-reuse tests. A second IP/cosine JVM search would repeat the
same bridge path without covering different delivery behavior.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
