This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 46ec02c3e0 [vector] Configure IVF-PQ batch table reuse (#8925)
46ec02c3e0 is described below
commit 46ec02c3e0d80971464a9a9e47bfa367be4f35ae
Author: shyjsarah <[email protected]>
AuthorDate: Thu Jul 30 19:18:46 2026 +0800
[vector] Configure IVF-PQ batch table reuse (#8925)
---
paimon-vector/pom.xml | 2 +-
.../vector/index/NativeVectorGlobalIndexReader.java | 20 +++++++++++---------
.../vector/index/NativeVectorGlobalIndexTest.java | 10 ++++++++++
3 files changed, 22 insertions(+), 10 deletions(-)
diff --git a/paimon-vector/pom.xml b/paimon-vector/pom.xml
index dc422b7cd8..88e875bf4b 100644
--- a/paimon-vector/pom.xml
+++ b/paimon-vector/pom.xml
@@ -32,7 +32,7 @@ under the License.
<name>Paimon : Vector Index</name>
<properties>
-
<paimon-vector-index-java.version>0.3.0</paimon-vector-index-java.version>
+
<paimon-vector-index-java.version>0.4.0-SNAPSHOT</paimon-vector-index-java.version>
</properties>
<dependencies>
diff --git
a/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexReader.java
b/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexReader.java
index 4056212e92..ae91147954 100644
---
a/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexReader.java
+++
b/paimon-vector/src/main/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexReader.java
@@ -65,6 +65,7 @@ public class NativeVectorGlobalIndexReader implements
GlobalIndexReader {
private static final String NPROBE_PARAMETER = "ivf.nprobe";
private static final String L_SEARCH_PARAMETER = "diskann.l_search";
+ private static final String IVF_PQ_BATCH_TABLE_REUSE_PARAMETER =
"ivf_pq.batch_table_reuse";
private static final int VECTOR_INDEX_MIN_SEEK_FOR_VECTOR_READS = 16 *
1024;
private static final int VECTOR_INDEX_PARALLELISM_FOR_VECTOR_READS = 32;
@@ -151,6 +152,8 @@ public class NativeVectorGlobalIndexReader implements
GlobalIndexReader {
if (scope == null) {
return emptyResults(n);
}
+ VectorSearchParams searchParams =
+ batchSearchParams(batchVectorSearch.options(),
scope.effectiveK);
// Flatten query vectors into one contiguous array for a single native
call.
float[] queries = new float[n * dim];
@@ -160,15 +163,8 @@ public class NativeVectorGlobalIndexReader implements
GlobalIndexReader {
VectorSearchBatchResult batchResult =
scope.filterBytes != null
- ? vectorReader.searchBatch(
- queries,
- n,
- searchParams(batchVectorSearch.options(),
scope.effectiveK),
- scope.filterBytes)
- : vectorReader.searchBatch(
- queries,
- n,
- searchParams(batchVectorSearch.options(),
scope.effectiveK));
+ ? vectorReader.searchBatch(queries, n, searchParams,
scope.filterBytes)
+ : vectorReader.searchBatch(queries, n, searchParams);
// result i corresponds to vectors[i], matching input order.
List<Optional<ScoredGlobalIndexResult>> results = new ArrayList<>(n);
@@ -300,6 +296,12 @@ public class NativeVectorGlobalIndexReader implements
GlobalIndexReader {
return VectorSearchParams.automatic(topK);
}
+ static VectorSearchParams batchSearchParams(Map<String, String>
parameters, int topK) {
+ VectorSearchParams searchParams = searchParams(parameters, topK);
+ String reuseMode = parameters.get(IVF_PQ_BATCH_TABLE_REUSE_PARAMETER);
+ return reuseMode == null ? searchParams :
searchParams.withIvfPqBatchTableReuse(reuseMode);
+ }
+
private static Integer intParameter(Map<String, String> parameters, String
key) {
String value = parameters.get(key);
if (value == null) {
diff --git
a/paimon-vector/src/test/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexTest.java
b/paimon-vector/src/test/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexTest.java
index 4995f3e886..d8a391ff2c 100644
---
a/paimon-vector/src/test/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexTest.java
+++
b/paimon-vector/src/test/java/org/apache/paimon/vector/index/NativeVectorGlobalIndexTest.java
@@ -27,6 +27,7 @@ import org.apache.paimon.globalindex.ResultEntry;
import org.apache.paimon.globalindex.ScoredGlobalIndexResult;
import org.apache.paimon.globalindex.io.GlobalIndexFileReader;
import org.apache.paimon.globalindex.io.GlobalIndexFileWriter;
+import org.apache.paimon.index.vector.IvfPqBatchTableReuseMode;
import org.apache.paimon.index.vector.VectorSearchParams;
import org.apache.paimon.options.Options;
import org.apache.paimon.predicate.BatchVectorSearch;
@@ -255,6 +256,15 @@ public class NativeVectorGlobalIndexTest {
assertThat(diskAnnParams.topK()).isEqualTo(10);
}
+ @Test
+ public void testIvfPqBatchTableReuseIsPropagatedToBatchSearchParams() {
+ VectorSearchParams params =
+ NativeVectorGlobalIndexReader.batchSearchParams(
+ Collections.singletonMap("ivf_pq.batch_table_reuse",
"on"), 10);
+
+
assertThat(params.ivfPqBatchTableReuse()).isEqualTo(IvfPqBatchTableReuseMode.ON);
+ }
+
@Test
public void testVectorSearchParameterRangeValidationDelegatedToNative() {
assertThat(
