This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new a107ac4964 [core] Vector search add offset back (#6984)
a107ac4964 is described below
commit a107ac49640e71d85f8eff5042b3bd07d6a46163
Author: YeJunHao <[email protected]>
AuthorDate: Thu Jan 8 21:30:55 2026 +0800
[core] Vector search add offset back (#6984)
---
.../globalindex/OffsetGlobalIndexReader.java | 7 ++-
.../org/apache/paimon/predicate/VectorSearch.java | 17 +++++++
.../apache/paimon/predicate/VectorSearchTest.java | 52 ++++++++++++++++++++++
.../globalindex/RowRangeGlobalIndexScanner.java | 3 +-
4 files changed, 76 insertions(+), 3 deletions(-)
diff --git
a/paimon-common/src/main/java/org/apache/paimon/globalindex/OffsetGlobalIndexReader.java
b/paimon-common/src/main/java/org/apache/paimon/globalindex/OffsetGlobalIndexReader.java
index ecd5b3fd66..5c3afecc7c 100644
---
a/paimon-common/src/main/java/org/apache/paimon/globalindex/OffsetGlobalIndexReader.java
+++
b/paimon-common/src/main/java/org/apache/paimon/globalindex/OffsetGlobalIndexReader.java
@@ -33,10 +33,12 @@ public class OffsetGlobalIndexReader implements
GlobalIndexReader {
private final GlobalIndexReader wrapped;
private final long offset;
+ private final long to;
- public OffsetGlobalIndexReader(GlobalIndexReader wrapped, long offset) {
+ public OffsetGlobalIndexReader(GlobalIndexReader wrapped, long offset,
long to) {
this.wrapped = wrapped;
this.offset = offset;
+ this.to = to;
}
@Override
@@ -111,7 +113,8 @@ public class OffsetGlobalIndexReader implements
GlobalIndexReader {
@Override
public Optional<GlobalIndexResult> visitVectorSearch(VectorSearch
vectorSearch) {
- return applyOffset(wrapped.visitVectorSearch(vectorSearch));
+ return applyOffset(
+
wrapped.visitVectorSearch(vectorSearch.offsetRange(this.offset, this.to)));
}
private Optional<GlobalIndexResult>
applyOffset(Optional<GlobalIndexResult> result) {
diff --git
a/paimon-common/src/main/java/org/apache/paimon/predicate/VectorSearch.java
b/paimon-common/src/main/java/org/apache/paimon/predicate/VectorSearch.java
index a6042ec7e9..8ed13e043b 100644
--- a/paimon-common/src/main/java/org/apache/paimon/predicate/VectorSearch.java
+++ b/paimon-common/src/main/java/org/apache/paimon/predicate/VectorSearch.java
@@ -20,6 +20,7 @@ package org.apache.paimon.predicate;
import org.apache.paimon.globalindex.GlobalIndexReader;
import org.apache.paimon.globalindex.GlobalIndexResult;
+import org.apache.paimon.utils.Range;
import org.apache.paimon.utils.RoaringNavigableMap64;
import javax.annotation.Nullable;
@@ -76,6 +77,22 @@ public class VectorSearch implements Serializable {
return this;
}
+ public VectorSearch offsetRange(long from, long to) {
+ if (includeRowIds != null) {
+ RoaringNavigableMap64 range = new RoaringNavigableMap64();
+ range.addRange(new Range(from, to));
+ RoaringNavigableMap64 and64 = RoaringNavigableMap64.and(range,
includeRowIds);
+ final RoaringNavigableMap64 roaringNavigableMap64Offset = new
RoaringNavigableMap64();
+ for (long rowId : and64) {
+ roaringNavigableMap64Offset.add(rowId - from);
+ }
+ VectorSearch target = new VectorSearch(vector, limit, fieldName);
+ target.withIncludeRowIds(roaringNavigableMap64Offset);
+ return target;
+ }
+ return this;
+ }
+
public Optional<GlobalIndexResult> visit(GlobalIndexReader visitor) {
return visitor.visitVectorSearch(this);
}
diff --git
a/paimon-common/src/test/java/org/apache/paimon/predicate/VectorSearchTest.java
b/paimon-common/src/test/java/org/apache/paimon/predicate/VectorSearchTest.java
new file mode 100644
index 0000000000..735874ce84
--- /dev/null
+++
b/paimon-common/src/test/java/org/apache/paimon/predicate/VectorSearchTest.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.predicate;
+
+import org.apache.paimon.utils.Range;
+import org.apache.paimon.utils.RoaringNavigableMap64;
+
+import org.junit.jupiter.api.Test;
+
+import java.util.List;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/** Test vector search. */
+public class VectorSearchTest {
+
+ @Test
+ public void testVectorSearchOffset() {
+ float[][] vectors =
+ new float[][] {
+ new float[] {1.0f, 0.0f}, new float[] {0.95f, 0.1f}, new
float[] {0.1f, 0.95f},
+ new float[] {0.98f, 0.05f}, new float[] {0.0f, 1.0f}, new
float[] {0.05f, 0.98f}
+ };
+
+ VectorSearch vectorSearch = new VectorSearch(vectors[0], 1, "test");
+
+ RoaringNavigableMap64 includeRowIds = new RoaringNavigableMap64();
+ includeRowIds.addRange(new Range(100L, 200L));
+ vectorSearch.withIncludeRowIds(includeRowIds);
+
+ vectorSearch = vectorSearch.offsetRange(60, 150);
+
+ List<Range> ranges = vectorSearch.includeRowIds().toRangeList();
+ assertThat(ranges.get(0)).isEqualTo(new Range(40L, 90L));
+ }
+}
diff --git
a/paimon-core/src/main/java/org/apache/paimon/globalindex/RowRangeGlobalIndexScanner.java
b/paimon-core/src/main/java/org/apache/paimon/globalindex/RowRangeGlobalIndexScanner.java
index fc87057027..f378669652 100644
---
a/paimon-core/src/main/java/org/apache/paimon/globalindex/RowRangeGlobalIndexScanner.java
+++
b/paimon-core/src/main/java/org/apache/paimon/globalindex/RowRangeGlobalIndexScanner.java
@@ -137,7 +137,8 @@ public class RowRangeGlobalIndexScanner implements
Closeable {
GlobalIndexReader innerReader =
new OffsetGlobalIndexReader(
globalIndexer.createReader(indexFileReadWrite, globalMetas),
- range.from);
+ range.from,
+ range.to);
unionReader.add(innerReader);
}
