This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 1a44193ec2 [global-index] Fix full-text range selection and
scalar-filter coverage for multi-field indexes (#8547)
1a44193ec2 is described below
commit 1a44193ec2bfa57a3c0dff18b67192e657e1c5d6
Author: CrownChu <[email protected]>
AuthorDate: Tue Jul 14 14:17:20 2026 +0800
[global-index] Fix full-text range selection and scalar-filter coverage for
multi-field indexes (#8547)
A global index may contain one primary field and multiple extra fields.
However, the current scan paths assume that:
1. a full-text search column must be the primary index field; and
2. a vector-primary index cannot also serve scalar predicates on its
extra fields.
These assumptions prevent multi-field indexes from serving full-text and
scalar queries correctly. They may also cause indexed row ranges to be
skipped when different index definitions cover different ranges of the
same column.
---
.../paimon/globalindex/GlobalIndexEvaluator.java | 69 ++++-
.../paimon/globalindex/UnionGlobalIndexReader.java | 9 +-
.../globalindex/GlobalIndexEvaluatorTest.java | 105 +++++++-
.../TestFullTextGlobalIndexerFactory.java | 20 ++
.../TestMultiFieldVectorGlobalIndexer.java | 261 +++++++++++++++++++
.../testvector/TestVectorGlobalIndexerFactory.java | 11 +
.../paimon/globalindex/GlobalIndexScanner.java | 49 ++--
.../table/source/DataEvolutionVectorScan.java | 22 +-
.../paimon/table/source/FullTextReadImpl.java | 58 +++--
.../paimon/table/source/FullTextScanImpl.java | 284 +++++++++++++++++++--
.../table/source/IndexFullTextSearchSplit.java | 41 ++-
.../table/source/FullTextSearchBuilderTest.java | 225 +++++++++++++++-
.../table/source/VectorSearchBuilderTest.java | 130 +++++++++-
13 files changed, 1193 insertions(+), 91 deletions(-)
diff --git
a/paimon-common/src/main/java/org/apache/paimon/globalindex/GlobalIndexEvaluator.java
b/paimon-common/src/main/java/org/apache/paimon/globalindex/GlobalIndexEvaluator.java
index 5343df6117..0f8a0e3ccd 100644
---
a/paimon-common/src/main/java/org/apache/paimon/globalindex/GlobalIndexEvaluator.java
+++
b/paimon-common/src/main/java/org/apache/paimon/globalindex/GlobalIndexEvaluator.java
@@ -20,7 +20,13 @@ package org.apache.paimon.globalindex;
import org.apache.paimon.predicate.CompoundPredicate;
import org.apache.paimon.predicate.FieldRef;
+import org.apache.paimon.predicate.IsNaN;
+import org.apache.paimon.predicate.IsNotNull;
+import org.apache.paimon.predicate.LeafBinaryFunction;
+import org.apache.paimon.predicate.LeafFunction;
+import org.apache.paimon.predicate.LeafNAryFunction;
import org.apache.paimon.predicate.LeafPredicate;
+import org.apache.paimon.predicate.LeafTernaryFunction;
import org.apache.paimon.predicate.Or;
import org.apache.paimon.predicate.Predicate;
import org.apache.paimon.types.RowType;
@@ -33,9 +39,11 @@ import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Deque;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
+import java.util.Set;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
@@ -125,7 +133,8 @@ public class GlobalIndexEvaluator implements Closeable {
private CompletableFuture<Optional<GlobalIndexResult>> visitCompoundAsync(
CompoundPredicate predicate) {
- List<Predicate> children = flattenChildren(predicate);
+ List<Predicate> children =
+ pruneRedundantIsNotNullForAnd(flattenChildren(predicate),
predicate);
List<CompletableFuture<Optional<GlobalIndexResult>>> childFutures =
new ArrayList<>(children.size());
for (Predicate child : children) {
@@ -192,6 +201,64 @@ public class GlobalIndexEvaluator implements Closeable {
return result;
}
+ private List<Predicate> pruneRedundantIsNotNullForAnd(
+ List<Predicate> children, CompoundPredicate predicate) {
+ if (predicate.function() instanceof Or) {
+ return children;
+ }
+
+ // Only a null-rejecting sibling makes "f IS NOT NULL" redundant. A
predicate such as
+ // "f IS NULL" must NOT count here: pruning IS NOT NULL from "f IS
NULL AND f IS NOT NULL"
+ // would turn the empty result into the set of null rows. So we
whitelist explicitly
+ // null-rejecting predicates instead of treating every non-IsNotNull
leaf as constraining.
+ Set<String> constrainedFields = new HashSet<>();
+ for (Predicate child : children) {
+ if (!isNullRejecting(child)) {
+ continue;
+ }
+ Optional<FieldRef> fieldRef = ((LeafPredicate)
child).fieldRefOptional();
+ fieldRef.ifPresent(ref -> constrainedFields.add(ref.name()));
+ }
+ if (constrainedFields.isEmpty()) {
+ return children;
+ }
+
+ List<Predicate> pruned = new ArrayList<>(children.size());
+ for (Predicate child : children) {
+ if (isIsNotNull(child)) {
+ Optional<FieldRef> fieldRef = ((LeafPredicate)
child).fieldRefOptional();
+ if (fieldRef.isPresent() &&
constrainedFields.contains(fieldRef.get().name())) {
+ continue;
+ }
+ }
+ pruned.add(child);
+ }
+ return pruned;
+ }
+
+ private boolean isIsNotNull(Predicate predicate) {
+ return predicate instanceof LeafPredicate
+ && ((LeafPredicate) predicate).function() instanceof IsNotNull;
+ }
+
+ /**
+ * A predicate is null-rejecting when it cannot match a row whose tested
field is null. Under
+ * SQL three-valued logic every comparison/match predicate (=, <>,
<, >, IN, NOT IN,
+ * BETWEEN, LIKE, ...) and IS NaN reject null; only IS NULL accepts it
(and IS NOT NULL is the
+ * predicate we are deciding whether to prune). We whitelist by arity base
class so future
+ * comparison functions are covered automatically without re-introducing
the IS NULL hazard.
+ */
+ private boolean isNullRejecting(Predicate predicate) {
+ if (!(predicate instanceof LeafPredicate)) {
+ return false;
+ }
+ LeafFunction function = ((LeafPredicate) predicate).function();
+ return function instanceof LeafBinaryFunction
+ || function instanceof LeafNAryFunction
+ || function instanceof LeafTernaryFunction
+ || function instanceof IsNaN;
+ }
+
@Override
public void close() {
IOUtils.closeAllQuietly(
diff --git
a/paimon-common/src/main/java/org/apache/paimon/globalindex/UnionGlobalIndexReader.java
b/paimon-common/src/main/java/org/apache/paimon/globalindex/UnionGlobalIndexReader.java
index eca23818bd..9914fa1115 100644
---
a/paimon-common/src/main/java/org/apache/paimon/globalindex/UnionGlobalIndexReader.java
+++
b/paimon-common/src/main/java/org/apache/paimon/globalindex/UnionGlobalIndexReader.java
@@ -20,6 +20,7 @@ package org.apache.paimon.globalindex;
import org.apache.paimon.predicate.FieldRef;
import org.apache.paimon.predicate.VectorSearch;
+import org.apache.paimon.utils.IOUtils;
import java.io.IOException;
import java.util.ArrayList;
@@ -183,8 +184,12 @@ public class UnionGlobalIndexReader implements
GlobalIndexReader {
@Override
public void close() throws IOException {
- for (GlobalIndexReader reader : readers) {
- reader.close();
+ try {
+ IOUtils.closeAll(readers);
+ } catch (IOException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new IOException("Failed to close union global index
readers", e);
}
}
}
diff --git
a/paimon-common/src/test/java/org/apache/paimon/globalindex/GlobalIndexEvaluatorTest.java
b/paimon-common/src/test/java/org/apache/paimon/globalindex/GlobalIndexEvaluatorTest.java
index 9a95d89cd0..b3ffad5263 100644
---
a/paimon-common/src/test/java/org/apache/paimon/globalindex/GlobalIndexEvaluatorTest.java
+++
b/paimon-common/src/test/java/org/apache/paimon/globalindex/GlobalIndexEvaluatorTest.java
@@ -32,6 +32,7 @@ import org.apache.paimon.utils.RoaringNavigableMap64;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Test;
+import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
@@ -40,9 +41,11 @@ import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
+import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
/** Tests for {@link GlobalIndexEvaluator}. */
class GlobalIndexEvaluatorTest {
@@ -566,6 +569,79 @@ class GlobalIndexEvaluatorTest {
evaluator.close();
}
+ @Test
+ void testIsNullAndIsNotNullSameFieldIsEmptyNotPruned() {
+ executor = Executors.newFixedThreadPool(2);
+ RowType rowType = rowType();
+
+ // A reader where IS NULL matches the null rows {3,7} and IS NOT NULL
matches the
+ // complementary non-null rows {1,2,4,5}. "a IS NULL AND a IS NOT
NULL" must be the
+ // intersection (empty). The old prune treated IS NULL as a
constraining sibling and
+ // dropped IS NOT NULL, wrongly yielding the null rows {3,7}.
+ GlobalIndexReader nullAwareReader =
+ new StubGlobalIndexReader(null) {
+ @Override
+ public CompletableFuture<Optional<GlobalIndexResult>>
visitIsNull(
+ FieldRef fieldRef) {
+ return
CompletableFuture.completedFuture(Optional.of(resultOf(3, 7)));
+ }
+
+ @Override
+ public CompletableFuture<Optional<GlobalIndexResult>>
visitIsNotNull(
+ FieldRef fieldRef) {
+ return
CompletableFuture.completedFuture(Optional.of(resultOf(1, 2, 4, 5)));
+ }
+ };
+
+ GlobalIndexEvaluator evaluator =
+ new GlobalIndexEvaluator(
+ rowType, fieldId ->
Collections.singletonList(nullAwareReader));
+
+ PredicateBuilder builder = new PredicateBuilder(rowType);
+ Predicate predicate = PredicateBuilder.and(builder.isNull(0),
builder.isNotNull(0));
+
+ Optional<GlobalIndexResult> result = evaluator.evaluate(predicate);
+
+ assertThat(result).isPresent();
+ assertThat(result.get().results().isEmpty()).isTrue();
+ evaluator.close();
+ }
+
+ @Test
+ void testRedundantIsNotNullStillPrunedWithNullRejectingSibling() {
+ executor = Executors.newFixedThreadPool(2);
+ RowType rowType = rowType();
+ AtomicBoolean isNotNullVisited = new AtomicBoolean();
+
+ // "a = 42 AND a IS NOT NULL": a = 42 is null-rejecting, so IS NOT
NULL is redundant and
+ // pruned. An unsupported Optional.empty() child is ignored by AND
evaluation, so the
+ // result alone cannot prove pruning; record whether the visitor is
invoked.
+ GlobalIndexReader equalOnlyReader =
+ new StubGlobalIndexReader(resultOf(1, 2, 3)) {
+ @Override
+ public CompletableFuture<Optional<GlobalIndexResult>>
visitIsNotNull(
+ FieldRef fieldRef) {
+ isNotNullVisited.set(true);
+ // Simulate a reader that cannot serve IS NOT NULL.
+ return
CompletableFuture.completedFuture(Optional.empty());
+ }
+ };
+
+ GlobalIndexEvaluator evaluator =
+ new GlobalIndexEvaluator(
+ rowType, fieldId ->
Collections.singletonList(equalOnlyReader));
+
+ PredicateBuilder builder = new PredicateBuilder(rowType);
+ Predicate predicate = PredicateBuilder.and(builder.equal(0, 42),
builder.isNotNull(0));
+
+ Optional<GlobalIndexResult> result = evaluator.evaluate(predicate);
+
+ assertThat(result).isPresent();
+ assertBitmapContainsExactly(result.get().results(), 1L, 2L, 3L);
+ assertThat(isNotNullVisited).isFalse();
+ evaluator.close();
+ }
+
@Test
void testNullPredicate() {
RowType rowType = rowType();
@@ -578,6 +654,33 @@ class GlobalIndexEvaluatorTest {
evaluator.close();
}
+ @Test
+ void testUnionReaderClosesRemainingReadersAfterFailure() {
+ AtomicBoolean secondClosed = new AtomicBoolean();
+ GlobalIndexReader failingReader =
+ new StubGlobalIndexReader(null) {
+ @Override
+ public void close() throws IOException {
+ throw new IOException("expected close failure");
+ }
+ };
+ GlobalIndexReader secondReader =
+ new StubGlobalIndexReader(null) {
+ @Override
+ public void close() {
+ secondClosed.set(true);
+ }
+ };
+
+ UnionGlobalIndexReader union =
+ new UnionGlobalIndexReader(Arrays.asList(failingReader,
secondReader));
+
+ assertThatThrownBy(union::close)
+ .isInstanceOf(IOException.class)
+ .hasMessageContaining("expected close failure");
+ assertThat(secondClosed).isTrue();
+ }
+
private static void assertBitmapContainsExactly(
RoaringNavigableMap64 bitmap, long... expected) {
assertThat(bitmap.getLongCardinality()).isEqualTo(expected.length);
@@ -677,6 +780,6 @@ class GlobalIndexEvaluatorTest {
}
@Override
- public void close() {}
+ public void close() throws IOException {}
}
}
diff --git
a/paimon-common/src/test/java/org/apache/paimon/globalindex/testfulltext/TestFullTextGlobalIndexerFactory.java
b/paimon-common/src/test/java/org/apache/paimon/globalindex/testfulltext/TestFullTextGlobalIndexerFactory.java
index e381fef383..e8894affbe 100644
---
a/paimon-common/src/test/java/org/apache/paimon/globalindex/testfulltext/TestFullTextGlobalIndexerFactory.java
+++
b/paimon-common/src/test/java/org/apache/paimon/globalindex/testfulltext/TestFullTextGlobalIndexerFactory.java
@@ -22,6 +22,9 @@ import org.apache.paimon.globalindex.GlobalIndexer;
import org.apache.paimon.globalindex.GlobalIndexerFactory;
import org.apache.paimon.options.Options;
import org.apache.paimon.types.DataField;
+import org.apache.paimon.types.VarCharType;
+
+import java.util.List;
/**
* A test-only {@link GlobalIndexerFactory} for full-text search. Uses
brute-force in-memory
@@ -45,4 +48,21 @@ public class TestFullTextGlobalIndexerFactory implements
GlobalIndexerFactory {
public GlobalIndexer create(DataField field, Options options) {
return new TestFullTextGlobalIndexer(field.type(), options);
}
+
+ @Override
+ public GlobalIndexer create(
+ DataField indexField, List<DataField> extraFields, Options
options) {
+ // Multi-column support: this brute-force backend indexes a single
text column, which may be
+ // the primary field or an extra field. Pick the VARCHAR/STRING column
among them.
+ DataField textField = indexField;
+ if (!(textField.type() instanceof VarCharType) && extraFields != null)
{
+ for (DataField extra : extraFields) {
+ if (extra.type() instanceof VarCharType) {
+ textField = extra;
+ break;
+ }
+ }
+ }
+ return new TestFullTextGlobalIndexer(textField.type(), options);
+ }
}
diff --git
a/paimon-common/src/test/java/org/apache/paimon/globalindex/testvector/TestMultiFieldVectorGlobalIndexer.java
b/paimon-common/src/test/java/org/apache/paimon/globalindex/testvector/TestMultiFieldVectorGlobalIndexer.java
new file mode 100644
index 0000000000..e984dd03b2
--- /dev/null
+++
b/paimon-common/src/test/java/org/apache/paimon/globalindex/testvector/TestMultiFieldVectorGlobalIndexer.java
@@ -0,0 +1,261 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.globalindex.testvector;
+
+import org.apache.paimon.data.InternalRow;
+import org.apache.paimon.globalindex.GlobalIndexIOMeta;
+import org.apache.paimon.globalindex.GlobalIndexMultiColumnWriter;
+import org.apache.paimon.globalindex.GlobalIndexReader;
+import org.apache.paimon.globalindex.GlobalIndexResult;
+import org.apache.paimon.globalindex.GlobalIndexSingleColumnWriter;
+import org.apache.paimon.globalindex.GlobalIndexWriter;
+import org.apache.paimon.globalindex.ResultEntry;
+import org.apache.paimon.globalindex.ScoredGlobalIndexResult;
+import org.apache.paimon.globalindex.VectorGlobalIndexer;
+import org.apache.paimon.globalindex.btree.BTreeGlobalIndexer;
+import org.apache.paimon.globalindex.io.GlobalIndexFileReader;
+import org.apache.paimon.globalindex.io.GlobalIndexFileWriter;
+import org.apache.paimon.options.Options;
+import org.apache.paimon.predicate.BatchVectorSearch;
+import org.apache.paimon.predicate.FieldRef;
+import org.apache.paimon.predicate.VectorSearch;
+import org.apache.paimon.types.DataField;
+
+import javax.annotation.Nullable;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutorService;
+
+import static org.apache.paimon.utils.Preconditions.checkArgument;
+
+/** Test-only vector indexer supporting one scalar extra field through a
companion B-tree file. */
+class TestMultiFieldVectorGlobalIndexer implements VectorGlobalIndexer {
+
+ private final TestVectorGlobalIndexer vectorIndexer;
+ private final BTreeGlobalIndexer scalarIndexer;
+ private final DataField vectorField;
+ private final DataField scalarField;
+
+ TestMultiFieldVectorGlobalIndexer(
+ DataField vectorField, List<DataField> extraFields, Options
options) {
+ checkArgument(
+ extraFields.size() == 1,
+ "Test multi-field vector index supports exactly one extra
field, but got: %s",
+ extraFields);
+ this.vectorField = vectorField;
+ this.scalarField = extraFields.get(0);
+ this.vectorIndexer = new TestVectorGlobalIndexer(vectorField.type(),
options);
+ this.scalarIndexer = new BTreeGlobalIndexer(scalarField, options);
+ }
+
+ @Override
+ public GlobalIndexWriter createWriter(GlobalIndexFileWriter fileWriter)
throws IOException {
+ return new MultiColumnWriter(
+ (GlobalIndexSingleColumnWriter)
vectorIndexer.createWriter(fileWriter),
+ (GlobalIndexSingleColumnWriter)
scalarIndexer.createWriter(fileWriter),
+ vectorField,
+ scalarField);
+ }
+
+ @Override
+ public GlobalIndexReader createReader(
+ GlobalIndexFileReader fileReader,
+ List<GlobalIndexIOMeta> files,
+ ExecutorService executor) {
+ List<GlobalIndexIOMeta> vectorFiles = new ArrayList<>();
+ List<GlobalIndexIOMeta> scalarFiles = new ArrayList<>();
+ for (GlobalIndexIOMeta file : files) {
+ // The test vector format has no file metadata; the B-tree
companion always has it.
+ (file.metadata() == null ? vectorFiles : scalarFiles).add(file);
+ }
+ checkArgument(vectorFiles.size() == 1, "Expected one test vector file,
got: %s", files);
+ checkArgument(
+ scalarFiles.size() == 1, "Expected one scalar companion file,
got: %s", files);
+ return new MultiColumnReader(
+ vectorIndexer.createReader(fileReader, vectorFiles, executor),
+ scalarIndexer.createReader(fileReader, scalarFiles, executor));
+ }
+
+ @Override
+ public String metric() {
+ return vectorIndexer.metric();
+ }
+
+ private static class MultiColumnWriter implements
GlobalIndexMultiColumnWriter {
+
+ private final GlobalIndexSingleColumnWriter vectorWriter;
+ private final GlobalIndexSingleColumnWriter scalarWriter;
+ private final InternalRow.FieldGetter vectorGetter;
+ private final InternalRow.FieldGetter scalarGetter;
+
+ private MultiColumnWriter(
+ GlobalIndexSingleColumnWriter vectorWriter,
+ GlobalIndexSingleColumnWriter scalarWriter,
+ DataField vectorField,
+ DataField scalarField) {
+ this.vectorWriter = vectorWriter;
+ this.scalarWriter = scalarWriter;
+ this.vectorGetter =
InternalRow.createFieldGetter(vectorField.type(), 0);
+ this.scalarGetter =
InternalRow.createFieldGetter(scalarField.type(), 1);
+ }
+
+ @Override
+ public void write(long rowId, @Nullable InternalRow row) {
+ Object vector = row == null ? null :
vectorGetter.getFieldOrNull(row);
+ Object scalar = row == null ? null :
scalarGetter.getFieldOrNull(row);
+ vectorWriter.write(vector, rowId);
+ scalarWriter.write(scalar, rowId);
+ }
+
+ @Override
+ public List<ResultEntry> finish() {
+ List<ResultEntry> results = new ArrayList<>();
+ results.addAll(vectorWriter.finish());
+ results.addAll(scalarWriter.finish());
+ return results;
+ }
+ }
+
+ /**
+ * Routes vector operations to the vector file and scalar predicates to
its B-tree companion.
+ */
+ private static class MultiColumnReader implements GlobalIndexReader {
+
+ private final GlobalIndexReader vectorReader;
+ private final GlobalIndexReader scalarReader;
+
+ private MultiColumnReader(GlobalIndexReader vectorReader,
GlobalIndexReader scalarReader) {
+ this.vectorReader = vectorReader;
+ this.scalarReader = scalarReader;
+ }
+
+ @Override
+ public CompletableFuture<Optional<ScoredGlobalIndexResult>>
visitVectorSearch(
+ VectorSearch vectorSearch) {
+ return vectorReader.visitVectorSearch(vectorSearch);
+ }
+
+ @Override
+ public CompletableFuture<List<Optional<ScoredGlobalIndexResult>>>
visitBatchVectorSearch(
+ BatchVectorSearch batchVectorSearch) {
+ return vectorReader.visitBatchVectorSearch(batchVectorSearch);
+ }
+
+ @Override
+ public CompletableFuture<Optional<GlobalIndexResult>>
visitIsNotNull(FieldRef fieldRef) {
+ return scalarReader.visitIsNotNull(fieldRef);
+ }
+
+ @Override
+ public CompletableFuture<Optional<GlobalIndexResult>>
visitIsNull(FieldRef fieldRef) {
+ return scalarReader.visitIsNull(fieldRef);
+ }
+
+ @Override
+ public CompletableFuture<Optional<GlobalIndexResult>> visitStartsWith(
+ FieldRef fieldRef, Object literal) {
+ return scalarReader.visitStartsWith(fieldRef, literal);
+ }
+
+ @Override
+ public CompletableFuture<Optional<GlobalIndexResult>> visitEndsWith(
+ FieldRef fieldRef, Object literal) {
+ return scalarReader.visitEndsWith(fieldRef, literal);
+ }
+
+ @Override
+ public CompletableFuture<Optional<GlobalIndexResult>> visitContains(
+ FieldRef fieldRef, Object literal) {
+ return scalarReader.visitContains(fieldRef, literal);
+ }
+
+ @Override
+ public CompletableFuture<Optional<GlobalIndexResult>> visitLike(
+ FieldRef fieldRef, Object literal) {
+ return scalarReader.visitLike(fieldRef, literal);
+ }
+
+ @Override
+ public CompletableFuture<Optional<GlobalIndexResult>> visitLessThan(
+ FieldRef fieldRef, Object literal) {
+ return scalarReader.visitLessThan(fieldRef, literal);
+ }
+
+ @Override
+ public CompletableFuture<Optional<GlobalIndexResult>>
visitGreaterOrEqual(
+ FieldRef fieldRef, Object literal) {
+ return scalarReader.visitGreaterOrEqual(fieldRef, literal);
+ }
+
+ @Override
+ public CompletableFuture<Optional<GlobalIndexResult>> visitNotEqual(
+ FieldRef fieldRef, Object literal) {
+ return scalarReader.visitNotEqual(fieldRef, literal);
+ }
+
+ @Override
+ public CompletableFuture<Optional<GlobalIndexResult>> visitLessOrEqual(
+ FieldRef fieldRef, Object literal) {
+ return scalarReader.visitLessOrEqual(fieldRef, literal);
+ }
+
+ @Override
+ public CompletableFuture<Optional<GlobalIndexResult>> visitEqual(
+ FieldRef fieldRef, Object literal) {
+ return scalarReader.visitEqual(fieldRef, literal);
+ }
+
+ @Override
+ public CompletableFuture<Optional<GlobalIndexResult>> visitGreaterThan(
+ FieldRef fieldRef, Object literal) {
+ return scalarReader.visitGreaterThan(fieldRef, literal);
+ }
+
+ @Override
+ public CompletableFuture<Optional<GlobalIndexResult>> visitIn(
+ FieldRef fieldRef, List<Object> literals) {
+ return scalarReader.visitIn(fieldRef, literals);
+ }
+
+ @Override
+ public CompletableFuture<Optional<GlobalIndexResult>> visitNotIn(
+ FieldRef fieldRef, List<Object> literals) {
+ return scalarReader.visitNotIn(fieldRef, literals);
+ }
+
+ @Override
+ public CompletableFuture<Optional<GlobalIndexResult>> visitBetween(
+ FieldRef fieldRef, Object from, Object to) {
+ return scalarReader.visitBetween(fieldRef, from, to);
+ }
+
+ @Override
+ public void close() throws IOException {
+ try {
+ vectorReader.close();
+ } finally {
+ scalarReader.close();
+ }
+ }
+ }
+}
diff --git
a/paimon-common/src/test/java/org/apache/paimon/globalindex/testvector/TestVectorGlobalIndexerFactory.java
b/paimon-common/src/test/java/org/apache/paimon/globalindex/testvector/TestVectorGlobalIndexerFactory.java
index 92c0d8126c..04543f7919 100644
---
a/paimon-common/src/test/java/org/apache/paimon/globalindex/testvector/TestVectorGlobalIndexerFactory.java
+++
b/paimon-common/src/test/java/org/apache/paimon/globalindex/testvector/TestVectorGlobalIndexerFactory.java
@@ -23,6 +23,8 @@ import org.apache.paimon.globalindex.GlobalIndexerFactory;
import org.apache.paimon.options.Options;
import org.apache.paimon.types.DataField;
+import java.util.List;
+
/**
* A test-only {@link GlobalIndexerFactory} for vector similarity search. Uses
brute-force linear
* scan, no native dependencies required.
@@ -40,4 +42,13 @@ public class TestVectorGlobalIndexerFactory implements
GlobalIndexerFactory {
public GlobalIndexer create(DataField field, Options options) {
return new TestVectorGlobalIndexer(field.type(), options);
}
+
+ @Override
+ public GlobalIndexer create(
+ DataField indexField, List<DataField> extraFields, Options
options) {
+ if (extraFields == null || extraFields.isEmpty()) {
+ return create(indexField, options);
+ }
+ return new TestMultiFieldVectorGlobalIndexer(indexField, extraFields,
options);
+ }
}
diff --git
a/paimon-core/src/main/java/org/apache/paimon/globalindex/GlobalIndexScanner.java
b/paimon-core/src/main/java/org/apache/paimon/globalindex/GlobalIndexScanner.java
index 1385970b24..e9ec45b041 100644
---
a/paimon-core/src/main/java/org/apache/paimon/globalindex/GlobalIndexScanner.java
+++
b/paimon-core/src/main/java/org/apache/paimon/globalindex/GlobalIndexScanner.java
@@ -119,23 +119,33 @@ public class GlobalIndexScanner implements Closeable {
IntFunction<Collection<GlobalIndexReader>> readersFunction =
fId -> {
+ List<IndexMetaFileGroup> groups = new ArrayList<>();
IndexMetaFileGroup group = indexMetas.get(fId);
if (group != null) {
- return createReaders(indexFileReader, group, rowType,
Long.MIN_VALUE);
+ groups.add(group);
}
List<IndexMetaFileGroup> extraGroups =
extraIndexMetas.get(fId);
- if (extraGroups == null || extraGroups.isEmpty()) {
- return Collections.emptyList();
+ if (extraGroups != null) {
+ for (IndexMetaFileGroup extraGroup : extraGroups) {
+ if (!groups.contains(extraGroup)) {
+ groups.add(extraGroup);
+ }
+ }
}
- long maxEnd = Long.MIN_VALUE;
- for (IndexMetaFileGroup g : extraGroups) {
- maxEnd = Math.max(maxEnd, g.coverageEnd());
+ if (groups.isEmpty()) {
+ return Collections.emptyList();
}
+
+ // A field can be covered by its dedicated primary index
and by one or more
+ // multi-column indexes that carry it as an extra field.
These are alternative
+ // sources of matches, possibly over different row ranges,
so union them. The
+ // previous primary-only choice made coverage planning
believe the extra-field
+ // tail was indexed while the evaluator silently ignored
it.
List<GlobalIndexReader> allReaders = new ArrayList<>();
- for (IndexMetaFileGroup g : extraGroups) {
- allReaders.addAll(createReaders(indexFileReader, g,
rowType, maxEnd));
+ for (IndexMetaFileGroup indexGroup : groups) {
+ allReaders.addAll(createReaders(indexFileReader,
indexGroup, rowType));
}
- return allReaders;
+ return Collections.singletonList(new
UnionGlobalIndexReader(allReaders));
};
this.globalIndexEvaluator = new GlobalIndexEvaluator(rowType,
readersFunction);
}
@@ -146,7 +156,6 @@ public class GlobalIndexScanner implements Closeable {
private final int indexFieldId;
private final List<Integer> fieldIds;
private final Map<String, Map<Range, List<IndexFileMeta>>> metas = new
HashMap<>();
- private long coverageEnd = Long.MIN_VALUE;
IndexMetaFileGroup(int indexFieldId, List<Integer> fieldIds) {
this.indexFieldId = indexFieldId;
@@ -154,17 +163,11 @@ public class GlobalIndexScanner implements Closeable {
}
void addFile(String indexType, Range range, IndexFileMeta indexFile) {
- coverageEnd = Math.max(coverageEnd, range.to);
metas.computeIfAbsent(indexType, k -> new HashMap<>())
.computeIfAbsent(range, k -> new ArrayList<>())
.add(indexFile);
}
- /** The largest indexed rowId across all files of this index (ranges
start at 0). */
- long coverageEnd() {
- return coverageEnd;
- }
-
/** The primary index column. */
DataField indexField(RowType rowType) {
return rowType.getField(indexFieldId);
@@ -286,10 +289,7 @@ public class GlobalIndexScanner implements Closeable {
}
private Collection<GlobalIndexReader> createReaders(
- GlobalIndexFileReader indexFileReadWrite,
- IndexMetaFileGroup group,
- RowType rowType,
- long padToEnd) {
+ GlobalIndexFileReader indexFileReadWrite, IndexMetaFileGroup
group, RowType rowType) {
DataField indexField = group.indexField(rowType);
List<DataField> extraFields = group.extraFields(rowType);
@@ -301,11 +301,9 @@ public class GlobalIndexScanner implements Closeable {
GlobalIndexer globalIndexer =
globalIndexerFactory.create(indexField, extraFields,
options);
- long typeEnd = Long.MIN_VALUE;
List<CompletableFuture<GlobalIndexReader>> futures = new
ArrayList<>(metas.size());
for (Map.Entry<Range, List<IndexFileMeta>> rangeMetas :
metas.entrySet()) {
Range range = rangeMetas.getKey();
- typeEnd = Math.max(typeEnd, range.to);
List<IndexFileMeta> indexFileMetas = rangeMetas.getValue();
List<GlobalIndexIOMeta> globalMetas =
indexFileMetas.stream()
@@ -327,13 +325,6 @@ public class GlobalIndexScanner implements Closeable {
for (CompletableFuture<GlobalIndexReader> future : futures) {
unionReader.add(future.join());
}
- // Pad this index's missing tail with an all-hit reader so AND-ing
it with a
- // longer-range index does not drop rows it has not indexed
(ranges start at 0).
- if (padToEnd > typeEnd) {
- unionReader.add(
- new ConstantGlobalIndexReader(
- GlobalIndexResult.fromRange(new Range(typeEnd
+ 1, padToEnd))));
- }
readers.add(new UnionGlobalIndexReader(unionReader));
}
diff --git
a/paimon-core/src/main/java/org/apache/paimon/table/source/DataEvolutionVectorScan.java
b/paimon-core/src/main/java/org/apache/paimon/table/source/DataEvolutionVectorScan.java
index 79a9fe81d0..6f123632c1 100644
---
a/paimon-core/src/main/java/org/apache/paimon/table/source/DataEvolutionVectorScan.java
+++
b/paimon-core/src/main/java/org/apache/paimon/table/source/DataEvolutionVectorScan.java
@@ -107,7 +107,12 @@ public class DataEvolutionVectorScan implements VectorScan
{
vectorIndexType = configuredVectorIndexType();
}
- // Group vector index files by (rowRangeStart, rowRangeEnd)
+ // Group vector index files by (rowRangeStart, rowRangeEnd). A file is
treated as the vector
+ // index for this search only when the vector column is its PRIMARY
field (indexFieldId);
+ // the canonical ES hybrid layout is vector-as-primary with
text/scalar columns carried as
+ // extra fields. If the vector column were instead an extra field, no
IndexVectorSearchSplit
+ // is produced here and the search falls back to a brute-force
RawVectorSearchSplit (correct
+ // results, but the ANN index is bypassed).
Map<Range, List<IndexFileMeta>> vectorByRange = new HashMap<>();
List<IndexFileMeta> vectorIndexFiles = new ArrayList<>();
for (IndexFileMeta indexFile : allIndexFiles) {
@@ -130,7 +135,7 @@ public class DataEvolutionVectorScan implements VectorScan {
f -> {
GlobalIndexMeta globalIndex =
checkNotNull(f.globalIndexMeta());
- if (isPrimaryColumn(globalIndex,
vectorColumn.id())) {
+ if
(!canServeScalarFilter(globalIndex)) {
return false;
}
return
range.hasIntersection(globalIndex.rowRange());
@@ -180,7 +185,7 @@ public class DataEvolutionVectorScan implements VectorScan {
.filter(
f -> {
GlobalIndexMeta globalIndex =
checkNotNull(f.globalIndexMeta());
- return !isPrimaryColumn(globalIndex,
vectorColumn.id());
+ return canServeScalarFilter(globalIndex);
})
.collect(Collectors.toList());
}
@@ -191,7 +196,7 @@ public class DataEvolutionVectorScan implements VectorScan {
.filter(
f -> {
GlobalIndexMeta globalIndex =
checkNotNull(f.globalIndexMeta());
- if (isPrimaryColumn(globalIndex,
vectorColumn.id())) {
+ if (!canServeScalarFilter(globalIndex)) {
return false;
}
return hasIntersection(rowRanges,
globalIndex.rowRange());
@@ -199,6 +204,15 @@ public class DataEvolutionVectorScan implements VectorScan
{
.collect(Collectors.toList());
}
+ private boolean canServeScalarFilter(GlobalIndexMeta meta) {
+ return !isPrimaryColumn(meta, vectorColumn.id()) ||
hasExtraFields(meta);
+ }
+
+ private static boolean hasExtraFields(GlobalIndexMeta meta) {
+ int[] extraFieldIds = meta.extraFieldIds();
+ return extraFieldIds != null && extraFieldIds.length > 0;
+ }
+
private static boolean hasIntersection(List<Range> ranges, Range range) {
for (Range r : ranges) {
if (r.hasIntersection(range)) {
diff --git
a/paimon-core/src/main/java/org/apache/paimon/table/source/FullTextReadImpl.java
b/paimon-core/src/main/java/org/apache/paimon/table/source/FullTextReadImpl.java
index 835630db28..0a2134c7a5 100644
---
a/paimon-core/src/main/java/org/apache/paimon/table/source/FullTextReadImpl.java
+++
b/paimon-core/src/main/java/org/apache/paimon/table/source/FullTextReadImpl.java
@@ -159,26 +159,14 @@ public class FullTextReadImpl implements FullTextRead {
return ScoredGlobalIndexResult.createEmpty();
}
- IndexFileMeta firstFile =
columnSplits.get(0).fullTextIndexFiles().get(0);
- String indexType = firstFile.indexType();
- GlobalIndexMeta firstMeta = checkNotNull(firstFile.globalIndexMeta());
- GlobalIndexer globalIndexer;
- if (firstMeta.extraFieldIds() != null) {
- globalIndexer =
- GlobalIndexerFactoryUtils.load(indexType)
- .create(
- firstMeta.getIndexField(table.rowType()),
- firstMeta.getExtraFields(table.rowType()),
- table.coreOptions().toConfiguration());
- } else {
- globalIndexer =
- GlobalIndexerFactoryUtils.load(indexType)
- .create(textColumn,
table.coreOptions().toConfiguration());
- }
-
+ // A column can carry splits from more than one index identity
(per-range selection in the
+ // scan when different indexes cover different ranges of the same
column), so build the
+ // reader from each split's own file meta rather than reusing the
first split's identity.
List<CompletableFuture<Optional<ScoredGlobalIndexResult>>> futures =
new ArrayList<>(columnSplits.size());
for (IndexFullTextSearchSplit split : columnSplits) {
+ GlobalIndexer globalIndexer =
+ createIndexer(split.fullTextIndexFiles().get(0),
textColumn);
futures.add(
eval(
globalIndexer,
@@ -188,8 +176,7 @@ public class FullTextReadImpl implements FullTextRead {
split.fullTextIndexFiles(),
indexFileReader,
executor,
- GlobalIndexLiveRowFilter.forRange(
- liveRows, split.rowRangeStart(),
split.rowRangeEnd())));
+ includeRowIds(split, liveRows)));
}
CompletableFuture.allOf(futures.toArray(new
CompletableFuture[0])).join();
@@ -205,6 +192,39 @@ public class FullTextReadImpl implements FullTextRead {
return result;
}
+ @Nullable
+ private static RoaringNavigableMap64 includeRowIds(
+ IndexFullTextSearchSplit split, @Nullable RoaringNavigableMap64
liveRows) {
+ RoaringNavigableMap64 include = new RoaringNavigableMap64();
+ for (Range range : split.searchRowRanges()) {
+ include.addRange(range);
+ }
+ if (liveRows != null) {
+ include.and(liveRows);
+ }
+ long physicalRowCount = split.rowRangeEnd() - split.rowRangeStart() +
1;
+ return include.getLongCardinality() == physicalRowCount ? null :
include;
+ }
+
+ /**
+ * Builds the {@link GlobalIndexer} for a single split from its own index
file meta, so a column
+ * served by several index identities (over different row ranges) reads
each split with the
+ * matching field configuration instead of the first split's.
+ */
+ private GlobalIndexer createIndexer(IndexFileMeta file, DataField
textColumn) {
+ String indexType = file.indexType();
+ GlobalIndexMeta meta = checkNotNull(file.globalIndexMeta());
+ if (meta.extraFieldIds() != null) {
+ return GlobalIndexerFactoryUtils.load(indexType)
+ .create(
+ meta.getIndexField(table.rowType()),
+ meta.getExtraFields(table.rowType()),
+ table.coreOptions().toConfiguration());
+ }
+ return GlobalIndexerFactoryUtils.load(indexType)
+ .create(textColumn, table.coreOptions().toConfiguration());
+ }
+
private CompletableFuture<Optional<ScoredGlobalIndexResult>> eval(
GlobalIndexer globalIndexer,
IndexPathFactory indexPathFactory,
diff --git
a/paimon-core/src/main/java/org/apache/paimon/table/source/FullTextScanImpl.java
b/paimon-core/src/main/java/org/apache/paimon/table/source/FullTextScanImpl.java
index 5fdebadf70..ecf6719f74 100644
---
a/paimon-core/src/main/java/org/apache/paimon/table/source/FullTextScanImpl.java
+++
b/paimon-core/src/main/java/org/apache/paimon/table/source/FullTextScanImpl.java
@@ -19,6 +19,7 @@
package org.apache.paimon.table.source;
import org.apache.paimon.Snapshot;
+import org.apache.paimon.annotation.VisibleForTesting;
import org.apache.paimon.globalindex.GlobalIndexCoverage;
import org.apache.paimon.globalindex.GlobalIndexerFactory;
import org.apache.paimon.globalindex.GlobalIndexerFactoryUtils;
@@ -34,13 +35,16 @@ import org.apache.paimon.utils.Filter;
import org.apache.paimon.utils.Range;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
+import java.util.TreeSet;
import java.util.stream.Collectors;
import static org.apache.paimon.utils.Preconditions.checkNotNull;
@@ -93,7 +97,7 @@ public class FullTextScanImpl implements FullTextScan {
if (globalIndex == null) {
return false;
}
- return textColumnIds.contains(globalIndex.indexFieldId())
+ return !matchedTextColumnIds(globalIndex,
textColumnIds).isEmpty()
&&
supportsFullTextSearch(entry.indexFile().indexType());
};
@@ -102,29 +106,16 @@ public class FullTextScanImpl implements FullTextScan {
.map(IndexManifestEntry::indexFile)
.collect(Collectors.toList());
- // Group full-text index files by column and row range.
- Map<String, Map<Range, List<IndexFileMeta>>> byColumnAndRange = new
HashMap<>();
- for (IndexFileMeta indexFile : allIndexFiles) {
- GlobalIndexMeta meta = checkNotNull(indexFile.globalIndexMeta());
- String columnName =
checkNotNull(idToColumn.get(meta.indexFieldId()));
- Range range = new Range(meta.rowRangeStart(), meta.rowRangeEnd());
- byColumnAndRange
- .computeIfAbsent(columnName, k -> new HashMap<>())
- .computeIfAbsent(range, k -> new ArrayList<>())
- .add(indexFile);
- }
-
List<FullTextSearchSplit> splits = new ArrayList<>();
- for (Map.Entry<String, Map<Range, List<IndexFileMeta>>> columnEntry :
- byColumnAndRange.entrySet()) {
- String columnName = columnEntry.getKey();
- for (Map.Entry<Range, List<IndexFileMeta>> rangeEntry :
- columnEntry.getValue().entrySet()) {
- Range range = rangeEntry.getKey();
- splits.add(
- new IndexFullTextSearchSplit(
- columnName, range.from, range.to,
rangeEntry.getValue()));
- }
+ for (IndexRangeSelection selection :
+ chooseIndexRanges(allIndexFiles, textColumnIds, idToColumn)) {
+ splits.add(
+ new IndexFullTextSearchSplit(
+ selection.columnName,
+ selection.fileRange.from,
+ selection.fileRange.to,
+ selection.files,
+ selection.searchRanges));
}
if (!allIndexFiles.isEmpty()) {
@@ -139,6 +130,253 @@ public class FullTextScanImpl implements FullTextScan {
return () -> splits;
}
+ /**
+ * Returns the searched text-column ids served by {@code meta}: its
primary {@code indexFieldId}
+ * plus any {@code extraFieldIds} present in {@code textColumnIds}. This
lets a multi-column
+ * index (e.g. vector primary + text extra) serve full-text search on its
extra text column.
+ */
+ private static List<Integer> matchedTextColumnIds(
+ GlobalIndexMeta meta, Set<Integer> textColumnIds) {
+ List<Integer> matched = new ArrayList<>();
+ if (textColumnIds.contains(meta.indexFieldId())) {
+ matched.add(meta.indexFieldId());
+ }
+ int[] extraFieldIds = meta.extraFieldIds();
+ if (extraFieldIds != null) {
+ for (int extraFieldId : extraFieldIds) {
+ if (textColumnIds.contains(extraFieldId)) {
+ matched.add(extraFieldId);
+ }
+ }
+ }
+ return matched;
+ }
+
+ /**
+ * Chooses one index definition for every disjoint row interval of each
searched text column. A
+ * dedicated index (the text column is its primary field) wins over an
index that only carries
+ * the column as an extra field; remaining ties are resolved
deterministically by index identity
+ * and physical file range.
+ *
+ * <p>Index files may cover overlapping but non-identical ranges.
Exact-range grouping would
+ * keep all such files and search the overlap more than once. This method
splits their coverage
+ * at every range boundary, assigns each resulting interval once, and
records the assigned
+ * sub-ranges separately from the physical file range. Readers still need
the physical range to
+ * translate file-local row ids correctly.
+ */
+ private static List<IndexRangeSelection> chooseIndexRanges(
+ List<IndexFileMeta> allIndexFiles,
+ Set<Integer> textColumnIds,
+ Map<Integer, String> idToColumn) {
+ // column -> physical range -> index identity -> candidate files
+ Map<String, Map<Range, Map<IndexIdentity, IndexRangeCandidate>>>
grouped = new HashMap<>();
+ for (IndexFileMeta indexFile : allIndexFiles) {
+ GlobalIndexMeta meta = checkNotNull(indexFile.globalIndexMeta());
+ IndexIdentity identity = IndexIdentity.of(indexFile);
+ Range range = new Range(meta.rowRangeStart(), meta.rowRangeEnd());
+ for (int columnId : matchedTextColumnIds(meta, textColumnIds)) {
+ String columnName = checkNotNull(idToColumn.get(columnId));
+ boolean primary = meta.indexFieldId() == columnId;
+ IndexRangeCandidate candidate =
+ grouped.computeIfAbsent(columnName, k -> new
HashMap<>())
+ .computeIfAbsent(range, k -> new HashMap<>())
+ .computeIfAbsent(
+ identity,
+ k ->
+ new IndexRangeCandidate(
+ columnName, range,
identity, primary));
+ candidate.files.add(indexFile);
+ }
+ }
+
+ List<IndexRangeSelection> selections = new ArrayList<>();
+ for (Map.Entry<String, Map<Range, Map<IndexIdentity,
IndexRangeCandidate>>> columnEntry :
+ grouped.entrySet()) {
+ List<IndexRangeCandidate> candidates = new ArrayList<>();
+ for (Map<IndexIdentity, IndexRangeCandidate> byIdentity :
+ columnEntry.getValue().values()) {
+ candidates.addAll(byIdentity.values());
+ }
+
+ TreeSet<Long> boundaries = new TreeSet<>();
+ Map<Long, List<IndexRangeCandidate>> startingAt = new HashMap<>();
+ Map<Long, List<IndexRangeCandidate>> endingAt = new HashMap<>();
+ for (IndexRangeCandidate candidate : candidates) {
+ boundaries.add(candidate.fileRange.from);
+ startingAt
+ .computeIfAbsent(candidate.fileRange.from, k -> new
ArrayList<>())
+ .add(candidate);
+ if (candidate.fileRange.to != Long.MAX_VALUE) {
+ long afterEnd = candidate.fileRange.to + 1;
+ boundaries.add(afterEnd);
+ endingAt.computeIfAbsent(afterEnd, k -> new
ArrayList<>()).add(candidate);
+ }
+ }
+
+ List<Long> sortedBoundaries = new ArrayList<>(boundaries);
+ Map<IndexRangeCandidate, List<Range>> assigned = new
LinkedHashMap<>();
+ TreeSet<IndexRangeCandidate> active =
+ new TreeSet<>(FullTextScanImpl::compareCandidates);
+ for (int i = 0; i < sortedBoundaries.size(); i++) {
+ long from = sortedBoundaries.get(i);
+ List<IndexRangeCandidate> ending = endingAt.get(from);
+ if (ending != null) {
+ active.removeAll(ending);
+ }
+ List<IndexRangeCandidate> starting = startingAt.get(from);
+ if (starting != null) {
+ active.addAll(starting);
+ }
+ long to =
+ i + 1 < sortedBoundaries.size()
+ ? sortedBoundaries.get(i + 1) - 1
+ : Long.MAX_VALUE;
+ if (!active.isEmpty()) {
+ IndexRangeCandidate best = active.first();
+ assigned.computeIfAbsent(best, k -> new
ArrayList<>()).add(new Range(from, to));
+ }
+ }
+
+ for (Map.Entry<IndexRangeCandidate, List<Range>> entry :
assigned.entrySet()) {
+ IndexRangeCandidate candidate = entry.getKey();
+ selections.add(
+ new IndexRangeSelection(
+ candidate.columnName,
+ candidate.fileRange,
+ candidate.identity,
+ candidate.files,
+ mergeAdjacent(entry.getValue())));
+ }
+ }
+
+ selections.sort(
+ (left, right) -> {
+ int result = left.columnName.compareTo(right.columnName);
+ if (result != 0) {
+ return result;
+ }
+ result = Long.compare(left.fileRange.from,
right.fileRange.from);
+ if (result != 0) {
+ return result;
+ }
+ result = Long.compare(left.fileRange.to,
right.fileRange.to);
+ return result != 0
+ ? result
+ :
left.identity.key().compareTo(right.identity.key());
+ });
+ return selections;
+ }
+
+ private static int compareCandidates(IndexRangeCandidate left,
IndexRangeCandidate right) {
+ if (left.primary != right.primary) {
+ return left.primary ? -1 : 1;
+ }
+ int result = left.identity.key().compareTo(right.identity.key());
+ if (result != 0) {
+ return result;
+ }
+ result = Long.compare(left.fileRange.from, right.fileRange.from);
+ return result != 0 ? result : Long.compare(left.fileRange.to,
right.fileRange.to);
+ }
+
+ private static List<Range> mergeAdjacent(List<Range> ranges) {
+ List<Range> merged = new ArrayList<>();
+ for (Range next : ranges) {
+ if (merged.isEmpty()) {
+ merged.add(next);
+ continue;
+ }
+ Range current = merged.get(merged.size() - 1);
+ if (current.to != Long.MAX_VALUE && current.to + 1 == next.from) {
+ merged.set(merged.size() - 1, new Range(current.from,
next.to));
+ } else {
+ merged.add(next);
+ }
+ }
+ return merged;
+ }
+
+ @VisibleForTesting
+ static boolean sameIndexIdentity(IndexFileMeta left, IndexFileMeta right) {
+ return IndexIdentity.of(left).equals(IndexIdentity.of(right));
+ }
+
+ /**
+ * Identity of a global index definition — its index type, primary {@code
indexFieldId} and
+ * extra field ids. Two index files with the same identity belong to the
same index definition
+ * (differing only by row range/shard); files with different identities
must not be merged into
+ * one reader input.
+ */
+ private static final class IndexIdentity {
+ private final String key;
+
+ private IndexIdentity(String key) {
+ this.key = key;
+ }
+
+ static IndexIdentity of(IndexFileMeta indexFile) {
+ GlobalIndexMeta meta = checkNotNull(indexFile.globalIndexMeta());
+ int[] extra = meta.extraFieldIds();
+ return new IndexIdentity(
+ indexFile.indexType()
+ + '|'
+ + meta.indexFieldId()
+ + '|'
+ + Arrays.toString(extra == null ? new int[0] :
extra));
+ }
+
+ String key() {
+ return key;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ return o instanceof IndexIdentity && key.equals(((IndexIdentity)
o).key);
+ }
+
+ @Override
+ public int hashCode() {
+ return key.hashCode();
+ }
+ }
+
+ private static final class IndexRangeCandidate {
+ private final String columnName;
+ private final Range fileRange;
+ private final IndexIdentity identity;
+ private final boolean primary;
+ private final List<IndexFileMeta> files = new ArrayList<>();
+
+ private IndexRangeCandidate(
+ String columnName, Range fileRange, IndexIdentity identity,
boolean primary) {
+ this.columnName = columnName;
+ this.fileRange = fileRange;
+ this.identity = identity;
+ this.primary = primary;
+ }
+ }
+
+ private static final class IndexRangeSelection {
+ private final String columnName;
+ private final Range fileRange;
+ private final IndexIdentity identity;
+ private final List<IndexFileMeta> files;
+ private final List<Range> searchRanges;
+
+ private IndexRangeSelection(
+ String columnName,
+ Range fileRange,
+ IndexIdentity identity,
+ List<IndexFileMeta> files,
+ List<Range> searchRanges) {
+ this.columnName = columnName;
+ this.fileRange = fileRange;
+ this.identity = identity;
+ this.files = files;
+ this.searchRanges = searchRanges;
+ }
+ }
+
private static boolean supportsFullTextSearch(String indexType) {
GlobalIndexerFactory factory;
try {
diff --git
a/paimon-core/src/main/java/org/apache/paimon/table/source/IndexFullTextSearchSplit.java
b/paimon-core/src/main/java/org/apache/paimon/table/source/IndexFullTextSearchSplit.java
index 976620e015..54a55536f1 100644
---
a/paimon-core/src/main/java/org/apache/paimon/table/source/IndexFullTextSearchSplit.java
+++
b/paimon-core/src/main/java/org/apache/paimon/table/source/IndexFullTextSearchSplit.java
@@ -22,6 +22,7 @@ import org.apache.paimon.index.IndexFileMeta;
import org.apache.paimon.index.IndexFileMetaSerializer;
import org.apache.paimon.io.DataInputViewStreamWrapper;
import org.apache.paimon.io.DataOutputViewStreamWrapper;
+import org.apache.paimon.utils.Range;
import java.io.IOException;
import java.io.ObjectInputStream;
@@ -44,6 +45,7 @@ public class IndexFullTextSearchSplit extends
FullTextSearchSplit {
private String columnName;
private long rowRangeStart;
private long rowRangeEnd;
+ private List<Range> searchRowRanges;
private transient List<IndexFileMeta> fullTextIndexFiles;
public IndexFullTextSearchSplit(
@@ -56,10 +58,34 @@ public class IndexFullTextSearchSplit extends
FullTextSearchSplit {
long rowRangeStart,
long rowRangeEnd,
List<IndexFileMeta> fullTextIndexFiles) {
+ this(
+ columnName,
+ rowRangeStart,
+ rowRangeEnd,
+ fullTextIndexFiles,
+ Collections.singletonList(new Range(rowRangeStart,
rowRangeEnd)));
+ }
+
+ public IndexFullTextSearchSplit(
+ String columnName,
+ long rowRangeStart,
+ long rowRangeEnd,
+ List<IndexFileMeta> fullTextIndexFiles,
+ List<Range> searchRowRanges) {
this.columnName = columnName;
this.rowRangeStart = rowRangeStart;
this.rowRangeEnd = rowRangeEnd;
this.fullTextIndexFiles = Collections.unmodifiableList(new
ArrayList<>(fullTextIndexFiles));
+ List<Range> ranges = new ArrayList<>(searchRowRanges);
+ for (Range range : ranges) {
+ if (range.from < rowRangeStart || range.to > rowRangeEnd) {
+ throw new IllegalArgumentException(
+ String.format(
+ "Search range %s is outside physical index
range [%s, %s]",
+ range, rowRangeStart, rowRangeEnd));
+ }
+ }
+ this.searchRowRanges = Collections.unmodifiableList(ranges);
}
public String columnName() {
@@ -74,6 +100,10 @@ public class IndexFullTextSearchSplit extends
FullTextSearchSplit {
return rowRangeEnd;
}
+ public List<Range> searchRowRanges() {
+ return searchRowRanges;
+ }
+
public List<IndexFileMeta> fullTextIndexFiles() {
return fullTextIndexFiles;
}
@@ -96,6 +126,11 @@ public class IndexFullTextSearchSplit extends
FullTextSearchSplit {
DataInputViewStreamWrapper view = new DataInputViewStreamWrapper(in);
this.fullTextIndexFiles =
Collections.unmodifiableList(new
ArrayList<>(serializer.deserializeList(view)));
+ if (searchRowRanges == null) {
+ searchRowRanges = Collections.singletonList(new
Range(rowRangeStart, rowRangeEnd));
+ } else {
+ searchRowRanges = Collections.unmodifiableList(new
ArrayList<>(searchRowRanges));
+ }
}
@Override
@@ -107,12 +142,14 @@ public class IndexFullTextSearchSplit extends
FullTextSearchSplit {
return rowRangeStart == that.rowRangeStart
&& rowRangeEnd == that.rowRangeEnd
&& Objects.equals(columnName, that.columnName)
+ && Objects.equals(searchRowRanges, that.searchRowRanges)
&& Objects.equals(fullTextIndexFiles, that.fullTextIndexFiles);
}
@Override
public int hashCode() {
- return Objects.hash(columnName, rowRangeStart, rowRangeEnd,
fullTextIndexFiles);
+ return Objects.hash(
+ columnName, rowRangeStart, rowRangeEnd, searchRowRanges,
fullTextIndexFiles);
}
@Override
@@ -125,6 +162,8 @@ public class IndexFullTextSearchSplit extends
FullTextSearchSplit {
+ rowRangeStart
+ ", rowRangeEnd="
+ rowRangeEnd
+ + ", searchRowRanges="
+ + searchRowRanges
+ ", fullTextIndexFiles="
+ fullTextIndexFiles
+ '}';
diff --git
a/paimon-core/src/test/java/org/apache/paimon/table/source/FullTextSearchBuilderTest.java
b/paimon-core/src/test/java/org/apache/paimon/table/source/FullTextSearchBuilderTest.java
index 052232e5bb..c092932ef7 100644
---
a/paimon-core/src/test/java/org/apache/paimon/table/source/FullTextSearchBuilderTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/table/source/FullTextSearchBuilderTest.java
@@ -32,6 +32,7 @@ import org.apache.paimon.globalindex.ResultEntry;
import org.apache.paimon.globalindex.ScoredGlobalIndexResult;
import org.apache.paimon.globalindex.btree.BTreeGlobalIndexerFactory;
import
org.apache.paimon.globalindex.testfulltext.TestFullTextGlobalIndexerFactory;
+import org.apache.paimon.index.GlobalIndexMeta;
import org.apache.paimon.index.IndexFileMeta;
import org.apache.paimon.io.CompactIncrement;
import org.apache.paimon.io.DataIncrement;
@@ -549,12 +550,14 @@ public class FullTextSearchBuilderTest extends
TableTestBase {
}
@Test
- public void testFullTextSearchRequiresTextColumnAsPrimaryField() throws
Exception {
+ public void testFullTextSearchServesTextColumnAsExtraField() throws
Exception {
createTableDefault();
FileStoreTable table = getTableDefault();
String[] documents = {"Apache Paimon", "vector search"};
writeDocuments(table, documents);
+ // Multi-column index: primary is "id", the text column is an EXTRA
field. Full-text search
+ // on the extra text column is served (matched via extraFieldIds).
buildAndCommitIndexWithFields(
table,
documents,
@@ -566,6 +569,175 @@ public class FullTextSearchBuilderTest extends
TableTestBase {
.withQuery(TEXT_FIELD_NAME, matchQuery("Paimon"))
.withLimit(2);
+
assertThat(searchBuilder.newFullTextScan().scan().splits()).isNotEmpty();
+ assertThat(searchBuilder.executeLocal().results().isEmpty()).isFalse();
+ }
+
+ @Test
+ public void
testFullTextSearchPrefersDedicatedIndexOverExtraFieldCoverage() throws
Exception {
+ createTableDefault();
+ FileStoreTable table = getTableDefault();
+
+ String[] documents = {"Apache Paimon", "vector search"};
+ writeDocuments(table, documents);
+
+ // Two full-text-capable indexes cover the same text column over the
same row range:
+ // (a) a dedicated full-text index whose PRIMARY field is the text
column, and
+ // (b) a multi-column index whose primary is "id" and carries the
text column as an EXTRA
+ // field.
+ // These are different index definitions and must not be merged into
one reader input; the
+ // dedicated index (a) takes precedence, so each split carries a
single index file.
+ buildAndCommitIndex(table, documents); // (a) primary = content
+ buildAndCommitIndexWithFields(
+ table,
+ documents,
+ Arrays.asList(
+ table.rowType().getField("id"),
+ table.rowType().getField(TEXT_FIELD_NAME))); // (b)
content as extra
+
+ FullTextSearchBuilder searchBuilder =
+ table.newFullTextSearchBuilder()
+ .withQuery(TEXT_FIELD_NAME, matchQuery("Paimon"))
+ .withLimit(2);
+
+ List<FullTextSearchSplit> splits =
searchBuilder.newFullTextScan().scan().splits();
+ assertThat(splits).isNotEmpty();
+ int textFieldId = table.rowType().getField(TEXT_FIELD_NAME).id();
+ for (FullTextSearchSplit split : splits) {
+ IndexFullTextSearchSplit indexSplit = (IndexFullTextSearchSplit)
split;
+ // Files from different index definitions are never merged into
one split ...
+ assertThat(indexSplit.fullTextIndexFiles()).hasSize(1);
+ // ... and the chosen definition is the dedicated one (text column
is its primary
+ // field).
+
assertThat(indexSplit.fullTextIndexFiles().get(0).globalIndexMeta().indexFieldId())
+ .isEqualTo(textFieldId);
+ }
+ assertThat(searchBuilder.executeLocal().results().isEmpty()).isFalse();
+ }
+
+ @Test
+ public void testFullTextSearchKeepsRangeCoveredOnlyByExtraFieldIndex()
throws Exception {
+ createTableDefault();
+ FileStoreTable table = getTableDefault();
+
+ String[] documents = {"paimon lake", "vector search", "paimon engine",
"streaming"};
+ writeDocuments(table, documents);
+
+ // Two index definitions cover DIFFERENT row ranges of the same text
column:
+ // - a dedicated full-text index (primary field = content) over rows
[0, 1], and
+ // - a multi-column index (primary = id, content as an extra field)
over rows [2, 3].
+ // Per-column selection would keep only the dedicated identity and
drop the extra-field
+ // index's [2, 3] file, while raw coverage (computed from all index
files) still treats
+ // [2, 3] as indexed, so row 2 ("paimon engine") would be silently
unsearchable. Per-range
+ // selection must keep both ranges searchable.
+ buildAndCommitIndexRange(
+ table,
+ new String[] {"paimon lake", "vector search"},
+
Collections.singletonList(table.rowType().getField(TEXT_FIELD_NAME)),
+ 0);
+ buildAndCommitIndexRange(
+ table,
+ new String[] {"paimon engine", "streaming"},
+ Arrays.asList(
+ table.rowType().getField("id"),
table.rowType().getField(TEXT_FIELD_NAME)),
+ 2);
+
+ GlobalIndexResult result =
+ table.newFullTextSearchBuilder()
+ .withQuery(TEXT_FIELD_NAME, matchQuery("paimon"))
+ .withLimit(10)
+ .executeLocal();
+
+ // Row 0 is served by the dedicated index over [0,1]; row 2 by the
extra-field index over
+ // [2,3]. Neither range may be dropped.
+ assertThat(readIds(table, result)).containsExactlyInAnyOrder(0, 2);
+ }
+
+ @Test
+ public void testFullTextSearchAssignsOverlappingRangesOnlyOnce() throws
Exception {
+ createTableDefault();
+ FileStoreTable table = getTableDefault();
+
+ String[] documents = {"needle zero", "other", "needle two", "needle
three"};
+ writeDocuments(table, documents);
+
+ // The dedicated index covers [0, 2], while an index carrying content
as an extra field
+ // covers [2, 3]. Row 2 is in both physical files, but must be
assigned only to the
+ // dedicated index. The second file is still read with its physical
[2, 3] offset and an
+ // include mask for its assigned tail [3, 3].
+ buildAndCommitIndexRange(
+ table,
+ new String[] {documents[0], documents[1], documents[2]},
+
Collections.singletonList(table.rowType().getField(TEXT_FIELD_NAME)),
+ 0);
+ buildAndCommitIndexRange(
+ table,
+ new String[] {documents[2], documents[3]},
+ Arrays.asList(
+ table.rowType().getField("id"),
table.rowType().getField(TEXT_FIELD_NAME)),
+ 2);
+
+ FullTextSearchBuilder searchBuilder =
+ table.newFullTextSearchBuilder()
+ .withQuery(TEXT_FIELD_NAME, matchQuery("needle"))
+ .withLimit(10);
+ List<IndexFullTextSearchSplit> indexSplits = new ArrayList<>();
+ for (FullTextSearchSplit split :
searchBuilder.newFullTextScan().scan().splits()) {
+ indexSplits.add((IndexFullTextSearchSplit) split);
+ }
+
+ assertThat(indexSplits).hasSize(2);
+ assertThat(indexSplits.get(0).rowRangeStart()).isEqualTo(0);
+ assertThat(indexSplits.get(0).rowRangeEnd()).isEqualTo(2);
+ assertThat(indexSplits.get(0).searchRowRanges()).containsExactly(new
Range(0, 2));
+ assertThat(indexSplits.get(1).rowRangeStart()).isEqualTo(2);
+ assertThat(indexSplits.get(1).rowRangeEnd()).isEqualTo(3);
+ assertThat(indexSplits.get(1).searchRowRanges()).containsExactly(new
Range(3, 3));
+
+ assertThat(readIds(table,
searchBuilder.executeLocal())).containsExactlyInAnyOrder(0, 2, 3);
+ }
+
+ @Test
+ public void testFullTextIndexIdentityPreservesExtraFieldOrder() {
+ IndexFileMeta titleThenBody =
+ new IndexFileMeta(
+ TestFullTextGlobalIndexerFactory.IDENTIFIER,
+ "title-body",
+ 1,
+ 2,
+ new GlobalIndexMeta(0, 1, 0, new int[] {1, 2}, null),
+ null);
+ IndexFileMeta bodyThenTitle =
+ new IndexFileMeta(
+ TestFullTextGlobalIndexerFactory.IDENTIFIER,
+ "body-title",
+ 1,
+ 2,
+ new GlobalIndexMeta(0, 1, 0, new int[] {2, 1}, null),
+ null);
+
+ // The reader's projected row layout follows this order, so these are
different index
+ // definitions even though they contain the same field-id set.
+ assertThat(FullTextScanImpl.sameIndexIdentity(titleThenBody,
bodyThenTitle)).isFalse();
+ }
+
+ @Test
+ public void testFullTextSearchSkipsIndexNotCoveringTextColumn() throws
Exception {
+ createTableDefault();
+ FileStoreTable table = getTableDefault();
+
+ String[] documents = {"Apache Paimon", "vector search"};
+ writeDocuments(table, documents);
+ // The index covers only "id" — the text column is neither the primary
nor an extra field —
+ // so full-text search on the text column finds no index and returns
empty.
+ buildAndCommitIndexWithFields(
+ table, documents,
Arrays.asList(table.rowType().getField("id")));
+
+ FullTextSearchBuilder searchBuilder =
+ table.newFullTextSearchBuilder()
+ .withQuery(TEXT_FIELD_NAME, matchQuery("Paimon"))
+ .withLimit(2);
+
FullTextScan.Plan plan = searchBuilder.newFullTextScan().scan();
assertThat(plan.splits()).isEmpty();
assertThat(searchBuilder.executeLocal().results().isEmpty()).isTrue();
@@ -604,6 +776,7 @@ public class FullTextSearchBuilderTest extends
TableTestBase {
assertThat(deserialized.columnName()).isEqualTo(original.columnName());
assertThat(deserialized.rowRangeStart()).isEqualTo(original.rowRangeStart());
assertThat(deserialized.rowRangeEnd()).isEqualTo(original.rowRangeEnd());
+
assertThat(deserialized.searchRowRanges()).isEqualTo(original.searchRowRanges());
assertThat(deserialized.fullTextIndexFiles()).hasSize(original.fullTextIndexFiles().size());
for (int i = 0; i < original.fullTextIndexFiles().size(); i++) {
assertThat(deserialized.fullTextIndexFiles().get(i).fileName())
@@ -731,6 +904,56 @@ public class FullTextSearchBuilderTest extends
TableTestBase {
}
}
+ /**
+ * Builds and commits a single full-text index file covering rows
[rowStart, rowStart+N-1].
+ * {@code indexFields} determines the index identity (primary = first
field, the rest are extra
+ * fields), letting a test place different index definitions over
different row ranges.
+ */
+ private void buildAndCommitIndexRange(
+ FileStoreTable table, String[] documents, List<DataField>
indexFields, int rowStart)
+ throws Exception {
+ Options options = table.coreOptions().toConfiguration();
+ DataField textField = table.rowType().getField(TEXT_FIELD_NAME);
+
+ GlobalIndexSingleColumnWriter writer =
+ (GlobalIndexSingleColumnWriter)
+ GlobalIndexBuilderUtils.createIndexWriter(
+ table,
+ TestFullTextGlobalIndexerFactory.IDENTIFIER,
+ textField,
+ options);
+ // Doc ids are file-local (0-based); the global row offset is carried
by rowRange.from,
+ // which
+ // the OffsetGlobalIndexReader adds back when mapping local doc ids to
global row ids.
+ for (int i = 0; i < documents.length; i++) {
+ writer.write(documents[i], i);
+ }
+ List<ResultEntry> entries = writer.finish();
+
+ Range rowRange = new Range(rowStart, rowStart + documents.length - 1);
+ List<IndexFileMeta> indexFiles =
+ GlobalIndexBuilderUtils.toIndexFileMetas(
+ table.fileIO(),
+ table.store().pathFactory().globalIndexFileFactory(),
+ table.coreOptions(),
+ rowRange,
+ indexFields,
+ TestFullTextGlobalIndexerFactory.IDENTIFIER,
+ entries);
+
+ DataIncrement dataIncrement = DataIncrement.indexIncrement(indexFiles);
+ CommitMessage message =
+ new CommitMessageImpl(
+ BinaryRow.EMPTY_ROW,
+ 0,
+ null,
+ dataIncrement,
+ CompactIncrement.emptyIncrement());
+ try (BatchTableCommit commit =
table.newBatchWriteBuilder().newCommit()) {
+ commit.commit(Collections.singletonList(message));
+ }
+ }
+
private FileStoreTable createMultiTextTable() throws Exception {
Identifier identifier = identifier("MultiTextTable");
Schema schema =
diff --git
a/paimon-core/src/test/java/org/apache/paimon/table/source/VectorSearchBuilderTest.java
b/paimon-core/src/test/java/org/apache/paimon/table/source/VectorSearchBuilderTest.java
index 432e5333bf..93e219f442 100644
---
a/paimon-core/src/test/java/org/apache/paimon/table/source/VectorSearchBuilderTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/table/source/VectorSearchBuilderTest.java
@@ -25,6 +25,7 @@ import org.apache.paimon.data.GenericRow;
import org.apache.paimon.data.InternalRow;
import org.apache.paimon.data.serializer.InternalRowSerializer;
import org.apache.paimon.globalindex.GlobalIndexBuilderUtils;
+import org.apache.paimon.globalindex.GlobalIndexMultiColumnWriter;
import org.apache.paimon.globalindex.GlobalIndexResult;
import org.apache.paimon.globalindex.GlobalIndexSingleColumnWriter;
import org.apache.paimon.globalindex.ResultEntry;
@@ -1143,6 +1144,96 @@ public class VectorSearchBuilderTest extends
TableTestBase {
assertThat(searchBuilder.executeLocal().results().isEmpty()).isTrue();
}
+ @Test
+ public void testVectorPrimaryMultiFieldIndexServesScalarExtraField()
throws Exception {
+ catalog.createTable(
+ identifier("vector_primary_scalar_extra_field_table"),
+ vectorSchemaBuilder(VECTOR_FIELD_NAME)
+ .option(CoreOptions.GLOBAL_INDEX_SEARCH_MODE.key(),
"full")
+ .build(),
+ false);
+ FileStoreTable table =
getTable(identifier("vector_primary_scalar_extra_field_table"));
+
+ float[][] vectors = {{1.0f, 0.0f}, {0.0f, 1.0f}};
+ writeVectors(table, vectors);
+
+ DataField vectorField = table.rowType().getField(VECTOR_FIELD_NAME);
+ DataField idField = table.rowType().getField("id");
+ buildAndCommitVectorIndexWithFields(
+ table, vectors, new Range(0, 1), Arrays.asList(vectorField,
idField));
+
+ Predicate idFilter = new
PredicateBuilder(table.rowType()).greaterOrEqual(0, 1);
+ VectorScan.Plan plan =
+ table.newVectorSearchBuilder()
+ .withVector(new float[] {1.0f, 0.0f})
+ .withLimit(2)
+ .withVectorColumn(VECTOR_FIELD_NAME)
+ .withFilter(idFilter)
+ .newVectorScan()
+ .scan();
+
+ // A vector-primary multi-field index can serve both the vector search
and a scalar
+ // predicate on an extra field. It must therefore be attached as both
vector and scalar
+ // index input, and full search mode must not create a raw fallback
for the covered range.
+ assertThat(indexVectorSearchSplits(plan.splits())).hasSize(1);
+ assertThat(rawVectorSearchSplits(plan.splits())).isEmpty();
+
+ IndexVectorSearchSplit split =
indexVectorSearchSplits(plan.splits()).get(0);
+ assertThat(split.vectorIndexFiles()).hasSize(2);
+
assertThat(split.scalarIndexFiles()).containsExactlyElementsOf(split.vectorIndexFiles());
+
+ for (IndexFileMeta indexFile : split.vectorIndexFiles()) {
+
assertThat(indexFile.globalIndexMeta().indexFieldId()).isEqualTo(vectorField.id());
+
assertThat(indexFile.globalIndexMeta().extraFieldIds()).containsExactly(idField.id());
+ }
+
+ GlobalIndexResult result =
+ table.newVectorSearchBuilder()
+ .withVector(new float[] {1.0f, 0.0f})
+ .withLimit(2)
+ .withVectorColumn(VECTOR_FIELD_NAME)
+ .withFilter(idFilter)
+ .executeLocal();
+ assertThat(result.results()).containsExactly(1L);
+ assertThat(readIds(table, result)).containsExactly(1);
+ }
+
+ @Test
+ public void testScalarExtraFieldComplementsPartialDedicatedIndex() throws
Exception {
+ catalog.createTable(
+ identifier("vector_extra_complements_dedicated_index_table"),
+ vectorSchemaBuilder(VECTOR_FIELD_NAME)
+ .option(CoreOptions.GLOBAL_INDEX_SEARCH_MODE.key(),
"full")
+ .build(),
+ false);
+ FileStoreTable table =
+
getTable(identifier("vector_extra_complements_dedicated_index_table"));
+
+ float[][] vectors = {{0.0f, 1.0f}, {0.1f, 0.9f}, {1.0f, 0.0f}, {0.9f,
0.1f}};
+ writeVectors(table, vectors);
+
+ DataField vectorField = table.rowType().getField(VECTOR_FIELD_NAME);
+ DataField idField = table.rowType().getField("id");
+ buildAndCommitVectorIndexWithFields(
+ table, vectors, new Range(0, 3), Arrays.asList(vectorField,
idField));
+ // The dedicated scalar index covers only the head. The vector index's
scalar extra field
+ // must still serve the tail; otherwise GlobalIndexScanner's
primary-field preference drops
+ // rows [2, 3] even though coverage planning treats them as indexed.
+ buildAndCommitBTreeIndex(table, new int[] {0, 1}, new Range(0, 1));
+
+ Predicate idFilter = new
PredicateBuilder(table.rowType()).greaterOrEqual(0, 2);
+ GlobalIndexResult result =
+ table.newVectorSearchBuilder()
+ .withVector(new float[] {1.0f, 0.0f})
+ .withLimit(1)
+ .withVectorColumn(VECTOR_FIELD_NAME)
+ .withFilter(idFilter)
+ .executeLocal();
+
+ assertThat(result.results()).containsExactly(2L);
+ assertThat(readIds(table, result)).containsExactly(2);
+ }
+
@Test
public void testVectorSearchSplitSerialization() throws Exception {
createTableDefault();
@@ -1713,17 +1804,36 @@ public class VectorSearchBuilderTest extends
TableTestBase {
Options options = table.coreOptions().toConfiguration();
DataField vectorField = table.rowType().getField(VECTOR_FIELD_NAME);
- GlobalIndexSingleColumnWriter writer =
- (GlobalIndexSingleColumnWriter)
- GlobalIndexBuilderUtils.createIndexWriter(
- table,
- TestVectorGlobalIndexerFactory.IDENTIFIER,
- vectorField,
- options);
- for (int i = 0; i < vectors.length; i++) {
- writer.write(vectors[i], i);
+ List<ResultEntry> entries;
+ if (indexFields.size() > 1 && indexFields.get(0).id() ==
vectorField.id()) {
+ GlobalIndexMultiColumnWriter writer =
+ (GlobalIndexMultiColumnWriter)
+ GlobalIndexBuilderUtils.createIndexWriter(
+ table,
+ TestVectorGlobalIndexerFactory.IDENTIFIER,
+ vectorField,
+ indexFields.subList(1, indexFields.size()),
+ options);
+ for (int i = 0; i < vectors.length; i++) {
+ writer.write(
+ i, GenericRow.of(new GenericArray(vectors[i]), (int)
(rowRange.from + i)));
+ }
+ entries = writer.finish();
+ } else {
+ // This path is used by the malformed-metadata test where the
vector is deliberately
+ // not the primary field. The scan rejects it before constructing
a reader.
+ GlobalIndexSingleColumnWriter writer =
+ (GlobalIndexSingleColumnWriter)
+ GlobalIndexBuilderUtils.createIndexWriter(
+ table,
+ TestVectorGlobalIndexerFactory.IDENTIFIER,
+ vectorField,
+ options);
+ for (int i = 0; i < vectors.length; i++) {
+ writer.write(vectors[i], i);
+ }
+ entries = writer.finish();
}
- List<ResultEntry> entries = writer.finish();
List<IndexFileMeta> indexFiles =
GlobalIndexBuilderUtils.toIndexFileMetas(
