This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new feecdfe064 [core] Add global index search mode (#8296)
feecdfe064 is described below
commit feecdfe064e7a79946467ec3e37cabdae3d912fd
Author: Jingsong Lee <[email protected]>
AuthorDate: Sat Jun 20 18:41:11 2026 +0800
[core] Add global index search mode (#8296)
Adds `global-index.search-mode` so global index scans can choose between
indexed-only search and scanning unindexed raw rows when needed.
---
docs/docs/multimodal-table/global-index.mdx | 17 ++-
docs/generated/core_configuration.html | 6 +
.../main/java/org/apache/paimon/CoreOptions.java | 43 ++++++
.../apache/paimon/predicate/PredicateVisitor.java | 15 +++
.../paimon/predicate/PredicateVisitorTest.java | 53 ++++++++
.../paimon/globalindex/DataEvolutionBatchScan.java | 7 +-
.../paimon/globalindex/GlobalIndexScanner.java | 147 ++++++++++++++++++---
.../paimon/table/BtreeGlobalIndexTableTest.java | 106 +++++++++++++++
8 files changed, 375 insertions(+), 19 deletions(-)
diff --git a/docs/docs/multimodal-table/global-index.mdx
b/docs/docs/multimodal-table/global-index.mdx
index 90a8ac33bc..1cecce7993 100644
--- a/docs/docs/multimodal-table/global-index.mdx
+++ b/docs/docs/multimodal-table/global-index.mdx
@@ -129,8 +129,20 @@ ALTER TABLE my_table SET (
Global index files cover row-id ranges. If more rows are appended after an
index is built, those
new rows are not automatically covered by the existing index files. Run
`create_global_index` again
-to build index files for newly uncovered data. A query that can be answered by
a matching global
-index reads indexed row ranges; rows in uncovered ranges are not returned for
that indexed query.
+to build index files for newly uncovered data. By default, queries use fast
search and only read
+indexed row ranges; rows in uncovered ranges are not returned for that indexed
query.
+
+To improve freshness for query types that support raw-data search, set:
+
+```sql
+ALTER TABLE my_table SET ('global-index.search-mode' = 'full');
+```
+
+With `full` search, supported global-index queries first use the snapshot
`nextRowId` and global
+index row-id coverage to detect whether any row range is missing from the
index. Raw data is scanned
+only when such a gap exists. Use `detail` search when data files may have been
rewritten or updated
+after index creation; it scans data file metadata to find the exact unindexed
row ranges and can
+handle index invalidation caused by updates or rewrites.
To temporarily disable global-index scan acceleration while keeping the index
files, set:
@@ -147,6 +159,7 @@ These table options affect global index build and read
behavior:
| Option | Default | Description |
|---|---|---|
| `global-index.enabled` | `true` | Whether scans can use global indexes. |
+| `global-index.search-mode` | `fast` | Search mode for global-index queries.
`fast` searches indexed data only. `full` checks snapshot `nextRowId` against
global index row-id coverage and scans raw data only if a gap exists. `detail`
scans data file metadata to find exact unindexed rows and can handle index
invalidation caused by updates or rewrites. |
| `global-index.external-path` | Not set | Root directory for global index
files. If not set, files are stored under the table index directory. |
| `sorted-index.records-per-range` | `10000000` | Expected number of records
per sorted global index file for BTree and Bitmap builds. |
| `sorted-index.build.max-parallelism` | `4096` | Maximum Flink or Spark
parallelism for building sorted global indexes. |
diff --git a/docs/generated/core_configuration.html
b/docs/generated/core_configuration.html
index 575c40e21f..9d1a962463 100644
--- a/docs/generated/core_configuration.html
+++ b/docs/generated/core_configuration.html
@@ -746,6 +746,12 @@ under the License.
<td>Long</td>
<td>Row count per shard for global index.</td>
</tr>
+ <tr>
+ <td><h5>global-index.search-mode</h5></td>
+ <td style="word-wrap: break-word;">fast</td>
+ <td><p>Enum</p></td>
+ <td>Search mode for global index queries. Supported values are
'fast', 'full', and 'detail'.<br /><br />Possible values:<ul><li>"fast": Only
search indexed data.</li><li>"full": Use snapshot next row id and global index
coverage to detect missing row ids, and scan raw data only when a gap
exists.</li><li>"detail": Scan data files to find exact unindexed rows. This
can handle index invalidation caused by updates or rewrites.</li></ul></td>
+ </tr>
<tr>
<td><h5>global-index.thread-num</h5></td>
<td style="word-wrap: break-word;">32</td>
diff --git a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java
b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java
index f2a0d32a4d..a884752434 100644
--- a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java
+++ b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java
@@ -2543,6 +2543,14 @@ public class CoreOptions implements Serializable {
.defaultValue(true)
.withDescription("Whether to enable global index for
scan.");
+ public static final ConfigOption<GlobalIndexSearchMode>
GLOBAL_INDEX_SEARCH_MODE =
+ key("global-index.search-mode")
+ .enumType(GlobalIndexSearchMode.class)
+ .defaultValue(GlobalIndexSearchMode.FAST)
+ .withDescription(
+ "Search mode for global index queries. "
+ + "Supported values are 'fast', 'full',
and 'detail'.");
+
public static final ConfigOption<Integer> GLOBAL_INDEX_THREAD_NUM =
key("global-index.thread-num")
.intType()
@@ -4049,6 +4057,10 @@ public class CoreOptions implements Serializable {
return options.get(GLOBAL_INDEX_ENABLED);
}
+ public GlobalIndexSearchMode globalIndexSearchMode() {
+ return options.get(GLOBAL_INDEX_SEARCH_MODE);
+ }
+
public Integer globalIndexThreadNum() {
return options.get(GLOBAL_INDEX_THREAD_NUM);
}
@@ -4846,4 +4858,35 @@ public class CoreOptions implements Serializable {
/** Drop all global index entries for the whole partitions affected by
the update. */
DROP_PARTITION_INDEX
}
+
+ /** Search mode for global index queries. */
+ public enum GlobalIndexSearchMode implements DescribedEnum {
+ FAST("fast", "Only search indexed data."),
+ FULL(
+ "full",
+ "Use snapshot next row id and global index coverage to detect
missing row ids, "
+ + "and scan raw data only when a gap exists."),
+ DETAIL(
+ "detail",
+ "Scan data files to find exact unindexed rows. "
+ + "This can handle index invalidation caused by
updates or rewrites.");
+
+ private final String value;
+ private final String description;
+
+ GlobalIndexSearchMode(String value, String description) {
+ this.value = value;
+ this.description = description;
+ }
+
+ @Override
+ public String toString() {
+ return value;
+ }
+
+ @Override
+ public InlineElement getDescription() {
+ return text(description);
+ }
+ }
}
diff --git
a/paimon-common/src/main/java/org/apache/paimon/predicate/PredicateVisitor.java
b/paimon-common/src/main/java/org/apache/paimon/predicate/PredicateVisitor.java
index f14cb32886..9f4047b8cb 100644
---
a/paimon-common/src/main/java/org/apache/paimon/predicate/PredicateVisitor.java
+++
b/paimon-common/src/main/java/org/apache/paimon/predicate/PredicateVisitor.java
@@ -18,6 +18,8 @@
package org.apache.paimon.predicate;
+import org.apache.paimon.types.RowType;
+
import javax.annotation.Nullable;
import java.util.Collections;
@@ -38,6 +40,19 @@ public interface PredicateVisitor<T> {
return predicate.visit(new FieldNameCollector());
}
+ static Set<Integer> collectFieldIds(RowType rowType, @Nullable Predicate
predicate) {
+ if (predicate == null) {
+ return Collections.emptySet();
+ }
+ Set<Integer> fieldIds = new HashSet<>();
+ for (String name : collectFieldNames(predicate)) {
+ if (rowType.containsField(name)) {
+ fieldIds.add(rowType.getField(name).id());
+ }
+ }
+ return fieldIds;
+ }
+
/** A visitor that collects all field names referenced by a predicate. */
class FieldNameCollector implements PredicateVisitor<Set<String>> {
diff --git
a/paimon-common/src/test/java/org/apache/paimon/predicate/PredicateVisitorTest.java
b/paimon-common/src/test/java/org/apache/paimon/predicate/PredicateVisitorTest.java
new file mode 100644
index 0000000000..393e05f3ea
--- /dev/null
+++
b/paimon-common/src/test/java/org/apache/paimon/predicate/PredicateVisitorTest.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.predicate;
+
+import org.apache.paimon.types.DataField;
+import org.apache.paimon.types.DataTypes;
+import org.apache.paimon.types.RowType;
+
+import org.junit.jupiter.api.Test;
+
+import java.util.Arrays;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/** Tests for {@link PredicateVisitor}. */
+public class PredicateVisitorTest {
+
+ @Test
+ public void testCollectFieldIdsUsesFieldIdInsteadOfFieldPosition() {
+ RowType rowType =
+ new RowType(
+ Arrays.asList(
+ new DataField(10, "a", DataTypes.INT()),
+ new DataField(20, "b", DataTypes.INT()),
+ new DataField(30, "c", DataTypes.INT())));
+ PredicateBuilder builder = new PredicateBuilder(rowType);
+
+ Predicate predicate =
+ PredicateBuilder.or(
+ builder.equal(0, 1),
+ PredicateBuilder.and(builder.equal(1, 2),
builder.equal(0, 3)));
+
+ assertThat(PredicateVisitor.collectFieldIds(rowType, predicate))
+ .containsExactlyInAnyOrder(10, 20);
+ assertThat(PredicateVisitor.collectFieldIds(rowType, null)).isEmpty();
+ }
+}
diff --git
a/paimon-core/src/main/java/org/apache/paimon/globalindex/DataEvolutionBatchScan.java
b/paimon-core/src/main/java/org/apache/paimon/globalindex/DataEvolutionBatchScan.java
index 8d6038ccc4..b03f27f3d3 100644
---
a/paimon-core/src/main/java/org/apache/paimon/globalindex/DataEvolutionBatchScan.java
+++
b/paimon-core/src/main/java/org/apache/paimon/globalindex/DataEvolutionBatchScan.java
@@ -282,8 +282,9 @@ public class DataEvolutionBatchScan implements
DataTableScan {
Optional<GlobalIndexResult> result = scanner.scan(filter);
if (result.isPresent()) {
LOG.info("Scan table '{}' with global index.", table.name());
+ return result;
}
- return result;
+ return Optional.empty();
} catch (IOException e) {
throw new RuntimeException(e);
}
@@ -296,7 +297,9 @@ public class DataEvolutionBatchScan implements
DataTableScan {
Function<Split, List<IndexedSplit>> process =
split ->
Collections.singletonList(
- wrap((DataSplit) split, rowRangeIndex,
scoreGetter));
+ split instanceof IndexedSplit
+ ? (IndexedSplit) split
+ : wrap((DataSplit) split,
rowRangeIndex, scoreGetter));
randomlyExecuteSequentialReturn(process, splits,
null).forEachRemaining(indexedSplits::add);
return () -> indexedSplits;
}
diff --git
a/paimon-core/src/main/java/org/apache/paimon/globalindex/GlobalIndexScanner.java
b/paimon-core/src/main/java/org/apache/paimon/globalindex/GlobalIndexScanner.java
index 1c8bbd3dd7..93504d317c 100644
---
a/paimon-core/src/main/java/org/apache/paimon/globalindex/GlobalIndexScanner.java
+++
b/paimon-core/src/main/java/org/apache/paimon/globalindex/GlobalIndexScanner.java
@@ -18,21 +18,29 @@
package org.apache.paimon.globalindex;
+import org.apache.paimon.CoreOptions.GlobalIndexSearchMode;
+import org.apache.paimon.Snapshot;
import org.apache.paimon.fs.FileIO;
import org.apache.paimon.fs.Path;
import org.apache.paimon.globalindex.io.GlobalIndexFileReader;
import org.apache.paimon.index.GlobalIndexMeta;
import org.apache.paimon.index.IndexFileMeta;
import org.apache.paimon.index.IndexPathFactory;
+import org.apache.paimon.io.DataFileMeta;
import org.apache.paimon.manifest.IndexManifestEntry;
import org.apache.paimon.options.Options;
import org.apache.paimon.partition.PartitionPredicate;
import org.apache.paimon.predicate.Predicate;
import org.apache.paimon.table.FileStoreTable;
+import org.apache.paimon.table.source.DataSplit;
+import org.apache.paimon.table.source.ScanMode;
+import org.apache.paimon.table.source.Split;
+import org.apache.paimon.table.source.snapshot.SnapshotReader;
import org.apache.paimon.types.DataField;
import org.apache.paimon.types.RowType;
import org.apache.paimon.utils.Filter;
import org.apache.paimon.utils.Range;
+import org.apache.paimon.utils.RoaringNavigableMap64;
import java.io.Closeable;
import java.io.IOException;
@@ -51,7 +59,7 @@ import java.util.function.IntFunction;
import java.util.stream.Collectors;
import static org.apache.paimon.CoreOptions.GLOBAL_INDEX_THREAD_NUM;
-import static org.apache.paimon.predicate.PredicateVisitor.collectFieldNames;
+import static org.apache.paimon.predicate.PredicateVisitor.collectFieldIds;
import static
org.apache.paimon.table.source.snapshot.TimeTravelUtil.tryTravelOrLatest;
import static org.apache.paimon.utils.Preconditions.checkArgument;
import static org.apache.paimon.utils.Preconditions.checkNotNull;
@@ -60,28 +68,47 @@ import static
org.apache.paimon.utils.Preconditions.checkNotNull;
public class GlobalIndexScanner implements Closeable {
private final Options options;
+ private final RowType rowType;
private final ExecutorService executor;
private final GlobalIndexEvaluator globalIndexEvaluator;
private final IndexPathFactory indexPathFactory;
+ private final Map<Integer, List<Range>> coverageByField;
+ private final FileStoreTable table;
+ private final Snapshot snapshot;
+ private final PartitionPredicate partitionFilter;
- public GlobalIndexScanner(
+ private GlobalIndexScanner(
+ FileStoreTable table,
+ Snapshot snapshot,
+ PartitionPredicate partitionFilter,
Options options,
RowType rowType,
FileIO fileIO,
IndexPathFactory indexPathFactory,
Collection<IndexFileMeta> indexFiles) {
+ this.table = table;
+ this.snapshot = snapshot;
+ this.partitionFilter = partitionFilter;
this.options = options;
+ this.rowType = rowType;
this.executor =
GlobalIndexReadThreadPool.getExecutorService(options.get(GLOBAL_INDEX_THREAD_NUM));
this.indexPathFactory = indexPathFactory;
GlobalIndexFileReader indexFileReader = meta ->
fileIO.newInputStream(meta.filePath());
Map<Integer, IndexMetaFileGroup> indexMetas = new HashMap<>();
Map<Integer, List<IndexMetaFileGroup>> extraIndexMetas = new
HashMap<>();
+ this.coverageByField = new HashMap<>();
for (IndexFileMeta indexFile : indexFiles) {
GlobalIndexMeta meta = checkNotNull(indexFile.globalIndexMeta());
String indexType = indexFile.indexType();
Range range = new Range(meta.rowRangeStart(), meta.rowRangeEnd());
int indexFieldId = meta.indexFieldId();
+ coverageByField.computeIfAbsent(indexFieldId, k -> new
ArrayList<>()).add(range);
+ if (meta.extraFieldIds() != null) {
+ for (int extra : meta.extraFieldIds()) {
+ coverageByField.computeIfAbsent(extra, k -> new
ArrayList<>()).add(range);
+ }
+ }
List<Integer> fieldIds = meta.getIndexedFieldIds();
IndexMetaFileGroup group = indexMetas.get(indexFieldId);
if (group == null) {
@@ -172,6 +199,9 @@ public class GlobalIndexScanner implements Closeable {
}
return Optional.of(
new GlobalIndexScanner(
+ null,
+ null,
+ null,
table.coreOptions().toConfiguration(),
table.rowType(),
table.fileIO(),
@@ -181,11 +211,33 @@ public class GlobalIndexScanner implements Closeable {
public static Optional<GlobalIndexScanner> create(
FileStoreTable table, PartitionPredicate partitionFilter,
Predicate filter) {
- Set<Integer> filterFieldIds =
- collectFieldNames(filter).stream()
- .filter(name -> table.rowType().containsField(name))
- .map(name -> table.rowType().getField(name).id())
- .collect(Collectors.toSet());
+ Snapshot snapshot = tryTravelOrLatest(table);
+ List<IndexFileMeta> indexFiles =
+ table.store().newIndexFileHandler()
+ .scan(snapshot, indexFileFilter(table,
partitionFilter, filter)).stream()
+ .map(IndexManifestEntry::indexFile)
+ .collect(Collectors.toList());
+ if (indexFiles.isEmpty()) {
+ return Optional.empty();
+ }
+ return Optional.of(
+ new GlobalIndexScanner(
+ table,
+ snapshot,
+ partitionFilter,
+ table.coreOptions().toConfiguration(),
+ table.rowType(),
+ table.fileIO(),
+ table.store().pathFactory().globalIndexFileFactory(),
+ indexFiles));
+ }
+
+ private static Filter<IndexManifestEntry> indexFileFilter(
+ FileStoreTable table, PartitionPredicate partitionFilter,
Predicate filter) {
+ if (filter == null) {
+ return entry -> false;
+ }
+ Set<Integer> filterFieldIds = collectFieldIds(table.rowType(), filter);
Filter<IndexManifestEntry> indexFileFilter =
entry -> {
if (partitionFilter != null &&
!partitionFilter.test(entry.partition())) {
@@ -209,17 +261,82 @@ public class GlobalIndexScanner implements Closeable {
}
return false;
};
-
- List<IndexFileMeta> indexFiles =
-
table.store().newIndexFileHandler().scan(tryTravelOrLatest(table),
indexFileFilter)
- .stream()
- .map(IndexManifestEntry::indexFile)
- .collect(Collectors.toList());
- return create(table, indexFiles);
+ return indexFileFilter;
}
public Optional<GlobalIndexResult> scan(Predicate predicate) {
- return globalIndexEvaluator.evaluate(predicate);
+ Optional<GlobalIndexResult> result =
globalIndexEvaluator.evaluate(predicate);
+ return result.map(indexedResultRows -> withUnindexedRows(predicate,
indexedResultRows));
+ }
+
+ private GlobalIndexResult withUnindexedRows(
+ Predicate predicate, GlobalIndexResult indexedResultRows) {
+ if (indexedResultRows instanceof ScoredGlobalIndexResult
+ || table == null
+ || table.coreOptions().globalIndexSearchMode() ==
GlobalIndexSearchMode.FAST) {
+ return indexedResultRows;
+ }
+
+ RoaringNavigableMap64 rows = new RoaringNavigableMap64();
+ rows.or(indexedResultRows.results());
+ for (Range range : unindexedRanges(predicate)) {
+ rows.addRange(range);
+ }
+ return GlobalIndexResult.create(rows);
+ }
+
+ private List<Range> indexedRanges(Predicate predicate) {
+ List<Range> ranges = null;
+ for (Integer fieldId : collectFieldIds(rowType, predicate)) {
+ List<Range> fieldRanges = coverageByField.get(fieldId);
+ if (fieldRanges == null || fieldRanges.isEmpty()) {
+ return Collections.emptyList();
+ }
+ fieldRanges = Range.sortAndMergeOverlap(fieldRanges, true);
+ ranges = ranges == null ? fieldRanges : Range.and(ranges,
fieldRanges);
+ }
+ return ranges == null ? Collections.emptyList() :
Range.sortAndMergeOverlap(ranges, true);
+ }
+
+ private List<Range> unindexedRanges(Predicate predicate) {
+ if (snapshot == null || snapshot.nextRowId() == null ||
snapshot.nextRowId() <= 0) {
+ return Collections.emptyList();
+ }
+
+ List<Range> dataRanges;
+ if (table.coreOptions().globalIndexSearchMode() ==
GlobalIndexSearchMode.DETAIL) {
+ dataRanges = dataRangesByDataFiles();
+ } else {
+ dataRanges = Collections.singletonList(new Range(0,
snapshot.nextRowId() - 1));
+ }
+
+ List<Range> predicateIndexedRanges =
+ Range.sortAndMergeOverlap(indexedRanges(predicate), true);
+ List<Range> unindexedRanges = new ArrayList<>();
+ for (Range dataRange : Range.sortAndMergeOverlap(dataRanges, true)) {
+ unindexedRanges.addAll(dataRange.exclude(predicateIndexedRanges));
+ }
+ return Range.sortAndMergeOverlap(unindexedRanges, true);
+ }
+
+ private List<Range> dataRangesByDataFiles() {
+ SnapshotReader snapshotReader =
+ table.newSnapshotReader()
+ .withPartitionFilter(partitionFilter)
+ .withMode(ScanMode.ALL)
+ .withSnapshot(snapshot);
+ List<Range> dataRanges = new ArrayList<>();
+ for (Split split : snapshotReader.read().splits()) {
+ if (!(split instanceof DataSplit)) {
+ continue;
+ }
+ for (DataFileMeta file : ((DataSplit) split).dataFiles()) {
+ if (file.firstRowId() != null) {
+ dataRanges.add(file.nonNullRowIdRange());
+ }
+ }
+ }
+ return dataRanges;
}
private Collection<GlobalIndexReader> createReaders(
diff --git
a/paimon-core/src/test/java/org/apache/paimon/table/BtreeGlobalIndexTableTest.java
b/paimon-core/src/test/java/org/apache/paimon/table/BtreeGlobalIndexTableTest.java
index 3fcd8f2ae4..8d963b11e9 100644
---
a/paimon-core/src/test/java/org/apache/paimon/table/BtreeGlobalIndexTableTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/table/BtreeGlobalIndexTableTest.java
@@ -18,6 +18,7 @@
package org.apache.paimon.table;
+import org.apache.paimon.CoreOptions;
import org.apache.paimon.data.BinaryString;
import org.apache.paimon.data.GenericRow;
import org.apache.paimon.globalindex.DataEvolutionBatchScan;
@@ -36,6 +37,7 @@ import org.apache.paimon.table.sink.CommitMessage;
import org.apache.paimon.table.source.DataSplit;
import org.apache.paimon.table.source.ReadBuilder;
import org.apache.paimon.table.source.Split;
+import org.apache.paimon.table.source.TableScan;
import org.apache.paimon.types.DataTypes;
import org.apache.paimon.utils.Range;
import org.apache.paimon.utils.RoaringNavigableMap64;
@@ -44,6 +46,7 @@ import org.junit.jupiter.api.Test;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import java.util.List;
import java.util.stream.Collectors;
@@ -130,6 +133,73 @@ public class BtreeGlobalIndexTableTest extends
DataEvolutionTestBase {
assertThat(readF1).containsExactly("a200", "a300", "a400", "a56789");
}
+ @Test
+ public void testBTreeGlobalIndexSearchModeControlsUnindexedData() throws
Exception {
+ write(500L);
+ createIndex("f1");
+ appendRows(500, 1000);
+
+ FileStoreTable table = (FileStoreTable) catalog.getTable(identifier());
+ Predicate predicate =
+ new PredicateBuilder(table.rowType())
+ .in(
+ 1,
+ Arrays.asList(
+ BinaryString.fromString("a100"),
+ BinaryString.fromString("a700")));
+
+ assertThat(readF1(table, predicate)).containsExactly("a100");
+
+ assertThat(readF1(tableWithSearchMode(table, "full"), predicate))
+ .containsExactly("a100", "a700");
+ assertThat(readF1(tableWithSearchMode(table, "detail"), predicate))
+ .containsExactly("a100", "a700");
+
+ PredicateBuilder builder = new PredicateBuilder(table.rowType());
+ Predicate andWithUnindexedField =
+ PredicateBuilder.and(
+ builder.equal(1, BinaryString.fromString("a700")),
+ builder.equal(2, BinaryString.fromString("b700")));
+
+ assertThat(readF1(table, andWithUnindexedField)).isEmpty();
+ assertThat(readF1(tableWithSearchMode(table, "full"),
andWithUnindexedField))
+ .containsExactly("a700");
+ assertThat(readF1(tableWithSearchMode(table, "detail"),
andWithUnindexedField))
+ .containsExactly("a700");
+ }
+
+ @Test
+ public void testBTreeGlobalIndexSearchModeUsesAllPredicateFieldCoverage()
throws Exception {
+ write(500L);
+ createIndex("f1");
+ appendRows(500, 1000);
+ createIndex("f2");
+
+ FileStoreTable table = (FileStoreTable) catalog.getTable(identifier());
+ PredicateBuilder builder = new PredicateBuilder(table.rowType());
+ Predicate andPredicate =
+ PredicateBuilder.and(
+ builder.equal(1, BinaryString.fromString("a700")),
+ builder.equal(2, BinaryString.fromString("b700")));
+
+ assertThat(readF1(table, andPredicate)).isEmpty();
+ assertThat(readF1(tableWithSearchMode(table, "full"), andPredicate))
+ .containsExactly("a700");
+ assertThat(readF1(tableWithSearchMode(table, "detail"), andPredicate))
+ .containsExactly("a700");
+
+ Predicate orPredicate =
+ PredicateBuilder.or(
+ builder.equal(1, BinaryString.fromString("a700")),
+ builder.equal(2, BinaryString.fromString("b701")));
+
+ assertThat(readF1(table, orPredicate)).containsExactly("a701");
+ assertThat(readF1(tableWithSearchMode(table, "full"), orPredicate))
+ .containsExactly("a700", "a701");
+ assertThat(readF1(tableWithSearchMode(table, "detail"), orPredicate))
+ .containsExactly("a700", "a701");
+ }
+
@Test
public void testMultipleBTreeIndices() throws Exception {
write(100000L);
@@ -237,6 +307,42 @@ public class BtreeGlobalIndexTableTest extends
DataEvolutionTestBase {
.collect(Collectors.toList());
}
+ private List<String> readF1(ReadBuilder readBuilder, TableScan.Plan plan)
throws Exception {
+ List<String> readF1 = new ArrayList<>();
+ readBuilder
+ .newRead()
+ .executeFilter()
+ .createReader(plan)
+ .forEachRemaining(row ->
readF1.add(row.getString(1).toString()));
+ return readF1;
+ }
+
+ private List<String> readF1(FileStoreTable table, Predicate predicate)
throws Exception {
+ ReadBuilder readBuilder = table.newReadBuilder().withFilter(predicate);
+ return readF1(readBuilder, readBuilder.newScan().plan());
+ }
+
+ private FileStoreTable tableWithSearchMode(FileStoreTable table, String
searchMode) {
+ return table.copy(
+
Collections.singletonMap(CoreOptions.GLOBAL_INDEX_SEARCH_MODE.key(),
searchMode));
+ }
+
+ private void appendRows(int fromInclusive, int toExclusive) throws
Exception {
+ FileStoreTable table = (FileStoreTable) catalog.getTable(identifier());
+ BatchWriteBuilder writeBuilder = table.newBatchWriteBuilder();
+ try (BatchTableWrite write = writeBuilder.newWrite();
+ BatchTableCommit commit = writeBuilder.newCommit()) {
+ for (int i = fromInclusive; i < toExclusive; i++) {
+ write.write(
+ GenericRow.of(
+ i,
+ BinaryString.fromString("a" + i),
+ BinaryString.fromString("b" + i)));
+ }
+ commit.commit(write.prepareCommit());
+ }
+ }
+
private RoaringNavigableMap64 globalIndexScan(FileStoreTable table,
Predicate predicate)
throws Exception {
try (GlobalIndexScanner scanner =
