This is an automated email from the ASF dual-hosted git repository.
junhao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 7fadf8b9c0 [core] Fix non-indexed row range planning for btree global
index (#7505)
7fadf8b9c0 is described below
commit 7fadf8b9c0bc4fc81f0719f3da8bd49b5870ca49
Author: YeJunHao <[email protected]>
AuthorDate: Tue Mar 24 11:34:34 2026 +0800
[core] Fix non-indexed row range planning for btree global index (#7505)
---
.../paimon/globalindex/DataEvolutionBatchScan.java | 2 +-
.../paimon/table/BtreeGlobalIndexTableTest.java | 84 +++++++++++++++++++++-
2 files changed, 83 insertions(+), 3 deletions(-)
diff --git
a/paimon-core/src/main/java/org/apache/paimon/globalindex/DataEvolutionBatchScan.java
b/paimon-core/src/main/java/org/apache/paimon/globalindex/DataEvolutionBatchScan.java
index 13b4b59345..f2b2c70b1b 100644
---
a/paimon-core/src/main/java/org/apache/paimon/globalindex/DataEvolutionBatchScan.java
+++
b/paimon-core/src/main/java/org/apache/paimon/globalindex/DataEvolutionBatchScan.java
@@ -291,7 +291,7 @@ public class DataEvolutionBatchScan implements
DataTableScan {
GlobalIndexResult result = resultOptional.get();
if (!nonIndexedRowRanges.isEmpty()) {
for (Range range : nonIndexedRowRanges) {
- result.or(GlobalIndexResult.fromRange(range));
+ result = result.or(GlobalIndexResult.fromRange(range));
}
}
diff --git
a/paimon-core/src/test/java/org/apache/paimon/table/BtreeGlobalIndexTableTest.java
b/paimon-core/src/test/java/org/apache/paimon/table/BtreeGlobalIndexTableTest.java
index 90d4e473c1..f48fcfd0b1 100644
---
a/paimon-core/src/test/java/org/apache/paimon/table/BtreeGlobalIndexTableTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/table/BtreeGlobalIndexTableTest.java
@@ -19,17 +19,24 @@
package org.apache.paimon.table;
import org.apache.paimon.data.BinaryString;
+import org.apache.paimon.data.GenericRow;
import org.apache.paimon.globalindex.DataEvolutionBatchScan;
import org.apache.paimon.globalindex.GlobalIndexResult;
import org.apache.paimon.globalindex.GlobalIndexScanBuilder;
+import org.apache.paimon.globalindex.IndexedSplit;
import org.apache.paimon.globalindex.RowRangeGlobalIndexScanner;
import org.apache.paimon.globalindex.btree.BTreeGlobalIndexBuilder;
+import org.apache.paimon.io.DataFileMeta;
import org.apache.paimon.predicate.Predicate;
import org.apache.paimon.predicate.PredicateBuilder;
import org.apache.paimon.table.sink.BatchTableCommit;
+import org.apache.paimon.table.sink.BatchTableWrite;
+import org.apache.paimon.table.sink.BatchWriteBuilder;
import org.apache.paimon.table.sink.CommitMessage;
import org.apache.paimon.table.source.DataSplit;
import org.apache.paimon.table.source.ReadBuilder;
+import org.apache.paimon.table.source.Split;
+import org.apache.paimon.types.RowType;
import org.apache.paimon.utils.Range;
import org.apache.paimon.utils.RoaringNavigableMap64;
@@ -37,8 +44,10 @@ import org.junit.jupiter.api.Test;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import java.util.List;
import java.util.Optional;
+import java.util.stream.Collectors;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertNotNull;
@@ -163,13 +172,54 @@ public class BtreeGlobalIndexTableTest extends
DataEvolutionTestBase {
assertThat(result).containsExactly("a200", "a56789");
}
+ @Test
+ public void testBtreeWithNonIndexedRowRange() throws Exception {
+ write(10L);
+ append(0, 10);
+
+ FileStoreTable table = (FileStoreTable) catalog.getTable(identifier());
+ createIndex("f1", Collections.singletonList(new Range(0L, 9L)));
+
+ assertThat(table.store().newGlobalIndexScanBuilder().shardList())
+ .containsExactly(new Range(0L, 9L));
+
+ Predicate predicate =
+ new PredicateBuilder(table.rowType()).equal(1,
BinaryString.fromString("a5"));
+ ReadBuilder readBuilder = table.newReadBuilder().withFilter(predicate);
+
+ List<Split> splits = readBuilder.newScan().plan().splits();
+ assertThat(splits).hasSize(1);
+
+ IndexedSplit indexedSplit = (IndexedSplit) splits.get(0);
+ assertThat(indexedSplit.rowRanges())
+ .containsExactly(new Range(5L, 5L), new Range(10L, 19L));
+ assertThat(
+ indexedSplit.dataSplit().dataFiles().stream()
+ .map(DataFileMeta::firstRowId)
+ .distinct()
+ .sorted()
+ .collect(Collectors.toList()))
+ .containsExactly(0L, 10L);
+
+ List<String> result = new ArrayList<>();
+ readBuilder
+ .newRead()
+ .createReader(splits)
+ .forEachRemaining(row ->
result.add(row.getString(1).toString()));
+ assertThat(result)
+ .containsExactly("a5", "a0", "a1", "a2", "a3", "a4", "a5",
"a6", "a7", "a8", "a9");
+ }
+
private void createIndex(String fieldName) throws Exception {
+ createIndex(fieldName, null);
+ }
+
+ private void createIndex(String fieldName, List<Range> rowRanges) throws
Exception {
FileStoreTable table = (FileStoreTable) catalog.getTable(identifier());
BTreeGlobalIndexBuilder builder =
new BTreeGlobalIndexBuilder(table).withIndexField(fieldName);
- List<DataSplit> dataSplits = builder.scan();
List<CommitMessage> commitMessages = new ArrayList<>();
- for (DataSplit dataSplit : dataSplits) {
+ for (DataSplit dataSplit : indexSplits(table, rowRanges,
builder.scan())) {
commitMessages.addAll(builder.build(dataSplit, ioManager));
}
try (BatchTableCommit commit =
table.newBatchWriteBuilder().newCommit()) {
@@ -177,6 +227,36 @@ public class BtreeGlobalIndexTableTest extends
DataEvolutionTestBase {
}
}
+ private List<DataSplit> indexSplits(
+ FileStoreTable table, List<Range> rowRanges, List<DataSplit>
fallbackSplits) {
+ if (rowRanges == null) {
+ return fallbackSplits;
+ }
+
+ List<Split> splits =
+
table.newReadBuilder().withRowRanges(rowRanges).newScan().plan().splits();
+ return splits.stream()
+ .map(split -> ((IndexedSplit) split).dataSplit())
+ .collect(Collectors.toList());
+ }
+
+ private void append(int startInclusive, int endExclusive) throws Exception
{
+ BatchWriteBuilder builder = getTableDefault().newBatchWriteBuilder();
+ RowType writeType = schemaDefault().rowType();
+ try (BatchTableWrite write0 =
builder.newWrite().withWriteType(writeType)) {
+ for (int i = startInclusive; i < endExclusive; i++) {
+ write0.write(
+ GenericRow.of(
+ i,
+ BinaryString.fromString("a" + i),
+ BinaryString.fromString("b" + i)));
+ }
+ try (BatchTableCommit commit = builder.newCommit()) {
+ commit.commit(write0.prepareCommit());
+ }
+ }
+ }
+
private RoaringNavigableMap64 globalIndexScan(FileStoreTable table,
Predicate predicate)
throws Exception {
GlobalIndexScanBuilder indexScanBuilder =
table.store().newGlobalIndexScanBuilder();
