This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 5ef21b7216 [core] Skip DV reads for file-index-rejected single files
(#10287)
5ef21b7216 is described below
commit 5ef21b7216e38042292c2a3c913a50d5b7974038
Author: sanshi <[email protected]>
AuthorDate: Fri Oct 2 09:45:54 2026 +0800
[core] Skip DV reads for file-index-rejected single files (#10287)
---
.../paimon/operation/DataEvolutionSplitRead.java | 76 ++++++++++++++++++++--
.../paimon/table/DataEvolutionFileIndexTest.java | 22 +++++++
2 files changed, 94 insertions(+), 4 deletions(-)
diff --git
a/paimon-core/src/main/java/org/apache/paimon/operation/DataEvolutionSplitRead.java
b/paimon-core/src/main/java/org/apache/paimon/operation/DataEvolutionSplitRead.java
index 60902d352a..470a15f4a5 100644
---
a/paimon-core/src/main/java/org/apache/paimon/operation/DataEvolutionSplitRead.java
+++
b/paimon-core/src/main/java/org/apache/paimon/operation/DataEvolutionSplitRead.java
@@ -236,16 +236,39 @@ public class DataEvolutionSplitRead implements
SplitRead<InternalRow> {
// No need to merge fields, just create a single file reader
suppliers.add(
() -> {
+ DataFileMeta file = needMergeFiles.get(0);
+ FileIndexResult fileIndexResult =
+ evaluateFileIndex(filters, file,
dataFilePathFactory);
+ if (fileIndexResult != null &&
!fileIndexResult.remain()) {
+ return new EmptyFileRecordReader<>();
+ }
DeletionVectorWithRange deletionVector =
readDeletionVector(needMergeFiles,
deletionVectorFactory);
+ if (fileIndexResult != null
+ && deletionVector != null
+ &&
!deletionVector.deletionVector.isEmpty()) {
+ fileIndexResult =
+
FileIndexEvaluator.intersectDeletionVector(
+ fileIndexResult,
+ file,
+ deletionVector.deletionVector,
+ deletionVectorOffset(
+
file.nonNullRowIdRange(),
+ rowRanges,
+ deletionVector));
+ if (!fileIndexResult.remain()) {
+ return new EmptyFileRecordReader<>();
+ }
+ }
return createFileReader(
partition,
dataFilePathFactory,
- needMergeFiles.get(0),
+ file,
filters,
rowRanges,
readRowType,
- deletionVector);
+ deletionVector,
+ fileIndexResult);
});
} else {
@@ -675,6 +698,27 @@ public class DataEvolutionSplitRead implements
SplitRead<InternalRow> {
RowType readRowType,
@Nullable DeletionVectorWithRange deletionVector)
throws IOException {
+ return createFileReader(
+ partition,
+ dataFilePathFactory,
+ file,
+ filters,
+ rowRanges,
+ readRowType,
+ deletionVector,
+ null);
+ }
+
+ private FileRecordReader<InternalRow> createFileReader(
+ BinaryRow partition,
+ DataFilePathFactory dataFilePathFactory,
+ DataFileMeta file,
+ @Nullable List<Predicate> filters,
+ List<Range> rowRanges,
+ RowType readRowType,
+ @Nullable DeletionVectorWithRange deletionVector,
+ @Nullable FileIndexResult precomputedFileIndexResult)
+ throws IOException {
FileReadTarget readTarget = readTarget(file, dataFilePathFactory,
rowRanges);
String formatIdentifier = readTarget.formatIdentifier;
long schemaId = file.schemaId();
@@ -696,8 +740,8 @@ public class DataEvolutionSplitRead implements
SplitRead<InternalRow> {
formatBuilder(readRowType, fileFilters,
nestedFieldEnabled)
.build(formatIdentifier, schema,
dataSchema));
- FileIndexResult fileIndexResult = null;
- if (fileIndexReadEnabled) {
+ FileIndexResult fileIndexResult = precomputedFileIndexResult;
+ if (fileIndexResult == null && fileIndexReadEnabled) {
DeletionVector dv = deletionVector == null ? null :
deletionVector.deletionVector;
long fileOffset =
dv == null || dv.isEmpty()
@@ -870,6 +914,30 @@ public class DataEvolutionSplitRead implements
SplitRead<InternalRow> {
return true;
}
+ @Nullable
+ private FileIndexResult evaluateFileIndex(
+ @Nullable List<Predicate> filters,
+ DataFileMeta file,
+ DataFilePathFactory dataFilePathFactory)
+ throws IOException {
+ if (!fileIndexReadEnabled) {
+ return null;
+ }
+
+ long schemaId = file.schemaId();
+ TableSchema dataSchema = schemaId == schema.id() ? schema :
schemaFetcher.apply(schemaId);
+ List<Predicate> fileFilters = fileFilters(filters, file);
+ return FileIndexEvaluator.evaluate(
+ fileIO,
+ dataSchema,
+ devolveFilters(fileFilters, dataSchema),
+ null,
+ null,
+ dataFilePathFactory,
+ file,
+ null);
+ }
+
/**
* Evaluates each applicable file index once for the winning fields of a
merged row-id group.
*
diff --git
a/paimon-core/src/test/java/org/apache/paimon/table/DataEvolutionFileIndexTest.java
b/paimon-core/src/test/java/org/apache/paimon/table/DataEvolutionFileIndexTest.java
index c350c424f6..e323817166 100644
---
a/paimon-core/src/test/java/org/apache/paimon/table/DataEvolutionFileIndexTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/table/DataEvolutionFileIndexTest.java
@@ -556,6 +556,28 @@ public class DataEvolutionFileIndexTest extends
DataEvolutionTestBase {
assertThat(rowIds(rows)).containsExactly(50L);
}
+ @Test
+ public void testSingleFileIndexSkipsBeforeReadingDeletionVector() throws
Exception {
+ Map<String, String> options = bitmapOptions("f1");
+ options.put(CoreOptions.DELETION_VECTORS_ENABLED.key(), "true");
+ FileStoreTable table = createTable("single_bitmap_before_dv", options);
+ writeAllColumns(table, ROW_COUNT);
+ deleteRows(table, 50);
+
+ FileStoreTable latest = getTable(identifier(table.name()));
+ DataSplit split = (DataSplit)
latest.newReadBuilder().newScan().plan().splits().get(0);
+ Path deletionVectorPath =
+ split.deletionFiles().get().stream()
+ .filter(Objects::nonNull)
+ .map(file -> new Path(file.path()))
+ .findFirst()
+ .orElseThrow(IllegalStateException::new);
+ assertThat(latest.fileIO().delete(deletionVectorPath, false)).isTrue();
+
+ // The bitmap index already rejects this value, so the missing DV file
must not be read.
+ assertThat(readWithFilter(table, equalF1(MISSING_F1))).isEmpty();
+ }
+
@Test
public void testMergedGroupKeptWhenFilterColumnOverwritten() throws
Exception {
FileStoreTable table = createTable("overwritten",
Collections.emptyMap());
