(paimon) branch master updated: [core] Fix dropping stats in data evolution scan (#8994)

[lzljs3620320]

This is an automated email from the ASF dual-hosted git repository.

JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git


The following commit(s) were added to refs/heads/master by this push:
     new 9d3d6e058d [core] Fix dropping stats in data evolution scan (#8994)
9d3d6e058d is described below

commit 9d3d6e058dbe334ab3d29f92c9b3415b871f51f7
Author: YeJunHao <41894543+leaves12...@users.noreply.github.com>
AuthorDate: Wed Aug 5 13:18:16 2026 +0800

    [core] Fix dropping stats in data evolution scan (#8994)
---
 .../operation/DataEvolutionFileStoreScan.java      | 31 ++++++++++++----------
 .../paimon/table/DataEvolutionTableTest.java       | 21 ++++++++++-----
 2 files changed, 32 insertions(+), 20 deletions(-)

diff --git 
a/paimon-core/src/main/java/org/apache/paimon/operation/DataEvolutionFileStoreScan.java
 
b/paimon-core/src/main/java/org/apache/paimon/operation/DataEvolutionFileStoreScan.java
index 06cde74f1f..66f1aadbee 100644
--- 
a/paimon-core/src/main/java/org/apache/paimon/operation/DataEvolutionFileStoreScan.java
+++ 
b/paimon-core/src/main/java/org/apache/paimon/operation/DataEvolutionFileStoreScan.java
@@ -181,24 +181,27 @@ public class DataEvolutionFileStoreScan extends 
AppendOnlyFileStoreScan {
 
     @Override
     protected boolean postFilterManifestEntriesEnabled() {
-        // Always enable post-filtering. The list filterByStats handles 
predicate-based pruning
-        // and pruneByReadType strips per-file columns that are not requested 
— both
-        // need row-id-range grouping that single filterByStats(ManifestEntry) 
cannot see.
-        return inputFilter != null || readType != null;
+        return true;
     }
 
     @Override
     protected List<ManifestEntry> 
postFilterManifestEntries(List<ManifestEntry> entries) {
-        // group by row id range
-        RangeHelper<ManifestEntry> rangeHelper =
-                new RangeHelper<>(e -> e.file().nonNullRowIdRange());
-        List<List<ManifestEntry>> splitByRowId = 
rangeHelper.mergeOverlappingRanges(entries);
-
-        return splitByRowId.stream()
-                .filter(group -> inputFilter == null || filterByStats(group))
-                .flatMap(group -> pruneByReadType(group).stream())
-                .map(entry -> dropStats ? dropStats(entry) : entry)
-                .collect(Collectors.toList());
+        if (inputFilter != null || readType != null) {
+            // group by row id range
+            RangeHelper<ManifestEntry> rangeHelper =
+                    new RangeHelper<>(e -> e.file().nonNullRowIdRange());
+            List<List<ManifestEntry>> splitByRowId = 
rangeHelper.mergeOverlappingRanges(entries);
+
+            return splitByRowId.stream()
+                    .filter(group -> inputFilter == null || 
filterByStats(group))
+                    .flatMap(group -> pruneByReadType(group).stream())
+                    .map(entry -> dropStats ? dropStats(entry) : entry)
+                    .collect(Collectors.toList());
+        } else if (dropStats) {
+            return 
entries.stream().map(this::dropStats).collect(Collectors.toList());
+        } else {
+            return entries;
+        }
     }
 
     private boolean filterByStats(List<ManifestEntry> entries) {
diff --git 
a/paimon-core/src/test/java/org/apache/paimon/table/DataEvolutionTableTest.java 
b/paimon-core/src/test/java/org/apache/paimon/table/DataEvolutionTableTest.java
index 4f080f9667..592dcf687c 100644
--- 
a/paimon-core/src/test/java/org/apache/paimon/table/DataEvolutionTableTest.java
+++ 
b/paimon-core/src/test/java/org/apache/paimon/table/DataEvolutionTableTest.java
@@ -70,6 +70,7 @@ import java.util.OptionalLong;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.stream.Collectors;
 
+import static org.apache.paimon.stats.SimpleStats.EMPTY_STATS;
 import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
 
 /** Test for table with data evolution. */
@@ -2096,12 +2097,7 @@ public class DataEvolutionTableTest extends 
DataEvolutionTestBase {
         assertThat(plannedFileCount(table, readF2, null)).isEqualTo(2);
     }
 
-    /**
-     * System-field-only projection is filtered out of readType in
-     * DataEvolutionFileStoreScan.withReadType — readType stays null and
-     * postFilterManifestEntriesEnabled returns false. The column-pruning path 
is not entered, so
-     * every file in every group flows through unchanged.
-     */
+    /** System-field-only projection is not used for per-file column pruning. 
*/
     @Test
     public void testSystemFieldOnlyProjectionIsNotPruned() throws Exception {
         write(5);
@@ -2110,6 +2106,19 @@ public class DataEvolutionTableTest extends 
DataEvolutionTestBase {
         assertThat(plannedFileCount(table, RowType.of(SpecialFields.ROW_ID), 
null)).isEqualTo(2);
     }
 
+    @Test
+    public void testDropStatsWithoutFilterOrReadType() throws Exception {
+        write(5);
+
+        List<ManifestEntry> entries =
+                getTableDefault().store().newScan().dropStats().plan().files();
+
+        assertThat(entries.isEmpty()).isFalse();
+        for (ManifestEntry entry : entries) {
+            assertThat(entry.file().valueStats()).isEqualTo(EMPTY_STATS);
+        }
+    }
+
     private List<DataFileMeta> 
writeOneFullRowAndCollectNewFiles(FileStoreTable table)
             throws Exception {
         Schema schema = schemaDefault();