This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new c77405ea58 [core] Skip TopN pushdown when a non-partition filter is
present (#8458)
c77405ea58 is described below
commit c77405ea58944d7d6ef7eac442605b8671d6ea4e
Author: Jiajia Li <[email protected]>
AuthorDate: Sat Jul 4 21:00:10 2026 +0800
[core] Skip TopN pushdown when a non-partition filter is present (#8458)
DataTableBatchScan.applyPushDownTopN prunes splits by the sort column's
min/max stats, but a non-partition WHERE filter drops rows at read time,
after pruning. A split kept for its stats may have its top rows filtered
out, while a split holding the true post-filter top-N may be pruned and
never read — so ORDER BY ... LIMIT can return wrong/missing rows.
Skip TopN pushdown when snapshotReader.hasNonPartitionFilter(),
mirroring applyPushDownLimit (which already guards both this and the
auth filter). Partition-only filters are unaffected, since all surviving
rows pass and stats-based pruning stays valid.
---
.../paimon/table/source/DataTableBatchScan.java | 4 +++-
.../org/apache/paimon/table/source/TableScanTest.java | 19 +++++++++++++++++++
2 files changed, 22 insertions(+), 1 deletion(-)
diff --git
a/paimon-core/src/main/java/org/apache/paimon/table/source/DataTableBatchScan.java
b/paimon-core/src/main/java/org/apache/paimon/table/source/DataTableBatchScan.java
index 7d528db6c3..634c95e6ea 100644
---
a/paimon-core/src/main/java/org/apache/paimon/table/source/DataTableBatchScan.java
+++
b/paimon-core/src/main/java/org/apache/paimon/table/source/DataTableBatchScan.java
@@ -188,9 +188,11 @@ public class DataTableBatchScan extends
AbstractDataTableScan {
}
private Optional<StartingScanner.Result> applyPushDownTopN() {
- // Auth drops rows at read time, so split-level TopN pruning could
drop authorized rows.
+ // A read-time filter (WHERE or auth) drops rows after split pruning,
so split-level TopN
+ // pruning could keep too few splits. Skip it when either is present.
if (topN == null
|| pushDownLimit != null
+ || snapshotReader.hasNonPartitionFilter()
|| authHasNonPartitionFilter
|| !schema.primaryKeys().isEmpty()) {
return Optional.empty();
diff --git
a/paimon-core/src/test/java/org/apache/paimon/table/source/TableScanTest.java
b/paimon-core/src/test/java/org/apache/paimon/table/source/TableScanTest.java
index 6c25574b02..cd417058a8 100644
---
a/paimon-core/src/test/java/org/apache/paimon/table/source/TableScanTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/table/source/TableScanTest.java
@@ -421,6 +421,25 @@ public class TableScanTest extends ScannerTestBase {
.isEqualTo(60);
assertThat(((DataSplit) splits2.get(2)).nullCount(field.id(),
evolutions)).isEqualTo(2);
+ // A non-partition filter must disable TopN pushdown, else it prunes
splits by sort-column
+ // stats unaware the filter removes rows. "b" >= 0 matches every row,
so all 9 splits are
+ // kept instead of pruned to 3.
+ PredicateBuilder builder = new PredicateBuilder(table.rowType());
+ TableScan.Plan planWithFilter =
+ table.newScan()
+ .withFilter(builder.greaterOrEqual(2, 0L))
+ .withTopN(new TopN(ref, ASCENDING, NULLS_FIRST, 1))
+ .plan();
+ assertThat(planWithFilter.splits().size()).isEqualTo(9);
+
+ // A partition-only filter keeps TopN pushdown enabled (surviving rows
all pass): still 3.
+ TableScan.Plan planWithPartitionFilter =
+ table.newScan()
+ .withFilter(builder.greaterOrEqual(0, 0))
+ .withTopN(new TopN(ref, ASCENDING, NULLS_FIRST, 1))
+ .plan();
+ assertThat(planWithPartitionFilter.splits().size()).isEqualTo(3);
+
// with bottom1 null last
TableScan.Plan plan3 =
table.newScan().withTopN(new TopN(ref, ASCENDING, NULLS_LAST,
1)).plan();
