This is an automated email from the ASF dual-hosted git repository.
BiteTheDDDDt pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new f45e0931636 [fix](runtime-filter) Preserve query semantics during
partition pruning (#65857)
f45e0931636 is described below
commit f45e0931636f56ded31e332302423356837f840a
Author: Pxl <[email protected]>
AuthorDate: Wed Jul 22 20:16:34 2026 +0800
[fix](runtime-filter) Preserve query semantics during partition pruning
(#65857)
With runtime filter partition pruning enabled, a join target expression
was eligible for partition pruning even when it contained a non-movable
function such as `assert_true`. The pruning path then evaluated
`assert_true` against LIST partition boundary values before normal row
predicates ran. A boundary belonging only to filtered-out rows could
therefore introduce an `INVALID_ARGUMENT` error, while the same query
returned `1` with partition pruning disabled.
The FE runtime-filter partition-prune classifier now rejects target
expressions containing `NoneMovableFunction`. This prevents speculative
partition-boundary evaluation of `assert_true`; both partition-pruning
modes preserve normal query semantics and return `1`.
---
.../RuntimeFilterPartitionPruneClassifier.java | 4 ++
.../RuntimeFilterPartitionPruneClassifierTest.java | 37 ++++++++++++++-
.../runtime_filter/rf_partition_pruning.out | 6 +++
.../runtime_filter/rf_partition_pruning.groovy | 53 ++++++++++++++++++++++
4 files changed, 99 insertions(+), 1 deletion(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/RuntimeFilterPartitionPruneClassifier.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/RuntimeFilterPartitionPruneClassifier.java
index b1dd098324e..00c61916329 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/RuntimeFilterPartitionPruneClassifier.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/RuntimeFilterPartitionPruneClassifier.java
@@ -33,6 +33,7 @@ import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.functions.Monotonic;
+import
org.apache.doris.nereids.trees.expressions.functions.NoneMovableFunction;
import org.apache.doris.nereids.trees.expressions.literal.Literal;
import org.apache.doris.planner.OlapScanNode;
import org.apache.doris.planner.PlanNode;
@@ -79,6 +80,9 @@ final class RuntimeFilterPartitionPruneClassifier {
if (hasUnsupportedAutomaticPartitionExpression(partitionInfo)) {
return Classification.unsupported("automatic partition expression
boundary is not modeled");
}
+ if (nereidsTargetExpr.containsType(NoneMovableFunction.class)) {
+ return Classification.unsupported("target expression contains
non-movable function");
+ }
if (targetExpr instanceof SlotRef) {
SlotRef slotRef = (SlotRef) targetExpr;
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/RuntimeFilterPartitionPruneClassifierTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/RuntimeFilterPartitionPruneClassifierTest.java
index 34de78c7a76..58e32440364 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/RuntimeFilterPartitionPruneClassifierTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/RuntimeFilterPartitionPruneClassifierTest.java
@@ -17,9 +17,14 @@
package org.apache.doris.nereids.glue.translator;
+import org.apache.doris.analysis.BinaryPredicate;
+import org.apache.doris.analysis.Expr;
+import org.apache.doris.analysis.FunctionCallExpr;
+import org.apache.doris.analysis.IntLiteral;
import org.apache.doris.analysis.SlotDescriptor;
import org.apache.doris.analysis.SlotId;
import org.apache.doris.analysis.SlotRef;
+import org.apache.doris.analysis.StringLiteral;
import org.apache.doris.analysis.TupleId;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.ListPartitionItem;
@@ -29,9 +34,13 @@ import org.apache.doris.catalog.PartitionItem;
import org.apache.doris.catalog.PartitionType;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.RangePartitionItem;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.GreaterThan;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.expressions.functions.Monotonic;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.AssertTrue;
import org.apache.doris.nereids.trees.expressions.functions.scalar.DateTrunc;
+import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral;
import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral;
import org.apache.doris.nereids.types.DateTimeV2Type;
import org.apache.doris.nereids.types.IntegerType;
@@ -45,6 +54,7 @@ import org.junit.jupiter.api.Test;
import org.mockito.Mockito;
import java.util.Map;
+import java.util.function.Function;
class RuntimeFilterPartitionPruneClassifierTest {
@Test
@@ -94,8 +104,32 @@ class RuntimeFilterPartitionPruneClassifierTest {
assertSupportedIncreasingPartitions(classification);
}
+ @Test
+ void testRejectNoneMovableListTargetExpression() {
+ RuntimeFilterPartitionPruneClassifier.Classification classification =
classify(
+ TRuntimeFilterType.IN, PartitionType.LIST,
ListPartitionItem.DUMMY_ITEM,
+ targetSlot -> new FunctionCallExpr("assert_true",
ImmutableList.of(
+ new BinaryPredicate(BinaryPredicate.Operator.NE,
targetSlot, new IntLiteral(0)),
+ new StringLiteral("rfpp_expr_in_only_error")), false),
+ targetSlot -> new AssertTrue(
+ new GreaterThan(targetSlot, new IntegerLiteral(0)),
+ new VarcharLiteral("rfpp_expr_in_only_error")));
+
+ Assertions.assertFalse(classification.canPrunePartitions());
+
Assertions.assertTrue(classification.getUnsupportedReason().contains("non-movable"));
+
Assertions.assertTrue(classification.getPartitionMonotonicity().isEmpty());
+ }
+
private RuntimeFilterPartitionPruneClassifier.Classification classify(
TRuntimeFilterType filterType, PartitionType partitionType,
PartitionItem partitionItem) {
+ return classify(filterType, partitionType, partitionItem, targetSlot
-> targetSlot,
+ targetSlot -> targetSlot);
+ }
+
+ private RuntimeFilterPartitionPruneClassifier.Classification classify(
+ TRuntimeFilterType filterType, PartitionType partitionType,
PartitionItem partitionItem,
+ Function<SlotRef, Expr> legacyTargetFactory,
+ Function<SlotReference, Expression> nereidsTargetFactory) {
Column partitionColumn = new Column("part_col", PrimitiveType.INT);
SlotDescriptor slotDescriptor = new SlotDescriptor(new SlotId(1), new
TupleId(1));
slotDescriptor.setColumn(partitionColumn);
@@ -114,7 +148,8 @@ class RuntimeFilterPartitionPruneClassifierTest {
Mockito.when(partitionInfo.getItem(1L)).thenReturn(partitionItem);
Mockito.when(partitionInfo.getItem(2L)).thenReturn(partitionItem);
- return RuntimeFilterPartitionPruneClassifier.classify(filterType,
targetSlot, nereidsTarget, scanNode);
+ return RuntimeFilterPartitionPruneClassifier.classify(filterType,
+ legacyTargetFactory.apply(targetSlot),
nereidsTargetFactory.apply(nereidsTarget), scanNode);
}
private void assertSupportedIncreasingPartitions(
diff --git
a/regression-test/data/query_p0/runtime_filter/rf_partition_pruning.out
b/regression-test/data/query_p0/runtime_filter/rf_partition_pruning.out
index 44295ef30a1..3fe042541fb 100644
--- a/regression-test/data/query_p0/runtime_filter/rf_partition_pruning.out
+++ b/regression-test/data/query_p0/runtime_filter/rf_partition_pruning.out
@@ -136,5 +136,11 @@ Beijing 3
5 3 e
6 3 f
+-- !rf_prune_expr_without_partition_prune --
+1
+
+-- !rf_prune_expr_with_partition_prune --
+1
+
-- !list_str_bloom --
3 c
diff --git
a/regression-test/suites/query_p0/runtime_filter/rf_partition_pruning.groovy
b/regression-test/suites/query_p0/runtime_filter/rf_partition_pruning.groovy
index 1b76f964aac..4776c967cea 100644
--- a/regression-test/suites/query_p0/runtime_filter/rf_partition_pruning.groovy
+++ b/regression-test/suites/query_p0/runtime_filter/rf_partition_pruning.groovy
@@ -1479,6 +1479,59 @@ suite("rf_partition_pruning", "nonConcurrent") {
+ "ON f.region_id + f.region_id = d.dim_region",
"BLOOM_FILTER", 5, 3)
+ // Test 50b: NoneMovable target expressions must not be evaluated
speculatively
+ // against LIST partition values. The part_key=0 row is removed by
keep_row=1
+ // before the join, so assert_true must not fail only because RF partition
pruning
+ // evaluates the target expression on the p_zero boundary.
+ sql "drop table if exists rf_prune_expr_fact"
+ sql """
+ CREATE TABLE rf_prune_expr_fact (
+ id BIGINT NOT NULL,
+ part_key INT NOT NULL,
+ keep_row INT NOT NULL
+ ) DUPLICATE KEY(id)
+ PARTITION BY LIST(part_key) (
+ PARTITION p_zero VALUES IN (0),
+ PARTITION p_one VALUES IN (1)
+ )
+ DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES("replication_num" = "1")
+ """
+ sql "INSERT INTO rf_prune_expr_fact VALUES (0, 0, 0), (1, 1, 1)"
+
+ sql "drop table if exists rf_prune_expr_dim"
+ sql """
+ CREATE TABLE rf_prune_expr_dim (
+ flag BOOLEAN NOT NULL
+ ) DUPLICATE KEY(flag)
+ DISTRIBUTED BY HASH(flag) BUCKETS 1
+ PROPERTIES("replication_num" = "1")
+ """
+ sql "INSERT INTO rf_prune_expr_dim VALUES (true)"
+
+ sql "set enable_runtime_filter_partition_prune=false"
+ qt_rf_prune_expr_without_partition_prune """
+ SELECT /*+ SET_VAR(runtime_filter_type='IN') */ COUNT(*)
+ FROM rf_prune_expr_fact fact
+ JOIN [broadcast] rf_prune_expr_dim dim
+ ON assert_true(fact.part_key != 0, 'rfpp_expr_in_only_error') =
dim.flag
+ WHERE fact.keep_row = 1
+ """
+
+ sql "set enable_runtime_filter_partition_prune=true"
+ qt_rf_prune_expr_with_partition_prune """
+ SELECT /*+ SET_VAR(runtime_filter_type='IN') */ COUNT(*)
+ FROM rf_prune_expr_fact fact
+ JOIN [broadcast] rf_prune_expr_dim dim
+ ON assert_true(fact.part_key != 0, 'rfpp_expr_in_only_error') =
dim.flag
+ WHERE fact.keep_row = 1
+ """
+ assertNoPartitionPruningProfile(
+ "COUNT(*) FROM rf_prune_expr_fact fact JOIN [broadcast]
rf_prune_expr_dim dim "
+ + "ON assert_true(fact.part_key != 0,
'rfpp_expr_in_only_error') = dim.flag "
+ + "WHERE fact.keep_row = 1",
+ "IN")
+
// ============================================================
// Test 51: String partition column (LIST partition on VARCHAR).
//
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
