This is an automated email from the ASF dual-hosted git repository.
englefly pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 67019ff97ed [fix](topnFilter)Fix TopN filter probe expressions to wrap
nullable slots when pushed through outer joins. (#59074)
67019ff97ed is described below
commit 67019ff97ed8f31dc96334d69a3257f45a7e3e41
Author: minghong <[email protected]>
AuthorDate: Fri Feb 13 14:57:50 2026 +0800
[fix](topnFilter)Fix TopN filter probe expressions to wrap nullable slots
when pushed through outer joins. (#59074)
### What problem does this PR solve?
Fix TopN filter probe expressions to wrap nullable slots when pushed
through outer joins. (#59074)
---
.../processor/post/TopnFilterPushDownVisitor.java | 32 +++++++++++--
.../nereids/postprocess/TopNRuntimeFilterTest.java | 53 ++++++++++++++++++++++
2 files changed, 82 insertions(+), 3 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/TopnFilterPushDownVisitor.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/TopnFilterPushDownVisitor.java
index 101b29d91f9..bf21d5baf9e 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/TopnFilterPushDownVisitor.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/TopnFilterPushDownVisitor.java
@@ -21,6 +21,8 @@ import
org.apache.doris.nereids.processor.post.TopnFilterPushDownVisitor.PushDow
import org.apache.doris.nereids.trees.expressions.Alias;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
+import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.Nullable;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitors;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.algebra.TopN;
@@ -42,6 +44,7 @@ import
org.apache.doris.nereids.trees.plans.physical.PhysicalSetOperation;
import org.apache.doris.nereids.trees.plans.physical.PhysicalTopN;
import org.apache.doris.nereids.trees.plans.physical.PhysicalWindow;
import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.nereids.util.ExpressionUtils;
import org.apache.doris.qe.ConnectContext;
import com.google.common.collect.Maps;
@@ -169,6 +172,26 @@ public class TopnFilterPushDownVisitor extends
PlanVisitor<Boolean, PushDownCont
return false;
}
+ private PushDownContext adjustProbeExprNullableThroughOuterJoin(Plan
leftChild, PushDownContext ctx) {
+ Expression probeExpr = ctx.probeExpr;
+ Map<Expression, Expression> replaceMap = Maps.newHashMap();
+ boolean changed = false;
+ for (Slot probSlot : probeExpr.getInputSlots()) {
+ for (Slot childSlot : leftChild.getOutput()) {
+ if (probSlot.getExprId().asInt() ==
childSlot.getExprId().asInt()
+ && probSlot.nullable() && !childSlot.nullable()) {
+ replaceMap.put(probSlot, new Nullable(childSlot));
+ changed = true;
+ break;
+ }
+ }
+ }
+ if (changed) {
+ return
ctx.withNewProbeExpression(ExpressionUtils.replace(probeExpr, replaceMap));
+ }
+ return ctx;
+ }
+
@Override
public Boolean visitPhysicalHashJoin(PhysicalHashJoin<? extends Plan, ?
extends Plan> join,
PushDownContext ctx) {
@@ -180,7 +203,8 @@ public class TopnFilterPushDownVisitor extends
PlanVisitor<Boolean, PushDownCont
return false;
}
if
(join.left().getOutputSet().containsAll(ctx.probeExpr.getInputSlots())) {
- return join.left().accept(this, ctx);
+ PushDownContext childPushDownContext =
adjustProbeExprNullableThroughOuterJoin(join.left(), ctx);
+ return join.left().accept(this, childPushDownContext);
}
if
(join.right().getOutputSet().containsAll(ctx.probeExpr.getInputSlots())) {
// expand expr to the other side of hash join condition:
@@ -189,8 +213,10 @@ public class TopnFilterPushDownVisitor extends
PlanVisitor<Boolean, PushDownCont
for (Expression conj : join.getHashJoinConjuncts()) {
if (ctx.probeExpr.equals(conj.child(1))) {
// push to left child. right child is blocking operator,
do not need topn-filter
- PushDownContext newCtx =
ctx.withNewProbeExpression(conj.child(0));
- return join.left().accept(this, newCtx);
+ PushDownContext ctxOnLeftChild =
ctx.withNewProbeExpression(conj.child(0));
+ PushDownContext childPushDownContext =
adjustProbeExprNullableThroughOuterJoin(join.left(),
+ ctxOnLeftChild);
+ return join.left().accept(this, childPushDownContext);
}
}
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/TopNRuntimeFilterTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/TopNRuntimeFilterTest.java
index 261ac8a813e..41c7001f792 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/TopNRuntimeFilterTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/TopNRuntimeFilterTest.java
@@ -19,16 +19,26 @@ package org.apache.doris.nereids.postprocess;
import org.apache.doris.nereids.datasets.ssb.SSBTestBase;
import org.apache.doris.nereids.processor.post.PlanPostProcessors;
+import org.apache.doris.nereids.processor.post.TopnFilterContext;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.Nullable;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.Substring;
+import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.SortPhase;
import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalRelation;
import org.apache.doris.nereids.trees.plans.physical.PhysicalTopN;
+import org.apache.doris.nereids.trees.plans.physical.TopnFilter;
import org.apache.doris.nereids.util.MemoPatternMatchSupported;
import org.apache.doris.nereids.util.PlanChecker;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
+import java.util.Map;
+
public class TopNRuntimeFilterTest extends SSBTestBase implements
MemoPatternMatchSupported {
@Override
public void runBeforeAll() throws Exception {
@@ -78,4 +88,47 @@ public class TopNRuntimeFilterTest extends SSBTestBase
implements MemoPatternMat
Assertions.assertTrue(localTopN.getSortPhase().isLocal());
Assertions.assertFalse(checker.getCascadesContext().getTopnFilterContext().isTopnFilterSource(localTopN));
}
+
+ @Test
+ public void testProbeExprNullableThroughRightOuterJoin() {
+ // topn node push down filter value to scan node.
+ // the filter value is nullable.
+ // but c_name in scan node is not nullable. so we use
+ // substring(nullable(c_name), 1, 5) as probe expr on scan node
+ String sql = "select substring(c_name, 1, 5) "
+ + "from customer c right outer join lineorder l "
+ + "on c.c_custkey = l.lo_custkey "
+ + "order by substring(c_name, 1, 5) nulls last limit 3";
+ connectContext.getSessionVariable().setDisableJoinReorder(true);
+ PlanChecker checker = PlanChecker.from(connectContext)
+ .analyze(sql)
+ .rewrite()
+ .implement();
+ PhysicalPlan plan = checker.getPhysicalPlan();
+ plan = new
PlanPostProcessors(checker.getCascadesContext()).process(plan);
+
+ TopnFilterContext ctx =
checker.getCascadesContext().getTopnFilterContext();
+ Assertions.assertFalse(ctx.getTopnFilters().isEmpty(), "topn filter
should be created");
+
+ TopnFilter filter = ctx.getTopnFilters().stream()
+ .filter(f -> f.targets.values().stream().anyMatch(expr -> expr
instanceof Substring))
+ .findFirst()
+ .orElseThrow(() -> new AssertionError("topn filter with
substring probe not found"));
+
+ Map.Entry<PhysicalRelation, Expression> target =
filter.targets.entrySet().stream()
+ .filter(entry -> entry.getValue() instanceof Substring)
+ .findFirst()
+ .orElseThrow(() -> new AssertionError("substring probe target
not found"));
+ PhysicalRelation relation = target.getKey();
+ Expression probeExpr = target.getValue();
+
+ Slot cNameSlot = relation.getOutput().stream()
+ .filter(slot -> slot.getName().equalsIgnoreCase("c_name"))
+ .findFirst()
+ .orElseThrow(() -> new IllegalStateException("c_name slot not
found"));
+
+ Expression expectedProbeExpr = new Substring(new Nullable(cNameSlot),
new IntegerLiteral(1),
+ new IntegerLiteral(5));
+ Assertions.assertEquals(expectedProbeExpr, probeExpr);
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
