This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 99c6e1f0a83 branch-3.0: [fix](nereids)avoid generate Runtime filter
whose target is not a base table column #48804 (#49310)
99c6e1f0a83 is described below
commit 99c6e1f0a833b756044f83cbba79f983ae5441e8
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Mon Apr 7 17:49:59 2025 +0800
branch-3.0: [fix](nereids)avoid generate Runtime filter whose target is not
a base table column #48804 (#49310)
Cherry-picked from #48804
Co-authored-by: minghong <[email protected]>
---
.../post/RuntimeFilterPushDownVisitor.java | 21 +++++-
.../nereids/postprocess/RuntimeFilterTest.java | 76 ++++++++++++++++++++++
2 files changed, 96 insertions(+), 1 deletion(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterPushDownVisitor.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterPushDownVisitor.java
index 9a5c6a1daa9..bf83f4d2afe 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterPushDownVisitor.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterPushDownVisitor.java
@@ -182,7 +182,9 @@ public class RuntimeFilterPushDownVisitor extends
PlanVisitor<Boolean, PushDownC
public Boolean visit(Plan plan, PushDownContext ctx) {
boolean pushed = false;
for (Plan child : plan.children()) {
- pushed |= child.accept(this, ctx);
+ if
(child.getOutputSet().containsAll(ctx.probeExpr.getInputSlots())) {
+ pushed |= child.accept(this, ctx);
+ }
}
return pushed;
}
@@ -233,6 +235,10 @@ public class RuntimeFilterPushDownVisitor extends
PlanVisitor<Boolean, PushDownC
@Override
public Boolean visitPhysicalHashJoin(PhysicalHashJoin<? extends Plan, ?
extends Plan> join,
PushDownContext ctx) {
+ if (!ctx.builderNode.equals(join)
+ &&
!join.getOutputSet().containsAll(ctx.probeExpr.getInputSlots())) {
+ return false;
+ }
boolean pushed = false;
if (ctx.builderNode instanceof PhysicalHashJoin) {
@@ -307,6 +313,9 @@ public class RuntimeFilterPushDownVisitor extends
PlanVisitor<Boolean, PushDownC
@Override
public Boolean visitPhysicalNestedLoopJoin(PhysicalNestedLoopJoin<?
extends Plan, ? extends Plan> join,
PushDownContext ctx) {
+ if (!join.getOutputSet().containsAll(ctx.probeExpr.getInputSlots())) {
+ return false;
+ }
if (ctx.builderNode instanceof PhysicalHashJoin) {
/*
hashJoin( t1.A <=> t2.A )
@@ -335,6 +344,9 @@ public class RuntimeFilterPushDownVisitor extends
PlanVisitor<Boolean, PushDownC
@Override
public Boolean visitPhysicalProject(PhysicalProject<? extends Plan>
project, PushDownContext ctx) {
+ if
(!project.getOutputSet().containsAll(ctx.probeExpr.getInputSlots())) {
+ return false;
+ }
// project ( A+1 as x)
// probeExpr: abs(x) => abs(A+1)
PushDownContext ctxProjectProbeExpr = ctx;
@@ -377,6 +389,9 @@ public class RuntimeFilterPushDownVisitor extends
PlanVisitor<Boolean, PushDownC
@Override
public Boolean visitPhysicalSetOperation(PhysicalSetOperation
setOperation, PushDownContext ctx) {
+ if
(!setOperation.getOutputSet().containsAll(ctx.probeExpr.getInputSlots())) {
+ return false;
+ }
boolean pushedDown = false;
int projIndex = -1;
Slot probeSlot =
RuntimeFilterGenerator.checkTargetChild(ctx.probeExpr);
@@ -422,6 +437,10 @@ public class RuntimeFilterPushDownVisitor extends
PlanVisitor<Boolean, PushDownC
@Override
public Boolean visitPhysicalWindow(PhysicalWindow<? extends Plan> window,
PushDownContext ctx) {
+ if (!window.getOutputSet().containsAll(ctx.probeExpr.getInputSlots()))
{
+ return false;
+ }
+
Set<SlotReference> commonPartitionKeys =
window.getCommonPartitionKeyFromWindowExpressions();
if (commonPartitionKeys.containsAll(ctx.probeExpr.getInputSlots())) {
return window.child().accept(this, ctx);
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/RuntimeFilterTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/RuntimeFilterTest.java
index f354ed5b02f..8f9e1a5b6eb 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/RuntimeFilterTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/RuntimeFilterTest.java
@@ -22,9 +22,23 @@ import org.apache.doris.nereids.datasets.ssb.SSBTestBase;
import org.apache.doris.nereids.datasets.ssb.SSBUtils;
import org.apache.doris.nereids.glue.translator.PhysicalPlanTranslator;
import org.apache.doris.nereids.glue.translator.PlanTranslatorContext;
+import org.apache.doris.nereids.hint.DistributeHint;
import org.apache.doris.nereids.processor.post.PlanPostProcessors;
import org.apache.doris.nereids.processor.post.RuntimeFilterContext;
+import org.apache.doris.nereids.trees.expressions.Alias;
+import org.apache.doris.nereids.trees.expressions.EqualTo;
+import org.apache.doris.nereids.trees.expressions.ExprId;
+import org.apache.doris.nereids.trees.expressions.NamedExpression;
+import org.apache.doris.nereids.trees.expressions.SlotReference;
+import org.apache.doris.nereids.trees.expressions.literal.NullLiteral;
+import org.apache.doris.nereids.trees.plans.DistributeType;
+import org.apache.doris.nereids.trees.plans.JoinType;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.physical.AbstractPhysicalPlan;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalHashJoin;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalProject;
import org.apache.doris.nereids.trees.plans.physical.RuntimeFilter;
import org.apache.doris.nereids.util.PlanChecker;
@@ -33,6 +47,7 @@ import com.google.common.collect.Sets;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
+import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import java.util.Set;
@@ -344,4 +359,65 @@ public class RuntimeFilterTest extends SSBTestBase {
Assertions.assertEquals(0, filters.size());
}
+ @Test
+ public void testNotGenerateRfOnDanglingSlot() {
+ String sql = "select lo_custkey from lineorder union all select
c_custkey from customer union all select p_partkey from part;";
+ PlanChecker checker = PlanChecker.from(connectContext)
+ .analyze(sql)
+ .rewrite()
+ .implement();
+ PhysicalPlan plan = checker.getPhysicalPlan();
+
+ /* construct plan for
+ join (#18=p_partkey)
+ -->join()
+ -->project(null as #18, ...)
+ -->lineorder
+ -->project(c_custkey#17)
+ -->customer(output: c_custkey#17, c_name#18, ...)
+ -->project(p_partkey#25)
+ -->part
+
+ test purpose:
+ do not generate RF by "#18=p_partkey" and apply this rf on customer
+ */
+ PhysicalProject<Plan> projectCustomer = (PhysicalProject<Plan>)
plan.child(0).child(1);
+ SlotReference cCustkey = (SlotReference)
projectCustomer.getProjects().get(0);
+ PhysicalProject<Plan> projectPart = (PhysicalProject<Plan>)
plan.child(0).child(2);
+ SlotReference pPartkey = (SlotReference)
projectPart.getProjects().get(0);
+
+ PhysicalOlapScan lo = (PhysicalOlapScan)
plan.child(0).child(0).child(0);
+ SlotReference loCustkey = (SlotReference) lo.getBaseOutputs().get(2);
+ SlotReference loPartkey = (SlotReference) lo.getBaseOutputs().get(3);
+ Alias nullAlias = new Alias(new ExprId(18), new NullLiteral(), ""); //
expr#18 is used by c_name
+ List<NamedExpression> projList = new ArrayList<>();
+ projList.add(loCustkey);
+ projList.add(loPartkey);
+ projList.add(nullAlias);
+ PhysicalProject projLo = new PhysicalProject(projList, null, lo);
+
+ PhysicalHashJoin joinLoC = new PhysicalHashJoin(JoinType.INNER_JOIN,
+ ImmutableList.of(new EqualTo(loCustkey, cCustkey)),
+ ImmutableList.of(),
+ new DistributeHint(DistributeType.NONE),
+ Optional.empty(),
+ null,
+ projLo,
+ projectCustomer
+ );
+ PhysicalHashJoin joinLoCP = new PhysicalHashJoin(JoinType.INNER_JOIN,
+ ImmutableList.of(new EqualTo(nullAlias.toSlot(), pPartkey)),
+ ImmutableList.of(),
+ new DistributeHint(DistributeType.NONE),
+ Optional.empty(),
+ null,
+ joinLoC,
+ projectPart
+ );
+
checker.getCascadesContext().getConnectContext().getSessionVariable().enableRuntimeFilterPrune
= false;
+ plan = new
PlanPostProcessors(checker.getCascadesContext()).process(joinLoCP);
+ System.out.println(plan.treeString());
+ Assertions.assertEquals(0, ((AbstractPhysicalPlan)
plan.child(0).child(1).child(0))
+ .getAppliedRuntimeFilters().size());
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
