This is an automated email from the ASF dual-hosted git repository.
starocean999 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new b9f23d64cb7 [fix](fe) Move SemiJoinCommute to rewrite phase (#66182)
b9f23d64cb7 is described below
commit b9f23d64cb7f419c2bfc24dca841f8a0f452264b
Author: seawinde <[email protected]>
AuthorDate: Thu Aug 6 10:12:31 2026 +0800
[fix](fe) Move SemiJoinCommute to rewrite phase (#66182)
---
.../doris/nereids/jobs/executor/Analyzer.java | 2 -
.../doris/nereids/jobs/executor/Rewriter.java | 3 ++
.../rules/analysis/CollectJoinConstraint.java | 31 +++++++-----
.../nereids/rules/rewrite/ReorderJoinTest.java | 56 +++++++++++++++++++++-
.../doris/nereids/sqltest/SqlPlanSuiteTest.java | 10 ++--
regression-test/data/query_p0/hint/fix_leading.out | 6 +++
.../suites/query_p0/hint/fix_leading.groovy | 20 ++++++++
7 files changed, 109 insertions(+), 19 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Analyzer.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Analyzer.java
index 0400e50c792..eb354d11a47 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Analyzer.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Analyzer.java
@@ -52,7 +52,6 @@ import
org.apache.doris.nereids.rules.analysis.SubqueryToApply;
import org.apache.doris.nereids.rules.analysis.VariableToLiteral;
import org.apache.doris.nereids.rules.rewrite.AdjustNullable;
import org.apache.doris.nereids.rules.rewrite.MergeFilters;
-import org.apache.doris.nereids.rules.rewrite.SemiJoinCommute;
import org.apache.doris.nereids.rules.rewrite.SimplifyAggGroupBy;
import org.apache.doris.nereids.trees.plans.logical.LogicalCTEAnchor;
import org.apache.doris.nereids.trees.plans.logical.LogicalView;
@@ -199,7 +198,6 @@ public class Analyzer extends AbstractBatchJobExecutor {
topDown(new NormalizeAggregate()),
topDown(new HavingToFilter()),
topDown(new QualifyToFilter()),
- bottomUp(new SemiJoinCommute()),
bottomUp(
new CollectSubQueryAlias(),
new CollectJoinConstraint()
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
index 549d299692e..e651a9e8fae 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
@@ -160,6 +160,7 @@ import
org.apache.doris.nereids.rules.rewrite.RewriteCteChildren;
import org.apache.doris.nereids.rules.rewrite.RewriteSearchToSlots;
import org.apache.doris.nereids.rules.rewrite.RewriteSimpleAggToConstantRule;
import org.apache.doris.nereids.rules.rewrite.SaltJoin;
+import org.apache.doris.nereids.rules.rewrite.SemiJoinCommute;
import org.apache.doris.nereids.rules.rewrite.SetPreAggStatus;
import org.apache.doris.nereids.rules.rewrite.SimplifyEncodeDecode;
import org.apache.doris.nereids.rules.rewrite.SimplifyWindowExpression;
@@ -335,6 +336,7 @@ public class Rewriter extends AbstractBatchJobExecutor {
),
// push down SEMI Join
bottomUp(
+ new SemiJoinCommute(),
new TransposeSemiJoinLogicalJoin(),
new
TransposeSemiJoinLogicalJoinProject(),
new TransposeSemiJoinAgg(),
@@ -575,6 +577,7 @@ public class Rewriter extends AbstractBatchJobExecutor {
),
// push down SEMI Join
bottomUp(
+ new SemiJoinCommute(),
new TransposeSemiJoinLogicalJoin(),
new
TransposeSemiJoinLogicalJoinProject(),
new TransposeSemiJoinAgg(),
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectJoinConstraint.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectJoinConstraint.java
index b8b6d8adfab..eacdd3fdd41 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectJoinConstraint.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectJoinConstraint.java
@@ -61,6 +61,14 @@ public class CollectJoinConstraint implements
RewriteRuleFactory {
Long leftHand =
LongBitmap.computeTableBitmap(join.left().getInputRelations());
Long rightHand =
LongBitmap.computeTableBitmap(join.right().getInputRelations());
join.setBitmap(LongBitmap.or(leftHand, rightHand));
+ JoinType joinType = join.getJoinType();
+ if (joinType.isRightJoin()) {
+ // LEADING constraints model the preserved/output side as
the left child.
+ Long originalLeftHand = leftHand;
+ leftHand = rightHand;
+ rightHand = originalLeftHand;
+ joinType = joinType.swap();
+ }
List<Expression> expressions = join.getHashJoinConjuncts();
Long totalFilterBitMap = 0L;
Long nonNullableSlotBitMap = 0L;
@@ -69,11 +77,11 @@ public class CollectJoinConstraint implements
RewriteRuleFactory {
nonNullableSlotBitMap =
LongBitmap.or(nonNullableSlotBitMap, nonNullable);
Long filterBitMap = calSlotsTableBitMap(leading,
expression.getInputSlots(), false);
totalFilterBitMap = LongBitmap.or(totalFilterBitMap,
filterBitMap);
- if (join.getJoinType().isLeftJoin()) {
+ if (joinType.isLeftJoin()) {
filterBitMap = LongBitmap.or(filterBitMap, rightHand);
}
leading.getFilters().add(Pair.of(filterBitMap,
expression));
- leading.putConditionJoinType(expression,
join.getJoinType());
+ leading.putConditionJoinType(expression, joinType);
}
expressions = join.getOtherJoinConjuncts();
for (Expression expression : expressions) {
@@ -81,13 +89,14 @@ public class CollectJoinConstraint implements
RewriteRuleFactory {
nonNullableSlotBitMap =
LongBitmap.or(nonNullableSlotBitMap, nonNullable);
Long filterBitMap = calSlotsTableBitMap(leading,
expression.getInputSlots(), false);
totalFilterBitMap = LongBitmap.or(totalFilterBitMap,
filterBitMap);
- if (join.getJoinType().isLeftJoin()) {
+ if (joinType.isLeftJoin()) {
filterBitMap = LongBitmap.or(filterBitMap, rightHand);
}
leading.getFilters().add(Pair.of(filterBitMap,
expression));
- leading.putConditionJoinType(expression,
join.getJoinType());
+ leading.putConditionJoinType(expression, joinType);
}
- collectJoinConstraintList(leading, leftHand, rightHand, join,
totalFilterBitMap, nonNullableSlotBitMap);
+ collectJoinConstraintList(
+ leading, leftHand, rightHand, joinType,
totalFilterBitMap, nonNullableSlotBitMap);
return ctx.root;
}).toRule(RuleType.COLLECT_JOIN_CONSTRAINT),
@@ -108,14 +117,14 @@ public class CollectJoinConstraint implements
RewriteRuleFactory {
);
}
- private void collectJoinConstraintList(LeadingHint leading, Long leftHand,
Long rightHand, LogicalJoin join,
+ private void collectJoinConstraintList(LeadingHint leading, Long leftHand,
Long rightHand, JoinType joinType,
Long filterTableBitMap, Long
nonNullableSlotBitMap) {
Long totalTables = LongBitmap.or(leftHand, rightHand);
- if (join.getJoinType().isInnerOrCrossJoin()) {
+ if (joinType.isInnerOrCrossJoin()) {
leading.setInnerJoinBitmap(LongBitmap.or(leading.getInnerJoinBitmap(),
totalTables));
return;
}
- if (join.getJoinType().isFullOuterJoin()) {
+ if (joinType.isFullOuterJoin()) {
JoinConstraint newJoinConstraint = new JoinConstraint(leftHand,
rightHand, leftHand, rightHand,
JoinType.FULL_OUTER_JOIN, false);
leading.getJoinConstraintList().add(newJoinConstraint);
@@ -148,7 +157,7 @@ public class CollectJoinConstraint implements
RewriteRuleFactory {
if (LongBitmap.isOverlap(leftHand, other.getRightHand())) {
if (LongBitmap.isOverlap(filterTableBitMap,
other.getRightHand())
- && (join.getJoinType().isSemiOrAntiJoin()
+ && (joinType.isSemiOrAntiJoin()
|| !LongBitmap.isOverlap(nonNullableSlotBitMap,
other.getMinRightHand()))) {
minLeftHand = LongBitmap.or(minLeftHand,
other.getLeftHand());
@@ -160,7 +169,7 @@ public class CollectJoinConstraint implements
RewriteRuleFactory {
if (LongBitmap.isOverlap(rightHand, other.getRightHand())) {
if (LongBitmap.isOverlap(filterTableBitMap,
other.getRightHand())
|| !LongBitmap.isOverlap(filterTableBitMap,
other.getMinLeftHand())
- || join.getJoinType().isSemiOrAntiJoin()
+ || joinType.isSemiOrAntiJoin()
|| other.getJoinType().isSemiOrAntiJoin()
|| !other.isLhsStrict()) {
minRightHand = LongBitmap.or(minRightHand,
other.getLeftHand());
@@ -176,7 +185,7 @@ public class CollectJoinConstraint implements
RewriteRuleFactory {
}
JoinConstraint newJoinConstraint = new JoinConstraint(minLeftHand,
minRightHand, leftHand, rightHand,
- join.getJoinType(), isStrict);
+ joinType, isStrict);
leading.getJoinConstraintList().add(newJoinConstraint);
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/ReorderJoinTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/ReorderJoinTest.java
index c8d8e8af7af..fc9011ff178 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/ReorderJoinTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/ReorderJoinTest.java
@@ -18,6 +18,8 @@
package org.apache.doris.nereids.rules.rewrite;
import org.apache.doris.common.Pair;
+import org.apache.doris.nereids.CascadesContext;
+import org.apache.doris.nereids.jobs.executor.Rewriter;
import org.apache.doris.nereids.trees.expressions.EqualTo;
import org.apache.doris.nereids.trees.plans.JoinType;
import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
@@ -68,6 +70,59 @@ class ReorderJoinTest implements MemoPatternMatchSupported {
testRightOuterJoinHelper(JoinType.RIGHT_OUTER_JOIN);
}
+ @Test
+ public void testSemiJoinCommuteInRewrite() {
+ for (JoinType joinType : ImmutableList.of(
+ JoinType.RIGHT_OUTER_JOIN, JoinType.RIGHT_SEMI_JOIN,
JoinType.RIGHT_ANTI_JOIN)) {
+ ConnectContext connectContext =
MemoTestUtils.createConnectContext();
+
connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION");
+ PlanChecker checker = PlanChecker.from(connectContext)
+ .analyze(new LogicalPlanBuilder(scan1)
+ .join(scan2, joinType, Pair.of(0, 0))
+ .build())
+ .matches(logicalJoin().when(join -> join.getJoinType() ==
joinType));
+
+ checker.rewrite()
+ .matches(logicalJoin().when(join -> join.getJoinType() ==
joinType.swap()));
+ }
+ }
+
+ @Test
+ public void testDisableJoinReorderBeforeRewrite() {
+ for (JoinType joinType : ImmutableList.of(
+ JoinType.RIGHT_OUTER_JOIN, JoinType.RIGHT_SEMI_JOIN,
JoinType.RIGHT_ANTI_JOIN)) {
+ ConnectContext connectContext =
MemoTestUtils.createConnectContext();
+
connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION");
+ PlanChecker checker = PlanChecker.from(connectContext)
+ .analyze(new LogicalPlanBuilder(scan1)
+ .join(scan2, joinType, Pair.of(0, 0))
+ .build());
+
+ connectContext.getSessionVariable().setDisableJoinReorder(true);
+ checker.rewrite()
+ .matches(logicalJoin().when(join -> join.getJoinType() ==
joinType));
+ }
+ }
+
+ @Test
+ public void testSemiJoinCommuteInMvPreRewrite() {
+ for (JoinType joinType : ImmutableList.of(
+ JoinType.RIGHT_OUTER_JOIN, JoinType.RIGHT_SEMI_JOIN,
JoinType.RIGHT_ANTI_JOIN)) {
+ ConnectContext connectContext =
MemoTestUtils.createConnectContext();
+
connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION");
+ PlanChecker checker = PlanChecker.from(connectContext)
+ .analyze(new LogicalPlanBuilder(scan1)
+ .join(scan2, joinType, Pair.of(0, 0))
+ .build());
+ CascadesContext cascadesContext = checker.getCascadesContext();
+
+ Rewriter.getCteChildrenRewriter(
+ cascadesContext,
Rewriter.CTE_CHILDREN_REWRITE_JOBS_MV_REWRITE_USED, false).execute();
+ MemoTestUtils.initMemoAndValidState(cascadesContext);
+ checker.matches(logicalJoin().when(join -> join.getJoinType() ==
joinType.swap()));
+ }
+ }
+
private void testRightOuterJoinHelper(JoinType joinType) {
ImmutableList<LogicalPlan> plans = ImmutableList.of(
new LogicalPlanBuilder(scan1)
@@ -153,7 +208,6 @@ class ReorderJoinTest implements MemoPatternMatchSupported {
ConnectContext connectContext = MemoTestUtils.createConnectContext();
connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION");
PlanChecker.from(connectContext, plan2)
- .applyBottomUp(new SemiJoinCommute())
.rewrite()
.matchesFromRoot(
logicalProject(innerLogicalJoin(
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/SqlPlanSuiteTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/SqlPlanSuiteTest.java
index 879d724c69e..72aa7e3b87c 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/SqlPlanSuiteTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/SqlPlanSuiteTest.java
@@ -243,13 +243,13 @@ public class SqlPlanSuiteTest extends SqlTestBase {
.printlnTree()
.matches(
innerLogicalJoin(
- logicalFilter().when(
- f -> f.getPredicate().toString().equals("(id#2
>= 4)")),
logicalFilter().when(
f ->
ExpressionUtils.and(f.getConjuncts().stream()
.sorted((a, b) ->
a.toString().compareTo(b.toString()))
.collect(Collectors.toList()))
- .toString().equals("(id#0 >= 4)"))
+ .toString().equals("(id#0 >= 4)")),
+ logicalFilter().when(
+ f -> f.getPredicate().toString().equals("(id#2
>= 4)"))
)
);
@@ -288,8 +288,8 @@ public class SqlPlanSuiteTest extends SqlTestBase {
.rewrite()
.matches(
innerLogicalJoin(
- logicalProject(),
- logicalProject(leftSemiLogicalJoin())
+ logicalProject(leftSemiLogicalJoin()),
+ logicalProject()
)
);
}
diff --git a/regression-test/data/query_p0/hint/fix_leading.out
b/regression-test/data/query_p0/hint/fix_leading.out
index 526c6eac8ea..df4c476d3d5 100644
--- a/regression-test/data/query_p0/hint/fix_leading.out
+++ b/regression-test/data/query_p0/hint/fix_leading.out
@@ -243,6 +243,12 @@ Used: leading(t1 t2 t3 )
UnUsed:
SyntaxError:
+-- !select4_4 --
+1
+
+-- !select4_6 --
+1
+
-- !select6_1 --
PhysicalResultSink
--hashAgg[GLOBAL]
diff --git a/regression-test/suites/query_p0/hint/fix_leading.groovy
b/regression-test/suites/query_p0/hint/fix_leading.groovy
index a08a5fa4186..c1709e0d3ea 100644
--- a/regression-test/suites/query_p0/hint/fix_leading.groovy
+++ b/regression-test/suites/query_p0/hint/fix_leading.groovy
@@ -38,6 +38,9 @@ suite("fix_leading") {
sql """drop table if exists t2;"""
sql """drop table if exists t3;"""
sql """drop table if exists t4;"""
+ sql """drop table if exists right_join_a;"""
+ sql """drop table if exists right_join_b;"""
+ sql """drop table if exists right_join_c;"""
sql """create table t1 (c1 int, c11 int) distributed by hash(c1) buckets 3
properties('replication_num' = '1');"""
sql """create table t2 (c2 int, c22 int) distributed by hash(c2) buckets 3
properties('replication_num' = '1');"""
@@ -45,6 +48,13 @@ suite("fix_leading") {
sql """create table t4 (c4 int, c44 int) distributed by hash(c4) buckets 3
properties('replication_num' = '1');"""
sql """create table t5 (c5 int, c55 int) distributed by hash(c5) buckets 3
properties('replication_num' = '1');"""
sql """create table t6 (c6 int, c66 int) distributed by hash(c6) buckets 3
properties('replication_num' = '1');"""
+ sql """create table right_join_a (k int) distributed by hash(k) buckets 1
properties('replication_num' = '1');"""
+ sql """create table right_join_b (k int) distributed by hash(k) buckets 1
properties('replication_num' = '1');"""
+ sql """create table right_join_c (k int) distributed by hash(k) buckets 1
properties('replication_num' = '1');"""
+
+ sql """insert into right_join_a values (1);"""
+ sql """insert into right_join_b values (0), (1);"""
+ sql """insert into right_join_c values (1), (2);"""
streamLoad {
table "t1"
@@ -196,6 +206,16 @@ suite("fix_leading") {
qt_select4_2 """select /*+ leading(t1 t2 t3)*/ count(*) from t1 left join
t2 on c1 > 500 and c2 >500 right join t3 on c3 > 500 and c1 < 200;"""
qt_select4_3 """explain shape plan select /*+ leading(t1 t2 t3)*/ count(*)
from t1 left join t2 on c1 > 500 and c2 >500 right join t3 on c3 > 500 and c1 <
200;"""
+ // check right semi join keeps its complete non-output side
+ qt_select4_4 """select /*+ leading(right_join_b right_join_a right_join_c)
*/ count(*)
+ from right_join_a cross join right_join_c
+ right semi join right_join_b on right_join_a.k = right_join_b.k;"""
+
+ // check right anti join does not push its preserved-side ON predicate
below the join
+ qt_select4_6 """select /*+ leading(right_join_b right_join_a right_join_c)
*/ count(*)
+ from right_join_a cross join right_join_c
+ right anti join right_join_b on right_join_a.k = right_join_b.k and
right_join_b.k > 0;"""
+
// check whether we have all tables
explain {
sql """shape plan select /*+ leading(t1 t2)*/ count(*) from t1 left
join t2 on c1 > 500 and c2 >500 right join t3 on c3 > 500 and c1 < 200;"""
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
