This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new e32eef5f0be [Fix](Nereids) fix leading syntax problems and data
mismatched problem (#32286)
e32eef5f0be is described below
commit e32eef5f0be08adc096d84fbbabe0dadc259952e
Author: LiBinfeng <[email protected]>
AuthorDate: Mon Mar 18 14:41:44 2024 +0800
[Fix](Nereids) fix leading syntax problems and data mismatched problem
(#32286)
- fix syntax problems of only one table used in leading or mistake usage of
brace
example: leading(t1),leading(t1 {t2})
- fix cte used in subquery of using leading
example: with cte as (select c1 from t1) select count(*) from t1 join
(select /*+ leading(cte t2) */ c2 from t2 join cte on c2 = cte.c1) as alias on
t1.c1 = alias.c2;
which cte used in subquery and subquery also have leading
- fix data mismatched with original plan cause of on predicate push to
nullable side
example: select count(*) from t1 left join t2 on c1 > 500 and c2 > 500
can not change to select count(*) from t1 left join t2 on c2 > 500 where c1 >
500
---
.../org/apache/doris/nereids/hint/LeadingHint.java | 5 +++
.../processor/pre/PullUpSubqueryAliasToCTE.java | 8 +++++
.../rules/analysis/CollectJoinConstraint.java | 12 +++++--
.../rules/analysis/EliminateLogicalSelectHint.java | 12 +++++--
.../data/nereids_p0/hint/fix_leading.out | 42 ++++++++++++++++++++++
.../data/nereids_p0/hint/multi_leading.out | 2 +-
.../suites/nereids_p0/hint/fix_leading.groovy | 17 +++++++++
7 files changed, 92 insertions(+), 6 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/hint/LeadingHint.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/hint/LeadingHint.java
index 6cefb8c6088..1f48e4873cb 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/hint/LeadingHint.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/hint/LeadingHint.java
@@ -533,6 +533,11 @@ public class LeadingHint extends Hint {
stackTopLevel = currentLevel;
}
}
+ if (stack.size() > 1) {
+ this.setStatus(HintStatus.SYNTAX_ERROR);
+ this.setErrorMessage("please check your brace pairs in leading");
+ return null;
+ }
LogicalJoin finalJoin = (LogicalJoin) stack.pop().second.first;
// we want all filters been remove
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/pre/PullUpSubqueryAliasToCTE.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/pre/PullUpSubqueryAliasToCTE.java
index e5d5c0fdf9e..0101a346478 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/pre/PullUpSubqueryAliasToCTE.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/pre/PullUpSubqueryAliasToCTE.java
@@ -41,6 +41,14 @@ public class PullUpSubqueryAliasToCTE extends
PlanPreprocessor {
StatementContext context) {
Plan topPlan = visitChildren(this, unboundResultSink, context);
if (!aliasQueries.isEmpty()) {
+ if (((UnboundResultSink) topPlan).child() instanceof LogicalCTE) {
+ LogicalCTE logicalCTE = (LogicalCTE) ((UnboundResultSink)
topPlan).child();
+ List<LogicalSubQueryAlias<Plan>> subQueryAliases = new
ArrayList<>();
+ subQueryAliases.addAll(logicalCTE.getAliasQueries());
+ subQueryAliases.addAll(aliasQueries);
+ return topPlan.withChildren(
+ new LogicalCTE<>(subQueryAliases, (LogicalPlan)
((UnboundResultSink) topPlan).child()));
+ }
return topPlan.withChildren(
new LogicalCTE<>(aliasQueries, (LogicalPlan)
((UnboundResultSink) topPlan).child()));
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectJoinConstraint.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectJoinConstraint.java
index 3b7f22a29bc..b8b6d8adfab 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectJoinConstraint.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectJoinConstraint.java
@@ -58,6 +58,9 @@ public class CollectJoinConstraint implements
RewriteRuleFactory {
leading.setStatus(Hint.HintStatus.UNUSED);
leading.setErrorMessage("condition does not matched
joinType");
}
+ Long leftHand =
LongBitmap.computeTableBitmap(join.left().getInputRelations());
+ Long rightHand =
LongBitmap.computeTableBitmap(join.right().getInputRelations());
+ join.setBitmap(LongBitmap.or(leftHand, rightHand));
List<Expression> expressions = join.getHashJoinConjuncts();
Long totalFilterBitMap = 0L;
Long nonNullableSlotBitMap = 0L;
@@ -66,6 +69,9 @@ public class CollectJoinConstraint implements
RewriteRuleFactory {
nonNullableSlotBitMap =
LongBitmap.or(nonNullableSlotBitMap, nonNullable);
Long filterBitMap = calSlotsTableBitMap(leading,
expression.getInputSlots(), false);
totalFilterBitMap = LongBitmap.or(totalFilterBitMap,
filterBitMap);
+ if (join.getJoinType().isLeftJoin()) {
+ filterBitMap = LongBitmap.or(filterBitMap, rightHand);
+ }
leading.getFilters().add(Pair.of(filterBitMap,
expression));
leading.putConditionJoinType(expression,
join.getJoinType());
}
@@ -75,12 +81,12 @@ public class CollectJoinConstraint implements
RewriteRuleFactory {
nonNullableSlotBitMap =
LongBitmap.or(nonNullableSlotBitMap, nonNullable);
Long filterBitMap = calSlotsTableBitMap(leading,
expression.getInputSlots(), false);
totalFilterBitMap = LongBitmap.or(totalFilterBitMap,
filterBitMap);
+ if (join.getJoinType().isLeftJoin()) {
+ filterBitMap = LongBitmap.or(filterBitMap, rightHand);
+ }
leading.getFilters().add(Pair.of(filterBitMap,
expression));
leading.putConditionJoinType(expression,
join.getJoinType());
}
- Long leftHand =
LongBitmap.computeTableBitmap(join.left().getInputRelations());
- Long rightHand =
LongBitmap.computeTableBitmap(join.right().getInputRelations());
- join.setBitmap(LongBitmap.or(leftHand, rightHand));
collectJoinConstraintList(leading, leftHand, rightHand, join,
totalFilterBitMap, nonNullableSlotBitMap);
return ctx.root;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/EliminateLogicalSelectHint.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/EliminateLogicalSelectHint.java
index 29d6b5e4a80..ac41323f5bc 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/EliminateLogicalSelectHint.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/EliminateLogicalSelectHint.java
@@ -122,11 +122,19 @@ public class EliminateLogicalSelectHint extends
OneRewriteRuleFactory {
context.setLeadingJoin(false);
return;
}
+ statementContext.addHint(hint);
+ context.getHintMap().put("Leading", hint);
+ if (hint.getTablelist().size() < 2) {
+ hint.setStatus(Hint.HintStatus.SYNTAX_ERROR);
+
context.getHintMap().get("Leading").setStatus(Hint.HintStatus.UNUSED);
+ hint.setErrorMessage("less than two tables is not allowed in
leading clause");
+ statementContext.addHint(hint);
+ context.setLeadingJoin(false);
+ return;
+ }
if (!hint.isSyntaxError()) {
hint.setStatus(Hint.HintStatus.SUCCESS);
}
- statementContext.addHint(hint);
- context.getHintMap().put("Leading", hint);
if (hints.get("ordered") != null ||
ConnectContext.get().getSessionVariable().isDisableJoinReorder()
|| context.isLeadingDisableJoinReorder()) {
context.setLeadingJoin(false);
diff --git a/regression-test/data/nereids_p0/hint/fix_leading.out
b/regression-test/data/nereids_p0/hint/fix_leading.out
index e4c2d1808c1..5f33c6e668a 100644
--- a/regression-test/data/nereids_p0/hint/fix_leading.out
+++ b/regression-test/data/nereids_p0/hint/fix_leading.out
@@ -215,3 +215,45 @@ SyntaxError:
-- !select2_5_13 --
816000
+-- !select3_1 --
+1719
+
+-- !select3_2 --
+1719
+
+-- !select3_3 --
+1719
+
+-- !select3_4 --
+5169
+
+-- !select4_1 --
+1000
+
+-- !select4_2 --
+1000
+
+-- !select4_3 --
+PhysicalResultSink
+--hashAgg[GLOBAL]
+----hashAgg[LOCAL]
+------PhysicalProject
+--------NestedLoopJoin[RIGHT_OUTER_JOIN](t3.c3 > 500)
+----------PhysicalDistribute[DistributionSpecGather]
+------------PhysicalProject
+--------------NestedLoopJoin[LEFT_OUTER_JOIN](t1.c1 < 200)(t1.c1 > 500)
+----------------PhysicalProject
+------------------PhysicalOlapScan[t1]
+----------------PhysicalDistribute[DistributionSpecReplicated]
+------------------PhysicalProject
+--------------------filter((t2.c2 > 500))
+----------------------PhysicalOlapScan[t2]
+----------PhysicalDistribute[DistributionSpecGather]
+------------PhysicalProject
+--------------PhysicalOlapScan[t3]
+
+Hint log:
+Used: leading(t1 t2 t3 )
+UnUsed:
+SyntaxError:
+
diff --git a/regression-test/data/nereids_p0/hint/multi_leading.out
b/regression-test/data/nereids_p0/hint/multi_leading.out
index 4ded091a3d1..0efabe2002e 100644
--- a/regression-test/data/nereids_p0/hint/multi_leading.out
+++ b/regression-test/data/nereids_p0/hint/multi_leading.out
@@ -330,7 +330,7 @@ PhysicalResultSink
----------------------PhysicalOlapScan[t1]
Hint log:
-Used: leading(t2 t1 ) leading(t2 t1 ) leading(t3 alias1 )
+Used: leading(t2 t1 ) leading(t2 t1 ) leading(t2 t1 ) leading(t3 alias1 )
UnUsed:
SyntaxError:
diff --git a/regression-test/suites/nereids_p0/hint/fix_leading.groovy
b/regression-test/suites/nereids_p0/hint/fix_leading.groovy
index 52f93e3d3e2..6185789d2ba 100644
--- a/regression-test/suites/nereids_p0/hint/fix_leading.groovy
+++ b/regression-test/suites/nereids_p0/hint/fix_leading.groovy
@@ -153,4 +153,21 @@ suite("fix_leading") {
qt_select2_5_11 """select /*+ leading(t3 {t1 t2}) */ count(*) from t1
right join t2 on c2 = c2 left semi join t3 on c2 = c3;"""
qt_select2_5_12 """select /*+ leading(t3 t2 t1) */ count(*) from t1 right
join t2 on c2 = c2 left semi join t3 on c2 = c3;"""
qt_select2_5_13 """select /*+ leading(t3 {t2 t1}) */ count(*) from t1
right join t2 on c2 = c2 left semi join t3 on c2 = c3;"""
+
+ // check only one table used in leading
+ qt_select3_1 """select /*+ leading(t1) */ count(*) from t1 join t2 on c1 =
c2;"""
+
+ // check only one table used in leading and add brace
+ qt_select3_2 """select /*+ leading({t1}) */ count(*) from t1 join t2 on c1
= c2;"""
+
+ // check mistake usage of brace
+ qt_select3_3 """select /*+ leading(t1 {t2}) */ count(*) from t1 join t2 on
c1 = c2;"""
+
+ // check using subquery alias to cte in cte query
+ qt_select3_4 """with cte as (select c1 from t1) select count(*) from t1
join (select /*+ leading(cte t2) */ c2 from t2 join cte on c2 = cte.c1) as
alias on t1.c1 = alias.c2;"""
+
+ // check left right join result
+ qt_select4_1 """select count(*) from t1 left join t2 on c1 > 500 and c2
>500 right join t3 on c3 > 500 and c1 < 200;"""
+ qt_select4_2 """select /*+ leading(t1 t2 t3)*/ count(*) from t1 left join
t2 on c1 > 500 and c2 >500 right join t3 on c3 > 500 and c1 < 200;"""
+ qt_select4_3 """explain shape plan select /*+ leading(t1 t2 t3)*/ count(*)
from t1 left join t2 on c1 > 500 and c2 >500 right join t3 on c3 > 500 and c1 <
200;"""
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
