This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new 09886853454 [fix](nereids)should normalize window expression by bottom
project's output (#31210)
09886853454 is described below
commit 098868534547c721fb3af24650e3287cc8e907f6
Author: starocean999 <[email protected]>
AuthorDate: Wed Feb 21 17:44:48 2024 +0800
[fix](nereids)should normalize window expression by bottom project's output
(#31210)
---
.../ExtractAndNormalizeWindowExpression.java | 20 +++++++++++----
.../nereids_p0/cte/test_cte_filter_pushdown.out | 18 ++++++++++++++
.../nereids_p0/cte/test_cte_filter_pushdown.groovy | 29 ++++++++++++++++++++++
3 files changed, 62 insertions(+), 5 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ExtractAndNormalizeWindowExpression.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ExtractAndNormalizeWindowExpression.java
index f45f7a287e8..4b080fb48fe 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ExtractAndNormalizeWindowExpression.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ExtractAndNormalizeWindowExpression.java
@@ -19,7 +19,6 @@ package org.apache.doris.nereids.rules.rewrite;
import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
-import
org.apache.doris.nereids.rules.rewrite.NormalizeToSlot.NormalizeToSlotContext;
import org.apache.doris.nereids.trees.expressions.Alias;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
@@ -34,6 +33,7 @@ import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Sets;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@@ -79,11 +79,21 @@ public class ExtractAndNormalizeWindowExpression extends
OneRewriteRuleFactory i
// 2. handle window's outputs and windowExprs
// need to replace exprs with SlotReference in WindowSpec, due to
LogicalWindow.getExpressions()
- List<NamedExpression> normalizedOutputs1 =
context.normalizeToUseSlotRef(outputs);
+
+ // because alias is pushed down to bottom project
+ // we need replace alias's child expr with corresponding alias's
slot in output
+ // so create a customNormalizeMap alias's child -> alias.toSlot to
do it
+ Map<Expression, Slot> customNormalizeMap = toBePushedDown.stream()
+ .filter(expr -> expr instanceof Alias)
+ .collect(Collectors.toMap(expr -> ((Alias) expr).child(),
expr -> ((Alias) expr).toSlot(),
+ (oldExpr, newExpr) -> oldExpr));
+
+ List<NamedExpression> normalizedOutputs =
context.normalizeToUseSlotRef(outputs,
+ (ctx, expr) -> customNormalizeMap.getOrDefault(expr,
null));
Set<WindowExpression> normalizedWindows =
- ExpressionUtils.collect(normalizedOutputs1,
WindowExpression.class::isInstance);
+ ExpressionUtils.collect(normalizedOutputs,
WindowExpression.class::isInstance);
- existedAlias = ExpressionUtils.collect(normalizedOutputs1,
Alias.class::isInstance);
+ existedAlias = ExpressionUtils.collect(normalizedOutputs,
Alias.class::isInstance);
NormalizeToSlotContext ctxForWindows =
NormalizeToSlotContext.buildContext(
existedAlias, Sets.newHashSet(normalizedWindows));
@@ -93,7 +103,7 @@ public class ExtractAndNormalizeWindowExpression extends
OneRewriteRuleFactory i
new
LogicalWindow<>(ImmutableList.copyOf(normalizedWindowWithAlias),
normalizedChild);
// 3. handle top projects
- List<NamedExpression> topProjects =
ctxForWindows.normalizeToUseSlotRef(normalizedOutputs1);
+ List<NamedExpression> topProjects =
ctxForWindows.normalizeToUseSlotRef(normalizedOutputs);
return project.withProjectsAndChild(topProjects,
normalizedLogicalWindow);
}).toRule(RuleType.EXTRACT_AND_NORMALIZE_WINDOW_EXPRESSIONS);
}
diff --git a/regression-test/data/nereids_p0/cte/test_cte_filter_pushdown.out
b/regression-test/data/nereids_p0/cte/test_cte_filter_pushdown.out
index 0c632f4fc29..36b8737a988 100644
--- a/regression-test/data/nereids_p0/cte/test_cte_filter_pushdown.out
+++ b/regression-test/data/nereids_p0/cte/test_cte_filter_pushdown.out
@@ -41,3 +41,21 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
--------------filter((m2.k1 = 1))
----------------PhysicalCteConsumer ( cteId=CTEId#0 )
+-- !cte_filter_pushdown_3 --
+PhysicalResultSink
+--PhysicalDistribute
+----PhysicalProject
+------hashJoin[INNER_JOIN](tmp2.k3 = dd.k3)
+--------PhysicalDistribute
+----------PhysicalProject
+------------filter((tmp2.k3 = 0))
+--------------PhysicalWindow
+----------------PhysicalQuickSort
+------------------PhysicalProject
+--------------------filter((tmp.k1 = 1))
+----------------------PhysicalOlapScan[test]
+--------PhysicalDistribute
+----------PhysicalProject
+------------filter((dd.k3 = 0))
+--------------PhysicalOlapScan[baseall]
+
diff --git
a/regression-test/suites/nereids_p0/cte/test_cte_filter_pushdown.groovy
b/regression-test/suites/nereids_p0/cte/test_cte_filter_pushdown.groovy
index 8f08721f6cd..3d65a73f657 100644
--- a/regression-test/suites/nereids_p0/cte/test_cte_filter_pushdown.groovy
+++ b/regression-test/suites/nereids_p0/cte/test_cte_filter_pushdown.groovy
@@ -44,4 +44,33 @@ suite("test_cte_filter_pushdown)") {
) temp
where k1 = 1;
"""
+ sql 'set exec_mem_limit=21G'
+ sql 'set be_number_for_test=3'
+ sql 'set parallel_fragment_exec_instance_num=8; '
+ sql 'set parallel_pipeline_task_num=8; '
+ sql 'set forbid_unknown_col_stats=true'
+ sql 'set enable_nereids_timeout = false'
+ sql 'set enable_runtime_filter_prune=false'
+ sql 'set runtime_filter_mode=off'
+ sql 'set dump_nereids_memo=false'
+ sql "set disable_join_reorder=true"
+ qt_cte_filter_pushdown_3 """
+ explain shape plan
+ with tmp as (
+ select
+ k1,
+ k3,
+ sum(k2) over (partition by l.k1 order by l.k3 ) pay_num
+ from ( select * from nereids_test_query_db.test)l
+ ),
+ tmp2 as (
+ select
+ tt.*
+ from
+ tmp tt join (select k3 from nereids_test_query_db.baseall ) dd
+ on tt.k3=dd.k3
+ )
+ SELECT * from tmp2
+ where k3=0 and k1=1;
+ """
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
