This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new 1647702c0ce [fix](Nereids) prune not required window expressions on
window operator (#35577)
1647702c0ce is described below
commit 1647702c0ce03b44d2ff68490dc50baf6ef0ad78
Author: morrySnow <[email protected]>
AuthorDate: Wed May 29 13:59:54 2024 +0800
[fix](Nereids) prune not required window expressions on window operator
(#35577)
pick from master #35504
if window expression is not required by its parent, we should prune this
column.
If all window expressions of window operator are pruned, we remove this
window operator directly.
---
.../nereids/rules/rewrite/AdjustNullable.java | 2 +-
.../doris/nereids/rules/rewrite/ColumnPruning.java | 25 +++++++++
.../nereids/trees/plans/logical/LogicalWindow.java | 5 +-
.../column_pruning/window_column_pruning.groovy | 59 ++++++++++++++++++++++
4 files changed, 87 insertions(+), 4 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AdjustNullable.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AdjustNullable.java
index 26e9403a006..c5c9b8520fe 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AdjustNullable.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AdjustNullable.java
@@ -223,7 +223,7 @@ public class AdjustNullable extends
DefaultPlanRewriter<Map<ExprId, Slot>> imple
List<NamedExpression> windowExpressions =
updateExpressions(window.getWindowExpressions(), replaceMap);
windowExpressions.forEach(w -> replaceMap.put(w.getExprId(),
w.toSlot()));
- return window.withExpression(windowExpressions, window.child());
+ return window.withExpressionsAndChild(windowExpressions,
window.child());
}
@Override
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ColumnPruning.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ColumnPruning.java
index ad95f1a552f..c759ada4a0c 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ColumnPruning.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ColumnPruning.java
@@ -34,6 +34,7 @@ import
org.apache.doris.nereids.trees.plans.logical.LogicalProject;
import org.apache.doris.nereids.trees.plans.logical.LogicalRepeat;
import org.apache.doris.nereids.trees.plans.logical.LogicalSink;
import org.apache.doris.nereids.trees.plans.logical.LogicalUnion;
+import org.apache.doris.nereids.trees.plans.logical.LogicalWindow;
import org.apache.doris.nereids.trees.plans.logical.OutputPrunable;
import org.apache.doris.nereids.trees.plans.visitor.CustomRewriter;
import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanRewriter;
@@ -170,6 +171,30 @@ public class ColumnPruning extends
DefaultPlanRewriter<PruneContext> implements
return pruneAggregate(repeat, context);
}
+ @Override
+ public Plan visitLogicalWindow(LogicalWindow<? extends Plan> window,
PruneContext context) {
+ boolean pruned = false;
+ boolean reserved = false;
+ ImmutableList.Builder<NamedExpression> reservedWindowExpressions =
ImmutableList.builder();
+ for (NamedExpression windowExpression : window.getWindowExpressions())
{
+ if (context.requiredSlots.contains(windowExpression.toSlot())) {
+ reservedWindowExpressions.add(windowExpression);
+ reserved = true;
+ } else {
+ pruned = true;
+ }
+ }
+ if (!pruned) {
+ return pruneChildren(window, context.requiredSlots);
+ }
+ if (!reserved) {
+ return window.child().accept(this, context);
+ }
+ LogicalWindow<? extends Plan> prunedWindow
+ =
window.withExpressionsAndChild(reservedWindowExpressions.build(),
window.child());
+ return pruneChildren(prunedWindow, context.requiredSlots);
+ }
+
private Plan pruneAggregate(Aggregate agg, PruneContext context) {
// first try to prune group by and aggregate functions
Aggregate prunedOutputAgg = pruneOutput(agg, agg.getOutputs(),
agg::pruneOutputs, context);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalWindow.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalWindow.java
index 67e18197796..13fc55796e9 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalWindow.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalWindow.java
@@ -88,9 +88,8 @@ public class LogicalWindow<CHILD_TYPE extends Plan> extends
LogicalUnary<CHILD_T
return windowExpressions;
}
- public LogicalWindow<Plan> withExpression(List<NamedExpression>
windowExpressions, Plan child) {
- return new LogicalWindow<>(windowExpressions, isChecked,
Optional.empty(),
- Optional.empty(), child);
+ public LogicalWindow<Plan> withExpressionsAndChild(List<NamedExpression>
windowExpressions, Plan child) {
+ return new LogicalWindow<>(windowExpressions, isChecked, child);
}
public LogicalWindow<Plan> withChecked(List<NamedExpression>
windowExpressions, Plan child) {
diff --git
a/regression-test/suites/nereids_rules_p0/column_pruning/window_column_pruning.groovy
b/regression-test/suites/nereids_rules_p0/column_pruning/window_column_pruning.groovy
new file mode 100644
index 00000000000..5cbaae7c9e7
--- /dev/null
+++
b/regression-test/suites/nereids_rules_p0/column_pruning/window_column_pruning.groovy
@@ -0,0 +1,59 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("window_column_pruning") {
+ sql "set disable_nereids_rules=PRUNE_EMPTY_PARTITION"
+
+ sql """
+ DROP TABLE IF EXISTS window_column_pruning
+ """
+ sql """
+ CREATE TABLE IF NOT EXISTS window_column_pruning(
+ `id` int NULL,
+ `c1` int NULL
+ ) ENGINE = OLAP
+ DISTRIBUTED BY HASH(id) BUCKETS 4
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1"
+ );
+ """
+
+ // should prune
+ explain {
+ sql "select id from (select id, row_number() over (partition by id) as
rn from window_column_pruning) tmp where id > 1;"
+ notContains "row_number"
+ }
+
+ // should not prune
+ explain {
+ sql "select id, rn from (select id, row_number() over (partition by
id) as rn from window_column_pruning) tmp where id > 1;"
+ contains "row_number"
+ }
+
+ // should prune rank, but not prune row_number
+ explain {
+ sql "select id, rn1 from (select id, row_number() over (partition by
id) as rn1, rank() over (partition by id) as rk from window_column_pruning) tmp
where id > 1;"
+ contains "row_number"
+ notContains "rank"
+ }
+
+ // prune through union all
+ explain {
+ sql "select id from (select id, rank() over() px from
window_column_pruning union all select id, rank() over() px from
window_column_pruning) a"
+ notContains "rank"
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
