This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new d35a8a24a5 [feature](nereids) push down Project through Limit (#12490)
d35a8a24a5 is described below
commit d35a8a24a5fc03e6dcdb17cde4bd3a76ec888710
Author: Kikyou1997 <[email protected]>
AuthorDate: Tue Sep 13 13:26:12 2022 +0800
[feature](nereids) push down Project through Limit (#12490)
This rule is rewrite project -> limit to limit -> project. The reason is we
could get tree like project -> limit -> project -> other node. If we do not
rewrite it. we could not merge the two project into one. And if we has more
than one project on one node, the second one will overwrite the first one when
translate. Then, be will core dump or return slot cannot find error.
---
.../doris/nereids/jobs/batch/RewriteJob.java | 7 +++-
.../org/apache/doris/nereids/rules/RuleType.java | 1 +
...ject.java => PushdownFilterThroughProject.java} | 2 +-
...oject.java => PushdownProjectThroughLimit.java} | 46 ++++++++++++--------
.../logical/PushdownProjectThroughLimitTest.java | 49 ++++++++++++++++++++++
5 files changed, 84 insertions(+), 21 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/RewriteJob.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/RewriteJob.java
index 5ec029005b..e8d347000d 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/RewriteJob.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/RewriteJob.java
@@ -31,8 +31,9 @@ import
org.apache.doris.nereids.rules.rewrite.logical.MergeConsecutiveProjects;
import org.apache.doris.nereids.rules.rewrite.logical.NormalizeAggregate;
import org.apache.doris.nereids.rules.rewrite.logical.PruneOlapScanPartition;
import org.apache.doris.nereids.rules.rewrite.logical.PushPredicateThroughJoin;
+import
org.apache.doris.nereids.rules.rewrite.logical.PushdownFilterThroughProject;
+import
org.apache.doris.nereids.rules.rewrite.logical.PushdownProjectThroughLimit;
import org.apache.doris.nereids.rules.rewrite.logical.ReorderJoin;
-import org.apache.doris.nereids.rules.rewrite.logical.SwapFilterAndProject;
import com.google.common.collect.ImmutableList;
@@ -54,6 +55,7 @@ public class RewriteJob extends BatchRulesJob {
* 1. Adjust the plan in correlated logicalApply
* so that there are no correlated columns in the subquery.
* 2. Convert logicalApply to a logicalJoin.
+ * TODO: group these rules to make sure the result plan is
what we expected.
*/
.addAll(new
AdjustApplyFromCorrelatToUnCorrelatJob(cascadesContext).rulesJob)
.addAll(new ConvertApplyToJoinJob(cascadesContext).rulesJob)
@@ -65,7 +67,8 @@ public class RewriteJob extends BatchRulesJob {
.add(topDownBatch(ImmutableList.of(new NormalizeAggregate())))
.add(topDownBatch(ImmutableList.of(new ColumnPruning())))
.add(topDownBatch(ImmutableList.of(new
AggregateDisassemble())))
- .add(topDownBatch(ImmutableList.of(new
SwapFilterAndProject())))
+ .add(topDownBatch(ImmutableList.of(new
PushdownProjectThroughLimit())))
+ .add(topDownBatch(ImmutableList.of(new
PushdownFilterThroughProject())))
.add(bottomUpBatch(ImmutableList.of(new
MergeConsecutiveProjects())))
.add(topDownBatch(ImmutableList.of(new
MergeConsecutiveFilters())))
.add(bottomUpBatch(ImmutableList.of(new
MergeConsecutiveLimits())))
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
index 31a028eb9e..78b2a42d33 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
@@ -105,6 +105,7 @@ public enum RuleType {
OLAP_SCAN_PARTITION_PRUNE(RuleTypeClass.REWRITE),
SWAP_FILTER_AND_PROJECT(RuleTypeClass.REWRITE),
LOGICAL_LIMIT_TO_LOGICAL_EMPTY_RELATION_RULE(RuleTypeClass.REWRITE),
+ SWAP_LIMIT_PROJECT(RuleTypeClass.REWRITE),
// exploration rules
TEST_EXPLORATION(RuleTypeClass.EXPLORATION),
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/SwapFilterAndProject.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughProject.java
similarity index 96%
copy from
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/SwapFilterAndProject.java
copy to
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughProject.java
index 7d6b0e46f7..aab04a0ac6 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/SwapFilterAndProject.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughProject.java
@@ -32,7 +32,7 @@ import org.apache.doris.nereids.util.ExpressionUtils;
* output:
* project(c+d as a, e as b) -> filter(c+d>2, e=0).
*/
-public class SwapFilterAndProject extends OneRewriteRuleFactory {
+public class PushdownFilterThroughProject extends OneRewriteRuleFactory {
@Override
public Rule build() {
return logicalFilter(logicalProject()).then(filter -> {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/SwapFilterAndProject.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownProjectThroughLimit.java
similarity index 55%
rename from
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/SwapFilterAndProject.java
rename to
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownProjectThroughLimit.java
index 7d6b0e46f7..230e5f2e98 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/SwapFilterAndProject.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownProjectThroughLimit.java
@@ -21,29 +21,39 @@ import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory;
import org.apache.doris.nereids.trees.plans.GroupPlan;
-import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
+import org.apache.doris.nereids.trees.plans.logical.LogicalLimit;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
-import org.apache.doris.nereids.util.ExpressionUtils;
/**
- * Push down filter through project.
- * input:
- * filter(a>2, b=0) -> project(c+d as a, e as b)
- * output:
- * project(c+d as a, e as b) -> filter(c+d>2, e=0).
+ * Before:
+ * project
+ * │
+ * ▼
+ * limit
+ * │
+ * ▼
+ * plan node
+ *
+ * After:
+ *
+ * limit
+ * │
+ * ▼
+ * project
+ * │
+ * ▼
+ * plan node
*/
-public class SwapFilterAndProject extends OneRewriteRuleFactory {
+public class PushdownProjectThroughLimit extends OneRewriteRuleFactory {
+
@Override
public Rule build() {
- return logicalFilter(logicalProject()).then(filter -> {
- LogicalProject<GroupPlan> project = filter.child();
- return new LogicalProject<>(
- project.getProjects(),
- new LogicalFilter<>(
- ExpressionUtils.replace(filter.getPredicates(),
project.getSlotToProducer()),
- project.child()
- )
- );
- }).toRule(RuleType.SWAP_FILTER_AND_PROJECT);
+ return logicalProject(logicalLimit(group())).thenApply(ctx -> {
+ LogicalProject<LogicalLimit<GroupPlan>> logicalProject = ctx.root;
+ LogicalLimit<GroupPlan> logicalLimit = logicalProject.child();
+ return new
LogicalLimit<LogicalProject<GroupPlan>>(logicalLimit.getLimit(),
+ logicalLimit.getOffset(), new
LogicalProject<>(logicalProject.getProjects(),
+ logicalLimit.child()));
+ }).toRule(RuleType.SWAP_LIMIT_PROJECT);
}
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownProjectThroughLimitTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownProjectThroughLimitTest.java
new file mode 100644
index 0000000000..602f4c7251
--- /dev/null
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownProjectThroughLimitTest.java
@@ -0,0 +1,49 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.rewrite.logical;
+
+import org.apache.doris.nereids.CascadesContext;
+import org.apache.doris.nereids.trees.expressions.SlotReference;
+import org.apache.doris.nereids.trees.plans.GroupPlan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalLimit;
+import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
+import org.apache.doris.nereids.types.IntegerType;
+
+import com.google.common.collect.ImmutableList;
+import mockit.Mocked;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+public class PushdownProjectThroughLimitTest {
+
+ @Mocked
+ private CascadesContext cascadesContext;
+
+ @Test
+ public void testPushdownProjectThroughLimit(@Mocked GroupPlan groupPlan) {
+ SlotReference slotRef = new SlotReference("col1",
IntegerType.INSTANCE);
+ LogicalLimit logicalLimit = new LogicalLimit(1, 1, groupPlan);
+ LogicalProject logicalProject = new
LogicalProject(ImmutableList.of(slotRef), logicalLimit);
+ PushdownProjectThroughLimit pushdownProjectThroughLimit = new
PushdownProjectThroughLimit();
+ LogicalPlan rewrittenPlan =
+ (LogicalPlan)
pushdownProjectThroughLimit.build().transform(logicalProject,
cascadesContext).get(0);
+ Assertions.assertTrue(rewrittenPlan instanceof LogicalLimit);
+ Assertions.assertTrue(rewrittenPlan.child(0) instanceof
LogicalProject);
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
