This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new 8a6c93d150f branch-3.1: [fix](nereids) stop merge project when
generating huge expression #55293 (#55519)
8a6c93d150f is described below
commit 8a6c93d150f8d442cf5b03497d76daa10dd0fb15
Author: yujun <[email protected]>
AuthorDate: Thu Sep 4 14:38:59 2025 +0800
branch-3.1: [fix](nereids) stop merge project when generating huge
expression #55293 (#55519)
cherry pick from #55293
---
.../nereids/exceptions/AnalysisException.java | 34 ++++++-
.../doris/nereids/exceptions/ParseException.java | 2 +-
.../processor/post/MergeProjectPostProcessor.java | 11 ++-
.../rules/rewrite/DeferMaterializeTopNResult.java | 14 ++-
.../doris/nereids/rules/rewrite/MergeProjects.java | 11 ++-
.../nereids/trees/expressions/Expression.java | 10 +-
.../doris/nereids/trees/plans/algebra/Project.java | 22 +++--
.../apache/doris/nereids/util/ExpressionUtils.java | 4 +-
.../org/apache/doris/nereids/util/PlanUtils.java | 23 ++++-
.../merge_project/test_merge_project.out | Bin 0 -> 346 bytes
.../merge_project/test_merge_project.groovy | 105 +++++++++++++++++++++
11 files changed, 203 insertions(+), 33 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/exceptions/AnalysisException.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/exceptions/AnalysisException.java
index 18f89725e2f..e8b5d142f8f 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/exceptions/AnalysisException.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/exceptions/AnalysisException.java
@@ -23,35 +23,48 @@ import java.util.Optional;
/** Nereids's AnalysisException. */
public class AnalysisException extends RuntimeException {
+ private final ErrorCode errorCode;
private final String message;
private final Optional<Integer> line;
private final Optional<Integer> startPosition;
private final Optional<LogicalPlan> plan;
- public AnalysisException(String message, Throwable cause,
Optional<Integer> line,
+ /** Constructor of AnalysisException. */
+ public AnalysisException(ErrorCode errorCode, String message, Throwable
cause, Optional<Integer> line,
Optional<Integer> startPosition, Optional<LogicalPlan> plan) {
super(message, cause);
+ this.errorCode = errorCode;
this.message = message;
this.line = line;
this.startPosition = startPosition;
this.plan = plan;
}
- public AnalysisException(String message, Optional<Integer> line,
+ /** Constructor of AnalysisException. */
+ public AnalysisException(ErrorCode errorCode, String message,
Optional<Integer> line,
Optional<Integer> startPosition, Optional<LogicalPlan> plan) {
super(message);
+ this.errorCode = errorCode;
this.message = message;
this.line = line;
this.startPosition = startPosition;
this.plan = plan;
}
+ public AnalysisException(ErrorCode errorCode, String message, Throwable
cause) {
+ this(errorCode, message, cause, Optional.empty(), Optional.empty(),
Optional.empty());
+ }
+
+ public AnalysisException(ErrorCode errorCode, String message) {
+ this(errorCode, message, Optional.empty(), Optional.empty(),
Optional.empty());
+ }
+
public AnalysisException(String message, Throwable cause) {
- this(message, cause, Optional.empty(), Optional.empty(),
Optional.empty());
+ this(ErrorCode.NONE, message, cause);
}
public AnalysisException(String message) {
- this(message, Optional.empty(), Optional.empty(), Optional.empty());
+ this(ErrorCode.NONE, message);
}
@Override
@@ -70,5 +83,16 @@ public class AnalysisException extends RuntimeException {
}
}
- // TODO: support ErrorCode
+ /** get error code.
+ */
+ public ErrorCode getErrorCode() {
+ return errorCode;
+ }
+
+ /** error code enum.
+ */
+ public enum ErrorCode {
+ NONE,
+ EXPRESSION_EXCEEDS_LIMIT,
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/exceptions/ParseException.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/exceptions/ParseException.java
index 697af739f89..e3c72f1d6e7 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/exceptions/ParseException.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/exceptions/ParseException.java
@@ -37,7 +37,7 @@ public class ParseException extends AnalysisException {
}
public ParseException(String message, Origin start, Optional<String>
command) {
- super(message, start.line, start.startPosition, Optional.empty());
+ super(ErrorCode.NONE, message, start.line, start.startPosition,
Optional.empty());
this.message = message;
this.start = start;
this.command = command;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/MergeProjectPostProcessor.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/MergeProjectPostProcessor.java
index 7466889fde6..b1f4284fb42 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/MergeProjectPostProcessor.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/MergeProjectPostProcessor.java
@@ -23,6 +23,7 @@ import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalProject;
import java.util.List;
+import java.util.Optional;
/**
* merge consecutive projects
@@ -34,10 +35,12 @@ public class MergeProjectPostProcessor extends
PlanPostProcessor {
project = (PhysicalProject<? extends Plan>) super.visit(project, ctx);
Plan child = project.child();
if (child instanceof PhysicalProject &&
project.canMergeProjections((PhysicalProject) child)) {
- List<NamedExpression> projections =
project.mergeProjections((PhysicalProject) child);
- return (PhysicalProject) project
- .withProjectionsAndChild(projections, child.child(0))
- .copyStatsAndGroupIdFrom(project);
+ Optional<List<NamedExpression>> projections =
project.mergeProjections((PhysicalProject) child);
+ if (projections.isPresent()) {
+ return (PhysicalProject) project
+ .withProjectionsAndChild(projections.get(),
child.child(0))
+ .copyStatsAndGroupIdFrom(project);
+ }
}
return project;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/DeferMaterializeTopNResult.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/DeferMaterializeTopNResult.java
index 765081ae016..18ff8bfeb2f 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/DeferMaterializeTopNResult.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/DeferMaterializeTopNResult.java
@@ -187,8 +187,11 @@ public class DeferMaterializeTopNResult implements
RewriteRuleFactory {
).when(project ->
project.canMergeProjections(project.child().child()))).then(r -> {
LogicalProject<?> upperProject = r.child();
LogicalProject<LogicalOlapScan> bottomProject =
r.child().child().child();
- List<NamedExpression> projections =
upperProject.mergeProjections(bottomProject);
- LogicalProject<?> project =
upperProject.withProjects(projections);
+ Optional<List<NamedExpression>> projections =
upperProject.mergeProjections(bottomProject);
+ if (!projections.isPresent()) {
+ return null;
+ }
+ LogicalProject<?> project =
upperProject.withProjects(projections.get());
return deferMaterialize(r, r.child().child(),
Optional.of(project),
Optional.empty(), bottomProject.child());
})
@@ -246,8 +249,11 @@ public class DeferMaterializeTopNResult implements
RewriteRuleFactory {
).when(project ->
project.canMergeProjections(project.child().child()))).then(r -> {
LogicalProject<?> upperProject = r.child();
LogicalProject<LogicalFilter<LogicalOlapScan>>
bottomProject = r.child().child().child();
- List<NamedExpression> projections =
upperProject.mergeProjections(bottomProject);
- LogicalProject<?> project =
upperProject.withProjects(projections);
+ Optional<List<NamedExpression>> projections =
upperProject.mergeProjections(bottomProject);
+ if (!projections.isPresent()) {
+ return null;
+ }
+ LogicalProject<?> project =
upperProject.withProjects(projections.get());
LogicalFilter<LogicalOlapScan> filter =
bottomProject.child();
return deferMaterialize(r, r.child().child(),
Optional.of(project),
Optional.of(filter), filter.child());
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MergeProjects.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MergeProjects.java
index bb6ca154136..572ff88ea59 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MergeProjects.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MergeProjects.java
@@ -24,6 +24,7 @@ import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
import java.util.List;
+import java.util.Optional;
/**
* this rule aims to merge consecutive project. For example:
@@ -43,13 +44,17 @@ public class MergeProjects extends OneRewriteRuleFactory {
// here we just don't merge two projects if there is any window
function
return logicalProject(logicalProject())
.when(project -> project.canMergeProjections(project.child()))
-
.then(MergeProjects::mergeProjects).toRule(RuleType.MERGE_PROJECTS);
+ .then(MergeProjects::mergeProjects)
+ .toRule(RuleType.MERGE_PROJECTS);
}
/** merge projects */
public static Plan mergeProjects(LogicalProject<?> project) {
LogicalProject<? extends Plan> childProject = (LogicalProject<?>)
project.child();
- List<NamedExpression> projectExpressions =
project.mergeProjections(childProject);
- return project.withProjectsAndChild(projectExpressions,
childProject.child(0));
+ Optional<List<NamedExpression>> projectExpressions =
project.mergeProjections(childProject);
+ if (!projectExpressions.isPresent()) {
+ return project;
+ }
+ return project.withProjectsAndChild(projectExpressions.get(),
childProject.child(0));
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Expression.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Expression.java
index 90cdfda818d..707bf05bb8e 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Expression.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Expression.java
@@ -21,6 +21,7 @@ import org.apache.doris.common.Config;
import org.apache.doris.nereids.analyzer.Unbound;
import org.apache.doris.nereids.analyzer.UnboundVariable;
import org.apache.doris.nereids.exceptions.AnalysisException;
+import org.apache.doris.nereids.exceptions.AnalysisException.ErrorCode;
import org.apache.doris.nereids.exceptions.UnboundException;
import org.apache.doris.nereids.trees.AbstractTreeNode;
import
org.apache.doris.nereids.trees.expressions.ArrayItemReference.ArrayItemSlot;
@@ -179,12 +180,13 @@ public abstract class Expression extends
AbstractTreeNode<Expression> implements
private void checkLimit() {
if (depth > Config.expr_depth_limit) {
- throw new AnalysisException(String.format("Exceeded the maximum
depth of an "
- + "expression tree (%s).", Config.expr_depth_limit));
+ throw new AnalysisException(ErrorCode.EXPRESSION_EXCEEDS_LIMIT,
+ String.format("Exceeded the maximum depth of an expression
tree (%s).", Config.expr_depth_limit));
}
if (width > Config.expr_children_limit) {
- throw new AnalysisException(String.format("Exceeded the maximum
children of an "
- + "expression tree (%s).", Config.expr_children_limit));
+ throw new AnalysisException(ErrorCode.EXPRESSION_EXCEEDS_LIMIT,
+ String.format("Exceeded the maximum children of an
expression tree (%s).",
+ Config.expr_children_limit));
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/Project.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/Project.java
index 33b351ba4b3..eeacd02468d 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/Project.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/Project.java
@@ -26,12 +26,13 @@ import
org.apache.doris.nereids.trees.expressions.functions.NoneMovableFunction;
import org.apache.doris.nereids.util.ExpressionUtils;
import org.apache.doris.nereids.util.PlanUtils;
+import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
-import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
/**
* Common interface for logical/physical project.
@@ -55,23 +56,30 @@ public interface Project {
}
/**
- * combine upper level and bottom level projections
+ * combine upper level and bottom level projections.
+ * if the combined expressions too huge, will return empty.
* 1. alias combination, for example
* proj(x as y, b) --> proj(a as x, b, c) =>(a as y, b)
* 2. remove used projection in bottom project
* @param childProject bottom project
* @return project list for merged project
*/
- default List<NamedExpression> mergeProjections(Project childProject) {
- List<NamedExpression> projects = new ArrayList<>();
- projects.addAll(PlanUtils.mergeProjections(childProject.getProjects(),
getProjects()));
+ default Optional<List<NamedExpression>> mergeProjections(Project
childProject) {
+ Optional<List<NamedExpression>> parentProjectsOpt
+ = PlanUtils.tryMergeProjections(childProject.getProjects(),
getProjects());
+ if (!parentProjectsOpt.isPresent()) {
+ return Optional.empty();
+ }
+ ImmutableList.Builder<NamedExpression> projectsBuilder
+ =
ImmutableList.builderWithExpectedSize(parentProjectsOpt.get().size());
+ projectsBuilder.addAll(parentProjectsOpt.get());
for (NamedExpression expression : childProject.getProjects()) {
// keep NoneMovableFunction for later use
if (expression.containsType(NoneMovableFunction.class)) {
- projects.add(expression);
+ projectsBuilder.add(expression);
}
}
- return projects;
+ return Optional.of(projectsBuilder.build());
}
/** check can merge two projects */
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java
index 5c5a4e8d060..f05060d58a7 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java
@@ -368,7 +368,7 @@ public class ExpressionUtils {
/**
* Generate replaceMap Slot -> Expression from NamedExpression[Expression
as name]
*/
- public static Map<Slot, Expression>
generateReplaceMap(List<NamedExpression> namedExpressions) {
+ public static Map<Slot, Expression> generateReplaceMap(List<? extends
NamedExpression> namedExpressions) {
Map<Slot, Expression> replaceMap =
Maps.newLinkedHashMapWithExpectedSize(namedExpressions.size());
for (NamedExpression namedExpression : namedExpressions) {
if (namedExpression instanceof Alias) {
@@ -457,7 +457,7 @@ public class ExpressionUtils {
/**
* Replace expression node in the expression tree by `replaceMap` in
top-down manner.
*/
- public static List<NamedExpression>
replaceNamedExpressions(List<NamedExpression> namedExpressions,
+ public static List<NamedExpression> replaceNamedExpressions(List<? extends
NamedExpression> namedExpressions,
Map<? extends Expression, ? extends Expression> replaceMap) {
Builder<NamedExpression> replaceExprs =
ImmutableList.builderWithExpectedSize(namedExpressions.size());
for (NamedExpression namedExpression : namedExpressions) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/PlanUtils.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/PlanUtils.java
index 9ab1cf0ae1c..0e0a39d01c5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/PlanUtils.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/PlanUtils.java
@@ -18,6 +18,7 @@
package org.apache.doris.nereids.util;
import org.apache.doris.catalog.TableIf;
+import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.trees.expressions.ComparisonPredicate;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
@@ -120,10 +121,26 @@ public class PlanUtils {
}
/**
- * merge childProjects with parentProjects
+ * try merge childProjects with parentProjects. if merged expression
exceeds limit, return empty.
*/
- public static List<NamedExpression> mergeProjections(List<NamedExpression>
childProjects,
- List<NamedExpression> parentProjects) {
+ public static Optional<List<NamedExpression>> tryMergeProjections(List<?
extends NamedExpression> childProjects,
+ List<? extends NamedExpression> parentProjects) {
+ try {
+ return Optional.of(mergeProjections(childProjects,
parentProjects));
+ } catch (AnalysisException e) {
+ if (e.getErrorCode() ==
AnalysisException.ErrorCode.EXPRESSION_EXCEEDS_LIMIT) {
+ return Optional.empty();
+ } else {
+ throw e;
+ }
+ }
+ }
+
+ /**
+ * merge childProjects with parentProjects. if merged expression exceeds
limit, will throw AnalysisException.
+ */
+ public static List<NamedExpression> mergeProjections(List<? extends
NamedExpression> childProjects,
+ List<? extends NamedExpression> parentProjects) {
Map<Slot, Expression> replaceMap =
ExpressionUtils.generateReplaceMap(childProjects);
return ExpressionUtils.replaceNamedExpressions(parentProjects,
replaceMap);
}
diff --git
a/regression-test/data/nereids_rules_p0/merge_project/test_merge_project.out
b/regression-test/data/nereids_rules_p0/merge_project/test_merge_project.out
new file mode 100644
index 00000000000..56c097d00f8
Binary files /dev/null and
b/regression-test/data/nereids_rules_p0/merge_project/test_merge_project.out
differ
diff --git
a/regression-test/suites/nereids_rules_p0/merge_project/test_merge_project.groovy
b/regression-test/suites/nereids_rules_p0/merge_project/test_merge_project.groovy
new file mode 100644
index 00000000000..f522dcdfa7d
--- /dev/null
+++
b/regression-test/suites/nereids_rules_p0/merge_project/test_merge_project.groovy
@@ -0,0 +1,105 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+// OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite('test_merge_project', 'nonConcurrent') {
+ setFeConfigTemporary([expr_children_limit : 200]) {
+ def tbl = 'tbl_test_merge_project'
+ multi_sql """
+ SET ignore_shape_nodes='PhysicalDistribute';
+ drop table if exists ${tbl} force;
+ create table ${tbl} (a int, b int) properties('replication_num' =
'1');
+ insert into ${tbl} values (2, 1), (4, 3);
+ """
+
+ explainAndOrderResult 'exceeds_expression_limit', """
+ select
+ case
+ when k15 > k16 then k15 + k16 + 1
+ when k15 > k16 - 1 then k15 + k16 - 1
+ when k15 > k16 - 2 then k15 + k16 - 2
+ else 0
+ end as k17,
+ k15 + k16 as k18
+ from
+ (select
+ case
+ when k13 > k14 then k13 + k14 + 1
+ when k13 > k14 - 1 then k13 + k14 - 1
+ when k13 > k14 - 2 then k13 + k14 - 2
+ else 0
+ end as k15,
+ k13 + k14 as k16
+ from
+ (select
+ case
+ when k11 > k12 then k11 + k12 + 1
+ when k11 > k12 - 1 then k11 + k12 - 1
+ when k11 > k12 - 2 then k11 + k12 - 2
+ else 0
+ end as k13,
+ k11 + k12 as k14
+ from
+ (select
+ case
+ when k9 > k10 then k9 + k10 + 1
+ when k9 > k10 - 1 then k9 + k10 - 1
+ when k9 > k10 - 2 then k9 + k10 - 2
+ else 0
+ end as k11,
+ k9 + k10 as k12
+ from
+ (select
+ case
+ when k7 > k8 then k7 + k8 + 1
+ when k7 > k8 - 1 then k7 + k8 - 1
+ when k7 > k8 - 2 then k7 + k8 - 2
+ else 0
+ end as k9,
+ k7 + k8 as k10
+ from
+ (select
+ case
+ when k5 > k6 then k5 + k6 + 1
+ when k5 > k6 - 1 then k5 + k6 - 1
+ when k5 > k6 - 2 then k5 + k6 - 2
+ else 0
+ end as k7,
+ k5 + k6 as k8
+ from
+ (select
+ case
+ when k3 > k4 then k3 + k4 + 1
+ when k3 > k4 - 1 then k3 + k4 - 1
+ when k3 > k4 - 2 then k3 + k4 - 2
+ else 0
+ end as k5,
+ k3 + k4 as k6
+ from
+ (select
+ case
+ when k1 > k2 then k1 + k2 + 1
+ when k1 > k2 - 1 then k1 + k2 - 1
+ when k1 > k2 - 2 then k1 + k2 - 2
+ else 0
+ end as k3,
+ k1 + k2 as k4
+ from
+ (select a as k1, b as k2
+ from ${tbl}) t1) t2) t3) t4) t5) t6) t7) t8;
+ """
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
