This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 0c39b888048 [Improvement](expr) fold child when const expr not folded
(#38493) (#38961)
0c39b888048 is described below
commit 0c39b8880486a991efa9c73b84d98620af570a98
Author: LiBinfeng <[email protected]>
AuthorDate: Mon Aug 12 15:13:48 2024 +0800
[Improvement](expr) fold child when const expr not folded (#38493) (#38961)
cherry-pick from master 38493
1. fold child when const expr not folded
2. do not fold function `sleep`
3. move all exceptional expression into shouldSkipFold
before
mysql [test]>explain select sleep(sign(1)*100);
+-----------------------------------------------+
| Explain String(Nereids Planner) |
+-----------------------------------------------+
| PLAN FRAGMENT 0 |
| OUTPUT EXPRS: |
| sleep(cast((sign(1.0) * 100) as INT))[#0] |
| PARTITION: UNPARTITIONED |
| |
| HAS_COLO_PLAN_NODE: false |
| |
| VRESULT SINK |
| MYSQL_PROTOCAL |
| |
| 0:VUNION(32) |
| constant exprs: |
| sleep(CAST((sign(1) * 100) AS int)) |
+-----------------------------------------------+
13 rows in set (15.02 sec)
mysql [test]>select sleep(sign(1)*100);
+-----------------------------------------------------+
| sleep(cast((sign(cast(1 as DOUBLE)) * 100) as INT)) |
+-----------------------------------------------------+
| 1 |
+-----------------------------------------------------+
1 row in set (1 min 55.34 sec)
after
mysql [test]>explain select sleep(sign(1)*100);
+---------------------------------+
| Explain String(Nereids Planner) |
+---------------------------------+
| PLAN FRAGMENT 0 |
| OUTPUT EXPRS: |
| sleep(100)[#0] |
| PARTITION: UNPARTITIONED |
| |
| HAS_COLO_PLAN_NODE: false |
| |
| VRESULT SINK |
| MYSQL_PROTOCAL |
| |
| 0:VUNION(32) |
| constant exprs: |
| sleep(100) |
+---------------------------------+
13 rows in set (0.23 sec)
mysql [test]> select sleep(sign(1)*100);
+-----------------------------------------------------+
| sleep(cast((sign(cast(1 as DOUBLE)) * 100) as INT)) |
+-----------------------------------------------------+
| 1 |
+-----------------------------------------------------+
1 row in set (1 min 40.37 sec)
Co-authored-by: Pxl <[email protected]>
---
.../expression/rules/FoldConstantRuleOnBE.java | 102 ++++++++++++---------
.../fold_constant/fold_constant_by_be.out | 18 ++++
.../fold_constant/fold_constant_by_be.groovy | 11 ++-
.../aggregate/select_random_distributed_tbl.groovy | 3 +-
4 files changed, 89 insertions(+), 45 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java
index 706d384be53..7b6446aaf37 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java
@@ -37,7 +37,10 @@ import
org.apache.doris.nereids.trees.expressions.ArrayItemReference;
import org.apache.doris.nereids.trees.expressions.Cast;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.Match;
+import org.apache.doris.nereids.trees.expressions.functions.BoundFunction;
import
org.apache.doris.nereids.trees.expressions.functions.generator.TableGeneratingFunction;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.NonNullable;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.Nullable;
import org.apache.doris.nereids.trees.expressions.functions.scalar.Sleep;
import org.apache.doris.nereids.trees.expressions.literal.ArrayLiteral;
import org.apache.doris.nereids.trees.expressions.literal.BigIntLiteral;
@@ -55,7 +58,6 @@ import
org.apache.doris.nereids.trees.expressions.literal.LargeIntLiteral;
import org.apache.doris.nereids.trees.expressions.literal.Literal;
import org.apache.doris.nereids.trees.expressions.literal.MapLiteral;
import org.apache.doris.nereids.trees.expressions.literal.NullLiteral;
-import org.apache.doris.nereids.trees.expressions.literal.NumericLiteral;
import org.apache.doris.nereids.trees.expressions.literal.SmallIntLiteral;
import org.apache.doris.nereids.trees.expressions.literal.StringLiteral;
import org.apache.doris.nereids.trees.expressions.literal.StructLiteral;
@@ -157,7 +159,11 @@ public class FoldConstantRuleOnBE implements
ExpressionPatternRuleFactory {
Expression root, Map<String, Expression> constMap, Map<String,
Expression> resultMap) {
for (Entry<String, Expression> entry : constMap.entrySet()) {
if (entry.getValue().equals(root)) {
- return resultMap.get(entry.getKey());
+ if (resultMap.containsKey(entry.getKey())) {
+ return resultMap.get(entry.getKey());
+ } else {
+ return root;
+ }
}
}
List<Expression> newChildren = new ArrayList<>();
@@ -174,38 +180,7 @@ public class FoldConstantRuleOnBE implements
ExpressionPatternRuleFactory {
private static void collectConst(Expression expr, Map<String, Expression>
constMap,
Map<String, TExpr> tExprMap, IdGenerator<ExprId> idGenerator) {
- if (expr.isConstant()) {
- // Do not constant fold cast(null as dataType) because we cannot
preserve the
- // cast-to-types and that can lead to query failures, e.g., CTAS
- if (expr instanceof Cast) {
- if (((Cast) expr).child().isNullLiteral()) {
- return;
- }
- if (skipSleepFunction(((Cast) expr).child())) {
- return;
- }
- }
- // skip literal expr
- if (expr.isLiteral()) {
- return;
- }
- // eg: avg_state(1) return is agg function serialize data
- // and some type can't find a literal to represent.
- // time type: need add a time literal in nereids
- // IPv6 type: need get a library to output the compressed address
format
- if (expr.getDataType().isAggStateType() ||
expr.getDataType().isObjectType()
- || expr.getDataType().isVariantType() ||
expr.getDataType().isTimeLikeType()
- || expr.getDataType().isIPv6Type()) {
- return;
- }
- // first need pass PlanTranslatorContext value,
- // and ArrayItemReference translate, can't findColumnRef
- // Match need give more info rather then as left child a NULL, in
- // match_phrase_prefix/MATCH_PHRASE/MATCH_PHRASE/MATCH_ANY
- if (skipSleepFunction(expr) || (expr instanceof
TableGeneratingFunction)
- || (expr instanceof ArrayItemReference) || (expr
instanceof Match)) {
- return;
- }
+ if (expr.isConstant() && !shouldSkipFold(expr)) {
String id = idGenerator.getNextId().toString();
constMap.put(id, expr);
Expr staleExpr;
@@ -229,17 +204,58 @@ public class FoldConstantRuleOnBE implements
ExpressionPatternRuleFactory {
}
}
- // if sleep(5) will cause rpc timeout
- private static boolean skipSleepFunction(Expression expr) {
+ // Some expressions should not do constant folding
+ private static boolean shouldSkipFold(Expression expr) {
+ // Skip literal expr
+ if (expr.isLiteral()) {
+ return true;
+ }
+
+ // Frontend can not represent those types
+ if (expr.getDataType().isAggStateType() ||
expr.getDataType().isObjectType()
+ || expr.getDataType().isVariantType() ||
expr.getDataType().isTimeLikeType()
+ || expr.getDataType().isIPv6Type()) {
+ return true;
+ }
+
+ // Frontend can not represent geo types
+ if (expr instanceof BoundFunction && ((BoundFunction)
expr).getName().toLowerCase().startsWith("st_")) {
+ return true;
+ }
+
+ // TableGeneratingFunction need pass PlanTranslatorContext value
+ if (expr instanceof TableGeneratingFunction) {
+ return true;
+ }
+
+ // ArrayItemReference translate can't findColumnRef
+ if (expr instanceof ArrayItemReference) {
+ return true;
+ }
+
+ // Match need give more info rather then as left child a NULL, in
+ // match_phrase_prefix/MATCH_PHRASE/MATCH_PHRASE/MATCH_ANY
+ if (expr instanceof Match) {
+ return true;
+ }
+
+ // sleep will cause rpc timeout
if (expr instanceof Sleep) {
- Expression param = expr.child(0);
- if (param instanceof Cast) {
- param = param.child(0);
- }
- if (param instanceof NumericLiteral) {
- return ((NumericLiteral) param).getDouble() >= 5.0;
- }
+ return true;
+ }
+
+ // Do not constant fold cast(null as dataType) because we cannot
preserve the
+ // cast-to-types and that can lead to query failures, e.g., CTAS
+ if (expr instanceof Cast && ((Cast) expr).child().isNullLiteral()) {
+ return true;
}
+
+ // This kind of function is often used to change the attributes of
columns.
+ // Folding will make it impossible to construct columns such as
nullable(1).
+ if (expr instanceof Nullable || expr instanceof NonNullable) {
+ return true;
+ }
+
return false;
}
diff --git
a/regression-test/data/nereids_p0/expression/fold_constant/fold_constant_by_be.out
b/regression-test/data/nereids_p0/expression/fold_constant/fold_constant_by_be.out
index 4ec7c146d7a..db738cf543f 100644
---
a/regression-test/data/nereids_p0/expression/fold_constant/fold_constant_by_be.out
+++
b/regression-test/data/nereids_p0/expression/fold_constant/fold_constant_by_be.out
@@ -2,3 +2,21 @@
-- !sql_1 --
80000
+-- !sql --
+PLAN FRAGMENT 0
+ OUTPUT EXPRS:
+ sleep(100)[#0]
+ PARTITION: UNPARTITIONED
+
+ HAS_COLO_PLAN_NODE: false
+
+ VRESULT SINK
+ MYSQL_PROTOCAL
+
+ 0:VUNION(32)
+ constant exprs:
+ sleep(100)
+
+-- !sql --
+true
+
diff --git
a/regression-test/suites/nereids_p0/expression/fold_constant/fold_constant_by_be.groovy
b/regression-test/suites/nereids_p0/expression/fold_constant/fold_constant_by_be.groovy
index e1162c662b9..10882287ad7 100644
---
a/regression-test/suites/nereids_p0/expression/fold_constant/fold_constant_by_be.groovy
+++
b/regression-test/suites/nereids_p0/expression/fold_constant/fold_constant_by_be.groovy
@@ -40,4 +40,13 @@ suite("fold_constant_by_be") {
sql """ INSERT INTO str_tb VALUES (2, repeat("test1111", 10000)); """
qt_sql_1 """ select length(v1) from str_tb; """
-}
\ No newline at end of file
+
+ def res1 = sql " select /*+SET_VAR(enable_fold_constant_by_be=true)*/
ST_CIRCLE(121.510651, 31.234391, 1918.0); "
+ def res2 = sql " select /*+SET_VAR(enable_fold_constant_by_be=false)*/
ST_CIRCLE(121.510651, 31.234391, 1918.0); "
+ log.info("result: {}, {}", res1, res2)
+ assertEquals(res1[0][0], res2[0][0])
+
+ qt_sql "explain select sleep(sign(1)*100);"
+ sql 'set query_timeout=12;'
+ qt_sql "select sleep(sign(1)*10);"
+}
diff --git
a/regression-test/suites/query_p0/aggregate/select_random_distributed_tbl.groovy
b/regression-test/suites/query_p0/aggregate/select_random_distributed_tbl.groovy
index 439559aa5e0..983c625c1ef 100644
---
a/regression-test/suites/query_p0/aggregate/select_random_distributed_tbl.groovy
+++
b/regression-test/suites/query_p0/aggregate/select_random_distributed_tbl.groovy
@@ -41,8 +41,9 @@ suite("select_random_distributed_tbl") {
"replication_allocation" = "tag.location.default: 1"
);
"""
-
+ sql 'set enable_fold_constant_by_be=true'
sql """ insert into ${tableName}
values(1,"a",1,1,1,avg_state(1),hll_hash(1),bitmap_hash(1),to_quantile_state(1,
2048)) """
+ sql 'set enable_fold_constant_by_be=false'
sql """ insert into ${tableName}
values(1,"a",2,2,2,avg_state(2),hll_hash(2),bitmap_hash(2),to_quantile_state(2,
2048)) """
sql """ insert into ${tableName}
values(1,"a",3,3,3,avg_state(3),hll_hash(3),bitmap_hash(3),to_quantile_state(3,
2048)) """
sql """ insert into ${tableName}
values(2,"b",4,4,4,avg_state(4),hll_hash(4),bitmap_hash(4),to_quantile_state(4,
2048)) """
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
