This is an automated email from the ASF dual-hosted git repository.
huajianlan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 06a1efdb01 [fix](Nerieds) fix tpch and support trace plan's change
event (#13957)
06a1efdb01 is described below
commit 06a1efdb018e6328e1b2e7426f193433602ed721
Author: 924060929 <[email protected]>
AuthorDate: Fri Nov 4 15:01:06 2022 +0800
[fix](Nerieds) fix tpch and support trace plan's change event (#13957)
This pr fix some bugs for run tpc-h
1. fix the avg(decimal) crash the backend. The fix code in
`Avg.getFinalType()` and every child class of `ComputeSinature`
2. fix the ReorderJoin dead loop. The fix code in
`ReorderJoin.findInnerJoin()`
3. fix the TimestampArithmetic can not bind the functions in the child. The
fix code in `BindFunction.FunctionBinder.visitTimestampArithmetic()`
New feature: support trace the plan's change event, you can `set
enable_nereids_trace=true` to open trace log and see some log like this:
```
2022-11-03 21:07:38,391 INFO (mysql-nio-pool-0|208)
[Job.printTraceLog():128] ========== RewriteBottomUpJob ANALYZE_FILTER_SUBQUERY
==========
before:
LogicalProject ( projects=[S_ACCTBAL#17, S_NAME#13, N_NAME#4, P_PARTKEY#19,
P_MFGR#21, S_ADDRESS#14, S_PHONE#16, S_COMMENT#18] )
+--LogicalFilter ( predicates=((((((((P_PARTKEY#19 = PS_PARTKEY#7) AND
(S_SUPPKEY#12 = PS_SUPPKEY#8)) AND (P_SIZE#24 = 15)) AND (P_TYPE#23 like
'%BRASS')) AND (S_NATIONKEY#15 = N_NATIONKEY#3)) AND (N_REGIONKEY#5 =
R_REGIONKEY#0)) AND (R_NAME#1 = 'EUROPE')) AND (PS_SUPPLYCOST#10 =
(SCALARSUBQUERY) (QueryPlan: LogicalAggregate ( phase=LOCAL,
outputExpr=[min(PS_SUPPLYCOST#31) AS `min(PS_SUPPLYCOST)`#33], groupByExpr=[]
)), (CorrelatedSlots: [P_PARTKEY#19, S_SUPPKEY#12, S_NATIONKEY#15, N [...]
+--LogicalJoin ( type=CROSS_JOIN, hashJoinCondition=[],
otherJoinCondition=[] )
|--LogicalJoin ( type=CROSS_JOIN, hashJoinCondition=[],
otherJoinCondition=[] )
| |--LogicalJoin ( type=CROSS_JOIN, hashJoinCondition=[],
otherJoinCondition=[] )
| | |--LogicalJoin ( type=CROSS_JOIN, hashJoinCondition=[],
otherJoinCondition=[] )
| | | |--LogicalOlapScan (
qualified=default_cluster:regression_test_tpch_sf1_p1_tpch_sf1.part,
output=[P_PARTKEY#19, P_NAME#20, P_MFGR#21, P_BRAND#22, P_TYPE#23, P_SIZE#24,
P_CONTAINER#25, P_RETAILPRICE#26, P_COMMENT#27], candidateIndexIds=[],
selectedIndexId=11076, preAgg=ON )
| | | +--LogicalOlapScan (
qualified=default_cluster:regression_test_tpch_sf1_p1_tpch_sf1.supplier,
output=[S_SUPPKEY#12, S_NAME#13, S_ADDRESS#14, S_NATIONKEY#15, S_PHONE#16,
S_ACCTBAL#17, S_COMMENT#18], candidateIndexIds=[], selectedIndexId=11124,
preAgg=ON )
| | +--LogicalOlapScan (
qualified=default_cluster:regression_test_tpch_sf1_p1_tpch_sf1.partsupp,
output=[PS_PARTKEY#7, PS_SUPPKEY#8, PS_AVAILQTY#9, PS_SUPPLYCOST#10,
PS_COMMENT#11], candidateIndexIds=[], selectedIndexId=11092, preAgg=ON )
| +--LogicalOlapScan (
qualified=default_cluster:regression_test_tpch_sf1_p1_tpch_sf1.nation,
output=[N_NATIONKEY#3, N_NAME#4, N_REGIONKEY#5, N_COMMENT#6],
candidateIndexIds=[], selectedIndexId=11044, preAgg=ON )
+--LogicalOlapScan (
qualified=default_cluster:regression_test_tpch_sf1_p1_tpch_sf1.region,
output=[R_REGIONKEY#0, R_NAME#1, R_COMMENT#2], candidateIndexIds=[],
selectedIndexId=11108, preAgg=ON )
after:
LogicalProject ( projects=[S_ACCTBAL#17, S_NAME#13, N_NAME#4, P_PARTKEY#19,
P_MFGR#21, S_ADDRESS#14, S_PHONE#16, S_COMMENT#18] )
+--LogicalFilter ( predicates=((((((((P_PARTKEY#19 = PS_PARTKEY#7) AND
(S_SUPPKEY#12 = PS_SUPPKEY#8)) AND (P_SIZE#24 = 15)) AND (P_TYPE#23 like
'%BRASS')) AND (S_NATIONKEY#15 = N_NATIONKEY#3)) AND (N_REGIONKEY#5 =
R_REGIONKEY#0)) AND (R_NAME#1 = 'EUROPE')) AND (PS_SUPPLYCOST#10 =
min(PS_SUPPLYCOST)#33)) )
+--LogicalProject ( projects=[P_PARTKEY#19, P_NAME#20, P_MFGR#21,
P_BRAND#22, P_TYPE#23, P_SIZE#24, P_CONTAINER#25, P_RETAILPRICE#26,
P_COMMENT#27, S_SUPPKEY#12, S_NAME#13, S_ADDRESS#14, S_NATIONKEY#15,
S_PHONE#16, S_ACCTBAL#17, S_COMMENT#18, PS_PARTKEY#7, PS_SUPPKEY#8,
PS_AVAILQTY#9, PS_SUPPLYCOST#10, PS_COMMENT#11, N_NATIONKEY#3, N_NAME#4,
N_REGIONKEY#5, N_COMMENT#6, R_REGIONKEY#0, R_NAME#1, R_COMMENT#2,
min(PS_SUPPLYCOST)#33] )
+--LogicalApply ( correlationSlot=[P_PARTKEY#19, S_SUPPKEY#12,
S_NATIONKEY#15, N_NATIONKEY#3, N_REGIONKEY#5, R_REGIONKEY#0, R_NAME#1],
correlationFilter=Optional.empty )
|--LogicalJoin ( type=CROSS_JOIN, hashJoinCondition=[],
otherJoinCondition=[] )
| |--LogicalJoin ( type=CROSS_JOIN, hashJoinCondition=[],
otherJoinCondition=[] )
| | |--LogicalJoin ( type=CROSS_JOIN, hashJoinCondition=[],
otherJoinCondition=[] )
| | | |--LogicalJoin ( type=CROSS_JOIN, hashJoinCondition=[],
otherJoinCondition=[] )
| | | | |--LogicalOlapScan (
qualified=default_cluster:regression_test_tpch_sf1_p1_tpch_sf1.part,
output=[P_PARTKEY#19, P_NAME#20, P_MFGR#21, P_BRAND#22, P_TYPE#23, P_SIZE#24,
P_CONTAINER#25, P_RETAILPRICE#26, P_COMMENT#27], candidateIndexIds=[],
selectedIndexId=11076, preAgg=ON )
| | | | +--LogicalOlapScan (
qualified=default_cluster:regression_test_tpch_sf1_p1_tpch_sf1.supplier,
output=[S_SUPPKEY#12, S_NAME#13, S_ADDRESS#14, S_NATIONKEY#15, S_PHONE#16,
S_ACCTBAL#17, S_COMMENT#18], candidateIndexIds=[], selectedIndexId=11124,
preAgg=ON )
| | | +--LogicalOlapScan (
qualified=default_cluster:regression_test_tpch_sf1_p1_tpch_sf1.partsupp,
output=[PS_PARTKEY#7, PS_SUPPKEY#8, PS_AVAILQTY#9, PS_SUPPLYCOST#10,
PS_COMMENT#11], candidateIndexIds=[], selectedIndexId=11092, preAgg=ON )
| | +--LogicalOlapScan (
qualified=default_cluster:regression_test_tpch_sf1_p1_tpch_sf1.nation,
output=[N_NATIONKEY#3, N_NAME#4, N_REGIONKEY#5, N_COMMENT#6],
candidateIndexIds=[], selectedIndexId=11044, preAgg=ON )
| +--LogicalOlapScan (
qualified=default_cluster:regression_test_tpch_sf1_p1_tpch_sf1.region,
output=[R_REGIONKEY#0, R_NAME#1, R_COMMENT#2], candidateIndexIds=[],
selectedIndexId=11108, preAgg=ON )
+--LogicalAggregate ( phase=LOCAL,
outputExpr=[min(PS_SUPPLYCOST#31) AS `min(PS_SUPPLYCOST)`#33], groupByExpr=[] )
+--LogicalFilter ( predicates=(((((P_PARTKEY#19 =
PS_PARTKEY#28) AND (S_SUPPKEY#12 = PS_SUPPKEY#29)) AND (S_NATIONKEY#15 =
N_NATIONKEY#3)) AND (N_REGIONKEY#5 = R_REGIONKEY#0)) AND (CAST(R_NAME AS
STRING) = CAST(EUROPE AS STRING))) )
+--LogicalOlapScan (
qualified=default_cluster:regression_test_tpch_sf1_p1_tpch_sf1.partsupp,
output=[PS_PARTKEY#28, PS_SUPPKEY#29, PS_AVAILQTY#30, PS_SUPPLYCOST#31,
PS_COMMENT#32], candidateIndexIds=[], selectedIndexId=11092, preAgg=ON )
```
---
.../apache/doris/analysis/FunctionCallExpr.java | 5 +-
.../doris/nereids/exceptions/ParseException.java | 2 +-
.../glue/translator/ExpressionTranslator.java | 13 +++--
.../java/org/apache/doris/nereids/jobs/Job.java | 65 +++++++++++++++++++++-
.../doris/nereids/jobs/cascades/ApplyRuleJob.java | 8 +++
.../nereids/jobs/rewrite/RewriteBottomUpJob.java | 19 ++-----
.../nereids/jobs/rewrite/RewriteTopDownJob.java | 23 +++-----
.../org/apache/doris/nereids/rules/RuleType.java | 2 +-
.../doris/nereids/rules/analysis/BindFunction.java | 4 ++
.../logical/PushdownFilterThroughProject.java | 2 +-
.../nereids/rules/rewrite/logical/ReorderJoin.java | 8 +--
.../functions/ExplicitlyCastableSignature.java | 2 +-
.../expressions/functions/IdenticalSignature.java | 2 +-
.../functions/ImplicitlyCastableSignature.java | 2 +-
.../functions/NullOrIdenticalSignature.java | 2 +-
.../trees/expressions/functions/agg/Avg.java | 13 +++--
.../trees/expressions/functions/scalar/Year.java | 4 +-
.../apache/doris/nereids/types/DecimalType.java | 3 +-
.../java/org/apache/doris/qe/SessionVariable.java | 2 +-
.../java/org/apache/doris/qe/StmtExecutor.java | 2 +-
.../nereids/datasets/tpch/AnalyzeTPCHTest.java | 16 ++++--
.../doris/nereids/parser/LimitClauseTest.java | 4 +-
.../doris/nereids/parser/NereidsParserTest.java | 2 +-
.../suites/nereids_syntax_p0/function.groovy | 11 ++++
24 files changed, 148 insertions(+), 68 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java
index a1345ef57b..8a86a2c736 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java
@@ -206,10 +206,12 @@ public class FunctionCallExpr extends Expr {
originChildSize = children.size();
}
- public FunctionCallExpr(String functionName, FunctionParams params,
Optional<List<Type>> argTypes) {
+ public FunctionCallExpr(String functionName, FunctionParams params,
FunctionParams aggFnParams,
+ Optional<List<Type>> argTypes) {
this.fnName = new FunctionName(functionName);
this.fnParams = params;
this.isMergeAggFn = false;
+ this.aggFnParams = aggFnParams;
if (fnParams.exprs() != null) {
children.addAll(fnParams.exprs());
}
@@ -222,6 +224,7 @@ public class FunctionCallExpr extends Expr {
this.fnName = functionName;
this.fn = function;
this.type = function.getReturnType();
+ this.fnParams = functionParams;
if (functionParams.exprs() != null) {
this.children.addAll(functionParams.exprs());
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/exceptions/ParseException.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/exceptions/ParseException.java
index 6eefc492b3..768bcfcb7c 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/exceptions/ParseException.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/exceptions/ParseException.java
@@ -50,7 +50,7 @@ public class ParseException extends AnalysisException {
if (start.line.isPresent() && start.startPosition.isPresent()) {
int line = start.line.get();
int startPosition = start.startPosition.get();
- sb.append("(line ").append(line).append(",
pos").append(startPosition).append(")").append("\n");
+ sb.append("(line ").append(line).append(", pos
").append(startPosition).append(")").append("\n");
if (command.isPresent()) {
sb.append("\n== SQL ==\n");
String cmd = command.get();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java
index 2dec21bb76..c6f8589515 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java
@@ -297,18 +297,21 @@ public class ExpressionTranslator extends
DefaultExpressionVisitor<Expr, PlanTra
catalogParams.add(catalogExpr);
}
+ boolean distinct = function.isDistinct();
+ FunctionParams aggFnParams = new FunctionParams(distinct,
catalogParams);
+
if (function instanceof Count) {
Count count = (Count) function;
if (count.isStar()) {
return new FunctionCallExpr(function.getName(),
FunctionParams.createStarParam(),
- inputTypesBeforeDissemble);
+ aggFnParams, inputTypesBeforeDissemble);
} else if (count.isDistinct()) {
- return new FunctionCallExpr(function.getName(), new
FunctionParams(true, catalogParams),
- inputTypesBeforeDissemble);
+ return new FunctionCallExpr(function.getName(), new
FunctionParams(distinct, catalogParams),
+ aggFnParams, inputTypesBeforeDissemble);
}
}
- return new FunctionCallExpr(function.getName(), new
FunctionParams(false, catalogParams),
- inputTypesBeforeDissemble);
+ return new FunctionCallExpr(function.getName(), new
FunctionParams(distinct, catalogParams),
+ aggFnParams, inputTypesBeforeDissemble);
}
@Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/Job.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/Job.java
index ef76d23922..b2775e1e77 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/Job.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/Job.java
@@ -18,32 +18,47 @@
package org.apache.doris.nereids.jobs;
import org.apache.doris.nereids.exceptions.AnalysisException;
+import org.apache.doris.nereids.memo.CopyInResult;
+import org.apache.doris.nereids.memo.Group;
import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleSet;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.qe.ConnectContext;
+
+import com.google.common.base.Preconditions;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
import java.util.List;
import java.util.Objects;
+import java.util.Optional;
import java.util.stream.Collectors;
/**
* Abstract class for all job using for analyze and optimize query plan in
Nereids.
*/
public abstract class Job {
+ public final Logger logger = LogManager.getLogger(getClass());
+
protected JobType type;
protected JobContext context;
protected boolean once;
+ protected final boolean enableTrace;
public Job(JobType type, JobContext context) {
- this.type = type;
- this.context = context;
- this.once = true;
+ this(type, context, true);
}
+ /** job full parameter constructor */
public Job(JobType type, JobContext context, boolean once) {
this.type = type;
this.context = context;
this.once = once;
+ ConnectContext connectContext = ConnectContext.get();
+ this.enableTrace = connectContext == null
+ ? false
+ : connectContext.getSessionVariable().isEnableNereidsTrace();
}
public void pushJob(Job job) {
@@ -73,4 +88,48 @@ public abstract class Job {
}
public abstract void execute() throws AnalysisException;
+
+ protected Optional<CopyInResult> invokeRewriteRuleWithTrace(Rule rule,
Plan before, Group targetGroup) {
+ context.onInvokeRule(rule.getRuleType());
+
+ String traceBefore = enableTrace ? getTraceLog(rule) : null;
+
+ List<Plan> afters = rule.transform(before,
context.getCascadesContext());
+ Preconditions.checkArgument(afters.size() == 1);
+ Plan after = afters.get(0);
+
+ if (after != before) {
+ CopyInResult result = context.getCascadesContext()
+ .getMemo()
+ .copyIn(after, targetGroup, true);
+
+ if (result.generateNewExpression && enableTrace) {
+ String traceAfter = getTraceLog(rule);
+ printTraceLog(rule, traceBefore, traceAfter);
+ }
+
+ return Optional.of(result);
+ }
+
+ return Optional.empty();
+ }
+
+ protected String getTraceLog(Rule rule) {
+ if (rule.isRewrite()) {
+ return context.getCascadesContext()
+ .getMemo()
+ .copyOut(false)
+ .treeString();
+ } else {
+ return context.getCascadesContext()
+ .getMemo()
+ .getRoot()
+ .treeString();
+ }
+ }
+
+ protected void printTraceLog(Rule rule, String traceBefore, String
traceAfter) {
+ logger.info("========== {} {} ==========\nbefore:\n{}\n\nafter:\n{}\n",
+ getClass().getSimpleName(), rule.getRuleType(), traceBefore,
traceAfter);
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/ApplyRuleJob.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/ApplyRuleJob.java
index f596635fc2..d263c8306f 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/ApplyRuleJob.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/ApplyRuleJob.java
@@ -59,7 +59,10 @@ public class ApplyRuleJob extends Job {
GroupExpressionMatching groupExpressionMatching
= new GroupExpressionMatching(rule.getPattern(),
groupExpression);
for (Plan plan : groupExpressionMatching) {
+ String traceBefore = enableTrace ? getTraceLog(rule) : null;
context.onInvokeRule(rule.getRuleType());
+
+ boolean changed = false;
List<Plan> newPlans = rule.transform(plan,
context.getCascadesContext());
for (Plan newPlan : newPlans) {
CopyInResult result = context.getCascadesContext()
@@ -69,6 +72,7 @@ public class ApplyRuleJob extends Job {
continue;
}
+ changed = true;
GroupExpression newGroupExpression =
result.correspondingExpression;
if (newPlan instanceof LogicalPlan) {
pushJob(new OptimizeGroupExpressionJob(newGroupExpression,
context));
@@ -77,6 +81,10 @@ public class ApplyRuleJob extends Job {
pushJob(new CostAndEnforcerJob(newGroupExpression,
context));
}
}
+ if (changed && enableTrace) {
+ String traceAfter = getTraceLog(rule);
+ printTraceLog(rule, traceBefore, traceAfter);
+ }
}
groupExpression.setApplied(rule);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/RewriteBottomUpJob.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/RewriteBottomUpJob.java
index f2841b114a..32c2b38c10 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/RewriteBottomUpJob.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/RewriteBottomUpJob.java
@@ -29,10 +29,9 @@ import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleFactory;
import org.apache.doris.nereids.trees.plans.Plan;
-import com.google.common.base.Preconditions;
-
import java.util.List;
import java.util.Objects;
+import java.util.Optional;
import java.util.stream.Collectors;
/**
@@ -77,18 +76,10 @@ public class RewriteBottomUpJob extends Job {
GroupExpressionMatching groupExpressionMatching
= new GroupExpressionMatching(rule.getPattern(),
logicalExpression);
for (Plan before : groupExpressionMatching) {
- context.onInvokeRule(rule.getRuleType());
- List<Plan> afters = rule.transform(before,
context.getCascadesContext());
- Preconditions.checkArgument(afters.size() == 1);
- Plan after = afters.get(0);
- if (after != before) {
- CopyInResult result = context.getCascadesContext()
- .getMemo()
- .copyIn(after, group, rule.isRewrite());
- if (result.generateNewExpression) {
- pushJob(new RewriteBottomUpJob(group, rules, context,
false));
- return;
- }
+ Optional<CopyInResult> copyInResult =
invokeRewriteRuleWithTrace(rule, before, group);
+ if (copyInResult.isPresent() &&
copyInResult.get().generateNewExpression) {
+ pushJob(new RewriteBottomUpJob(group, rules, context,
false));
+ return;
}
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/RewriteTopDownJob.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/RewriteTopDownJob.java
index 674a14f63e..103d07cf5f 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/RewriteTopDownJob.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/RewriteTopDownJob.java
@@ -32,6 +32,7 @@ import com.google.common.base.Preconditions;
import java.util.List;
import java.util.Objects;
+import java.util.Optional;
import java.util.stream.Collectors;
/**
@@ -77,21 +78,13 @@ public class RewriteTopDownJob extends Job {
// In topdown job, there must be only one matching plan.
// This `for` loop runs at most once.
for (Plan before : groupExpressionMatching) {
- context.onInvokeRule(rule.getRuleType());
- List<Plan> afters = rule.transform(before,
context.getCascadesContext());
- Preconditions.checkArgument(afters.size() == 1);
- Plan after = afters.get(0);
- if (after != before) {
- CopyInResult result = context.getCascadesContext()
- .getMemo()
- .copyIn(after, group, rule.isRewrite());
- if (result.generateNewExpression) {
- // new group-expr replaced the origin group-expr in
`group`,
- // run this rule against this `group` again.
- context.setRewritten(true);
- pushJob(new RewriteTopDownJob(group, rules, context));
- return;
- }
+ Optional<CopyInResult> copyInResult =
invokeRewriteRuleWithTrace(rule, before, group);
+ if (copyInResult.isPresent() &&
copyInResult.get().generateNewExpression) {
+ // new group-expr replaced the origin group-expr in
`group`,
+ // run this rule against this `group` again.
+ context.setRewritten(true);
+ pushJob(new RewriteTopDownJob(group, rules, context));
+ return;
}
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
index a6f4641bb6..6231312601 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
@@ -89,7 +89,7 @@ public enum RuleType {
PUSHDOWN_FILTER_THROUGH_JOIN(RuleTypeClass.REWRITE),
PUSHDOWN_FILTER_THROUGH_LEFT_SEMI_JOIN(RuleTypeClass.REWRITE),
PUSH_FILTER_INSIDE_JOIN(RuleTypeClass.REWRITE),
- PUSHDOWN_FILTER_THROUGH_PROJET(RuleTypeClass.REWRITE),
+ PUSHDOWN_FILTER_THROUGH_PROJECT(RuleTypeClass.REWRITE),
PUSHDOWN_PROJECT_THROUGHT_LIMIT(RuleTypeClass.REWRITE),
// column prune rules,
COLUMN_PRUNE_AGGREGATION_CHILD(RuleTypeClass.REWRITE),
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindFunction.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindFunction.java
index a93c537fa1..80d6144cbb 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindFunction.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindFunction.java
@@ -115,6 +115,8 @@ public class BindFunction implements AnalysisRuleFactory {
@Override
public BoundFunction visitUnboundFunction(UnboundFunction
unboundFunction, Env env) {
+ unboundFunction = (UnboundFunction)
super.visitUnboundFunction(unboundFunction, env);
+
// FunctionRegistry can't support boolean arg now, tricky here.
if (unboundFunction.getName().equalsIgnoreCase("count")) {
List<Expression> arguments = unboundFunction.getArguments();
@@ -141,6 +143,8 @@ public class BindFunction implements AnalysisRuleFactory {
*/
@Override
public Expression visitTimestampArithmetic(TimestampArithmetic
arithmetic, Env context) {
+ arithmetic = (TimestampArithmetic)
super.visitTimestampArithmetic(arithmetic, context);
+
String funcOpName;
if (arithmetic.getFuncName() == null) {
// e.g. YEARS_ADD, MONTHS_SUB
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughProject.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughProject.java
index d38e6904bb..2a41a75de9 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughProject.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughProject.java
@@ -44,6 +44,6 @@ public class PushdownFilterThroughProject extends
OneRewriteRuleFactory {
project.child()
)
);
- }).toRule(RuleType.PUSHDOWN_FILTER_THROUGH_PROJET);
+ }).toRule(RuleType.PUSHDOWN_FILTER_THROUGH_PROJECT);
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/ReorderJoin.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/ReorderJoin.java
index c0c8622348..3ee308527b 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/ReorderJoin.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/ReorderJoin.java
@@ -351,15 +351,15 @@ public class ReorderJoin extends OneRewriteRuleFactory {
}
// All { left -> one in [candidates] } is CrossJoin
// Generate a CrossJoin
- for (int j = candidates.size() - 1; j >= 0; j--) {
- if (usedPlansIndex.contains(j)) {
+ for (int i = 0; i < candidates.size(); i++) {
+ if (usedPlansIndex.contains(i)) {
continue;
}
- usedPlansIndex.add(j);
+ usedPlansIndex.add(i);
return new LogicalJoin<>(JoinType.CROSS_JOIN,
ExpressionUtils.EMPTY_CONDITION,
otherJoinConditions,
- left, candidates.get(j));
+ left, candidates.get(i));
}
throw new RuntimeException("findInnerJoin: can't reach here");
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ExplicitlyCastableSignature.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ExplicitlyCastableSignature.java
index 1edf15f1f4..b728d0320c 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ExplicitlyCastableSignature.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ExplicitlyCastableSignature.java
@@ -30,7 +30,7 @@ import
org.apache.doris.nereids.types.coercion.AbstractDataType;
public interface ExplicitlyCastableSignature extends ComputeSignature {
static boolean isExplicitlyCastable(AbstractDataType signatureType,
AbstractDataType realType) {
// TODO: copy canCastTo method to DataType
- return Type.canCastTo(signatureType.toCatalogDataType(),
realType.toCatalogDataType());
+ return Type.canCastTo(realType.toCatalogDataType(),
signatureType.toCatalogDataType());
}
@Override
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/IdenticalSignature.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/IdenticalSignature.java
index c2a3dc7390..2beab26027 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/IdenticalSignature.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/IdenticalSignature.java
@@ -29,7 +29,7 @@ import
org.apache.doris.nereids.types.coercion.AbstractDataType;
public interface IdenticalSignature extends ComputeSignature {
static boolean isIdentical(AbstractDataType signatureType,
AbstractDataType realType) {
// TODO: copy matchesType to DataType
- return
signatureType.toCatalogDataType().matchesType(realType.toCatalogDataType());
+ return
realType.toCatalogDataType().matchesType(signatureType.toCatalogDataType());
}
@Override
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ImplicitlyCastableSignature.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ImplicitlyCastableSignature.java
index d9891d4d2f..3c830ef31b 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ImplicitlyCastableSignature.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ImplicitlyCastableSignature.java
@@ -30,7 +30,7 @@ import
org.apache.doris.nereids.types.coercion.AbstractDataType;
public interface ImplicitlyCastableSignature extends ComputeSignature {
static boolean isImplicitlyCastable(AbstractDataType signatureType,
AbstractDataType realType) {
// TODO: copy isImplicitlyCastable method to DataType
- return Type.isImplicitlyCastable(signatureType.toCatalogDataType(),
realType.toCatalogDataType(), true);
+ return Type.isImplicitlyCastable(realType.toCatalogDataType(),
signatureType.toCatalogDataType(), true);
}
@Override
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/NullOrIdenticalSignature.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/NullOrIdenticalSignature.java
index cc2d2fdf92..164d175729 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/NullOrIdenticalSignature.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/NullOrIdenticalSignature.java
@@ -31,7 +31,7 @@ public interface NullOrIdenticalSignature extends
ComputeSignature {
static boolean isNullOrIdentical(AbstractDataType signatureType,
AbstractDataType realType) {
// TODO: copy matchesType to DataType
return realType instanceof NullType
- ||
signatureType.toCatalogDataType().matchesType(realType.toCatalogDataType());
+ ||
realType.toCatalogDataType().matchesType(signatureType.toCatalogDataType());
}
@Override
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Avg.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Avg.java
index cea26f3344..406004236d 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Avg.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Avg.java
@@ -54,18 +54,21 @@ public class Avg extends AggregateFunction implements
UnaryExpression, ImplicitC
@Override
public DataType getFinalType() {
- if (child().getDataType() instanceof DecimalType) {
- return child().getDataType();
- } else if (child().getDataType().isDate()) {
+ DataType argumentType = inputTypesBeforeDissemble()
+ .map(types -> types.get(0))
+ .orElse(child().getDataType());
+ if (argumentType instanceof DecimalType) {
+ return argumentType;
+ } else if (argumentType.isDate()) {
return DateType.INSTANCE;
- } else if (child().getDataType().isDateTime()) {
+ } else if (argumentType.isDateTime()) {
return DateTimeType.INSTANCE;
} else {
return DoubleType.INSTANCE;
}
}
- // TODO: We should return a complex type:
PartialAggType(bufferTypes=[Double, Int], inputType=Int)
+ // TODO: We should return a complex type:
PartialAggType(bufferTypes=[Double, Int], inputTypes=[Int])
// to denote sum(double) and count(int)
@Override
public DataType getIntermediateType() {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Year.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Year.java
index 688eb5516d..58e6df906e 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Year.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Year.java
@@ -40,9 +40,9 @@ public class Year extends ScalarFunction
implements UnaryExpression, ExplicitlyCastableSignature,
AlwaysNullable {
public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
+
FunctionSignature.ret(IntegerType.INSTANCE).args(DateV2Type.INSTANCE),
FunctionSignature.ret(IntegerType.INSTANCE).args(DateTimeType.INSTANCE),
-
FunctionSignature.ret(IntegerType.INSTANCE).args(DateTimeV2Type.INSTANCE),
-
FunctionSignature.ret(IntegerType.INSTANCE).args(DateV2Type.INSTANCE)
+
FunctionSignature.ret(IntegerType.INSTANCE).args(DateTimeV2Type.INSTANCE)
);
/**
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DecimalType.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DecimalType.java
index 545bdae390..e19b3ee52f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DecimalType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DecimalType.java
@@ -17,6 +17,7 @@
package org.apache.doris.nereids.types;
+import org.apache.doris.catalog.ScalarType;
import org.apache.doris.catalog.Type;
import org.apache.doris.nereids.types.coercion.AbstractDataType;
import org.apache.doris.nereids.types.coercion.FractionalType;
@@ -99,7 +100,7 @@ public class DecimalType extends FractionalType {
@Override
public Type toCatalogDataType() {
- return Type.MAX_DECIMALV2_TYPE;
+ return ScalarType.createDecimalType(precision, scale);
}
public int getPrecision() {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index d91581b02c..205779b77e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -517,7 +517,7 @@ public class SessionVariable implements Serializable,
Writable {
private boolean nereidsStarSchemaSupport = true;
@VariableMgr.VarAttr(name = ENABLE_NEREIDS_TRACE)
- private boolean enableNereidsTrace = true;
+ private boolean enableNereidsTrace = false;
@VariableMgr.VarAttr(name = ENABLE_NEREIDS_RUNTIME_FILTER)
private boolean enableNereidsRuntimeFilter = true;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
index d60b7fb97d..ae2a321b94 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
@@ -724,7 +724,7 @@ public class StmtExecutor implements ProfileWriter {
} catch (UserException e) {
throw e;
} catch (Exception e) {
- LOG.warn("Analyze failed. {}",
context.getQueryIdentifier(), e);
+ LOG.error("Analyze failed. {}",
context.getQueryIdentifier(), e);
if (parsedStmt instanceof LogicalPlanAdapter) {
throw new NereidsException(new
AnalysisException("Unexpected exception: " + e.getMessage(), e));
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/datasets/tpch/AnalyzeTPCHTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/datasets/tpch/AnalyzeTPCHTest.java
index 44de365d5d..ae0dfc754e 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/datasets/tpch/AnalyzeTPCHTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/datasets/tpch/AnalyzeTPCHTest.java
@@ -17,13 +17,15 @@
package org.apache.doris.nereids.datasets.tpch;
+import org.junit.jupiter.api.Test;
+
/**
* There are still many functions that have not been implemented,
* so the tpch cannot be fully parsed, and the interface is only reserved here.
* When the related functions of tpch are supported, the comments will be
deleted and the analyze of tpch will be verified.
*/
public class AnalyzeTPCHTest extends TPCHTestBase {
- /*
+
@Test
public void q1() {
checkAnalyze(TPCHUtils.Q1);
@@ -139,10 +141,11 @@ public class AnalyzeTPCHTest extends TPCHTestBase {
checkAnalyze(TPCHUtils.Q17);
}
- @Test
+ // TODO: support [broadcast] hint
+ /*@Test
public void q17_rewrite() {
checkAnalyze(TPCHUtils.Q17_rewrite);
- }
+ }*/
@Test
public void q18() {
@@ -179,13 +182,14 @@ public class AnalyzeTPCHTest extends TPCHTestBase {
checkAnalyze(TPCHUtils.Q21_rewrite);
}
- @Test
+ // NOTE: not support '1 for 2' syntax
+ /*@Test
public void q22() {
checkAnalyze(TPCHUtils.Q22);
- }
+ }*/
@Test
public void q22_rewrite() {
checkAnalyze(TPCHUtils.Q22_rewrite);
- }*/
+ }
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/LimitClauseTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/LimitClauseTest.java
index 8312877364..fdf42fbcdc 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/LimitClauseTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/LimitClauseTest.java
@@ -54,7 +54,7 @@ public class LimitClauseTest extends ParserTestBase {
parsePlan("SELECT b FROM test limit 3 offset 100")
.assertThrowsExactly(ParseException.class)
.assertMessageContains("\n"
- + "OFFSET requires an ORDER BY clause(line 1, pos19)\n"
+ + "OFFSET requires an ORDER BY clause(line 1, pos
19)\n"
+ "\n"
+ "== SQL ==\n"
+ "SELECT b FROM test limit 3 offset 100\n"
@@ -63,7 +63,7 @@ public class LimitClauseTest extends ParserTestBase {
parsePlan("SELECT b FROM test limit 100, 3")
.assertThrowsExactly(ParseException.class)
.assertMessageContains("\n"
- + "OFFSET requires an ORDER BY clause(line 1, pos19)\n"
+ + "OFFSET requires an ORDER BY clause(line 1, pos
19)\n"
+ "\n"
+ "== SQL ==\n"
+ "SELECT b FROM test limit 100, 3\n"
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java
index 4e1330ad55..573ee9e905 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java
@@ -69,7 +69,7 @@ public class NereidsParserTest extends ParserTestBase {
public void testErrorListener() {
parsePlan("select * from t1 where a = 1 illegal_symbol")
.assertThrowsExactly(ParseException.class)
- .assertMessageEquals("\nextraneous input 'illegal_symbol'
expecting {<EOF>, ';'}(line 1, pos29)\n");
+ .assertMessageEquals("\nextraneous input 'illegal_symbol'
expecting {<EOF>, ';'}(line 1, pos 29)\n");
}
@Test
diff --git a/regression-test/suites/nereids_syntax_p0/function.groovy
b/regression-test/suites/nereids_syntax_p0/function.groovy
index 1ad34fdedf..08886cb0ed 100644
--- a/regression-test/suites/nereids_syntax_p0/function.groovy
+++ b/regression-test/suites/nereids_syntax_p0/function.groovy
@@ -52,5 +52,16 @@ suite("function") {
order_qt_avg """
SELECT avg(lo_tax), avg(lo_extendedprice) AS avg_extendedprice FROM
lineorder;
"""
+
+ // nested function
+ test {
+ sql "select cast(date('1994-01-01') + interval '1' YEAR as varchar)"
+ result([["1995-01-01 00:00:00"]])
+ }
+
+ test {
+ sql "select substring(substring('1994-01-01', 5), 3)"
+ result([["1-01"]])
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
