This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 6c13126e5c [enhancement](Nereids) analyze check input slots must in
child's output (#14107)
6c13126e5c is described below
commit 6c13126e5cacf90829c202ac1eea306e134bae1e
Author: morrySnow <[email protected]>
AuthorDate: Thu Nov 10 19:28:01 2022 +0800
[enhancement](Nereids) analyze check input slots must in child's output
(#14107)
---
.../org/apache/doris/nereids/CascadesContext.java | 4 +-
.../doris/nereids/analyzer/NereidsAnalyzer.java | 9 -----
.../jobs/batch/NereidsRewriteJobExecutor.java | 3 ++
.../{CheckAnalysis.java => CheckAfterRewrite.java} | 47 ++++++++++------------
.../nereids/rules/analysis/CheckAnalysis.java | 13 +++---
5 files changed, 34 insertions(+), 42 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java
index fd2730e5ba..9aca1d2f8b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java
@@ -55,8 +55,8 @@ public class CascadesContext {
private final JobScheduler jobScheduler;
private JobContext currentJobContext;
// subqueryExprIsAnalyzed: whether the subquery has been analyzed.
- private Map<SubqueryExpr, Boolean> subqueryExprIsAnalyzed;
- private RuntimeFilterContext runtimeFilterContext;
+ private final Map<SubqueryExpr, Boolean> subqueryExprIsAnalyzed;
+ private final RuntimeFilterContext runtimeFilterContext;
/**
* Constructor of OptimizerContext.
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/NereidsAnalyzer.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/NereidsAnalyzer.java
index b52c57f5e1..75bcfc8117 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/NereidsAnalyzer.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/NereidsAnalyzer.java
@@ -56,13 +56,4 @@ public class NereidsAnalyzer {
// check whether analyze result is meaningful
new CheckAnalysisJob(cascadesContext).execute();
}
-
- public CascadesContext getCascadesContext() {
- return cascadesContext;
- }
-
- public Optional<Scope> getOuterScope() {
- return outerScope;
- }
-
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/NereidsRewriteJobExecutor.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/NereidsRewriteJobExecutor.java
index 634e4ca9fb..32fc226e5d 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/NereidsRewriteJobExecutor.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/NereidsRewriteJobExecutor.java
@@ -20,6 +20,7 @@ package org.apache.doris.nereids.jobs.batch;
import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.jobs.Job;
import org.apache.doris.nereids.rules.RuleSet;
+import org.apache.doris.nereids.rules.analysis.CheckAfterRewrite;
import
org.apache.doris.nereids.rules.expression.rewrite.ExpressionNormalization;
import
org.apache.doris.nereids.rules.expression.rewrite.ExpressionOptimization;
import org.apache.doris.nereids.rules.mv.SelectMaterializedIndexWithAggregate;
@@ -82,6 +83,8 @@ public class NereidsRewriteJobExecutor extends BatchRulesJob {
// we need to execute this rule at the end of rewrite
// to avoid two consecutive same project appear when we do
optimization.
.add(topDownBatch(ImmutableList.of(new
EliminateUnnecessaryProject())))
+ // this rule batch must keep at the end of rewrite to do some
plan check
+ .add(bottomUpBatch(ImmutableList.of(new CheckAfterRewrite())))
.build();
rulesJob.addAll(jobs);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAnalysis.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAfterRewrite.java
similarity index 52%
copy from
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAnalysis.java
copy to
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAfterRewrite.java
index 9d50db7c36..1cd0a67813 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAnalysis.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAfterRewrite.java
@@ -17,52 +17,47 @@
package org.apache.doris.nereids.rules.analysis;
-import org.apache.doris.nereids.analyzer.UnboundSlot;
import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.trees.expressions.Expression;
-import org.apache.doris.nereids.trees.expressions.typecoercion.TypeCheckResult;
+import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.expressions.functions.ExpressionTrait;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.commons.lang.StringUtils;
-import java.util.Optional;
+import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
/**
- * Check analysis rule to check semantic correct after analysis by Nereids.
+ * some check need to do after analyze whole plan.
*/
-public class CheckAnalysis extends OneAnalysisRuleFactory {
+public class CheckAfterRewrite extends OneAnalysisRuleFactory {
@Override
public Rule build() {
- return any().then(plan ->
checkExpressionInputTypes(checkBound(plan))).toRule(RuleType.CHECK_ANALYSIS);
+ return any().then(plan -> {
+ checkAllSlotReferenceFromChildren(plan);
+ return null;
+ }).toRule(RuleType.CHECK_ANALYSIS);
}
- private Plan checkExpressionInputTypes(Plan plan) {
- final Optional<TypeCheckResult> firstFailed =
plan.getExpressions().stream()
- .map(Expression::checkInputDataTypes)
- .filter(TypeCheckResult::failed)
- .findFirst();
-
- if (firstFailed.isPresent()) {
- throw new AnalysisException(firstFailed.get().getMessage());
- }
- return plan;
- }
-
- private Plan checkBound(Plan plan) {
- Set<UnboundSlot> unboundSlots = plan.getExpressions().stream()
- .<Set<UnboundSlot>>map(e ->
e.collect(UnboundSlot.class::isInstance))
+ private void checkAllSlotReferenceFromChildren(Plan plan) {
+ Set<Slot> notFromChildren = plan.getExpressions().stream()
+ .map(Expression::getInputSlots)
.flatMap(Set::stream)
.collect(Collectors.toSet());
- if (!unboundSlots.isEmpty()) {
- throw new AnalysisException(String.format("Cannot find column %s.",
- StringUtils.join(unboundSlots.stream()
- .map(UnboundSlot::toSql)
+ Set<Slot> childrenOutput = plan.children().stream()
+ .map(Plan::getOutput)
+ .flatMap(List::stream)
+ .collect(Collectors.toSet());
+ notFromChildren.removeAll(childrenOutput);
+ if (!notFromChildren.isEmpty()) {
+ throw new AnalysisException(String.format("Input slot(s) not in
child's output: %s",
+ StringUtils.join(notFromChildren.stream()
+ .map(ExpressionTrait::toSql)
.collect(Collectors.toSet()), ", ")));
}
- return plan;
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAnalysis.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAnalysis.java
index 9d50db7c36..0f8bf93669 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAnalysis.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAnalysis.java
@@ -35,12 +35,17 @@ import java.util.stream.Collectors;
* Check analysis rule to check semantic correct after analysis by Nereids.
*/
public class CheckAnalysis extends OneAnalysisRuleFactory {
+
@Override
public Rule build() {
- return any().then(plan ->
checkExpressionInputTypes(checkBound(plan))).toRule(RuleType.CHECK_ANALYSIS);
+ return any().then(plan -> {
+ checkBound(plan);
+ checkExpressionInputTypes(plan);
+ return null;
+ }).toRule(RuleType.CHECK_ANALYSIS);
}
- private Plan checkExpressionInputTypes(Plan plan) {
+ private void checkExpressionInputTypes(Plan plan) {
final Optional<TypeCheckResult> firstFailed =
plan.getExpressions().stream()
.map(Expression::checkInputDataTypes)
.filter(TypeCheckResult::failed)
@@ -49,10 +54,9 @@ public class CheckAnalysis extends OneAnalysisRuleFactory {
if (firstFailed.isPresent()) {
throw new AnalysisException(firstFailed.get().getMessage());
}
- return plan;
}
- private Plan checkBound(Plan plan) {
+ private void checkBound(Plan plan) {
Set<UnboundSlot> unboundSlots = plan.getExpressions().stream()
.<Set<UnboundSlot>>map(e ->
e.collect(UnboundSlot.class::isInstance))
.flatMap(Set::stream)
@@ -63,6 +67,5 @@ public class CheckAnalysis extends OneAnalysisRuleFactory {
.map(UnboundSlot::toSql)
.collect(Collectors.toSet()), ", ")));
}
- return plan;
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
