This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new 3239ef3cec7 [fix](Nereids) should derive stats asap to avoid npe
(#34238) (#35451)
3239ef3cec7 is described below
commit 3239ef3cec76dcefbebdfe65fcd23aeed01c0f9f
Author: morrySnow <[email protected]>
AuthorDate: Mon May 27 18:16:00 2024 +0800
[fix](Nereids) should derive stats asap to avoid npe (#34238) (#35451)
pick from master #34238
we do derive stats job eager to avoid un derive stats due to merge group
and optimize group
consider:
we have two groups burned by order: G1 and G2
then we have job by order derive G2, optimize group expression in G2,
derive G1, optimize group expression in G1
if G1 merged into G2, then we maybe generated job optimize group G2
before derive G1
in this case, we will do get stats from G1's child before derive G1's
child stats
then we will meet NPE in CostModel.
---
.../apache/doris/nereids/jobs/cascades/ApplyRuleJob.java | 16 +++++++++++++++-
1 file changed, 15 insertions(+), 1 deletion(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/ApplyRuleJob.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/ApplyRuleJob.java
index 0c366907a51..30a2b1a4959 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/ApplyRuleJob.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/ApplyRuleJob.java
@@ -33,6 +33,8 @@ import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
+import com.google.common.collect.Lists;
+
import java.util.HashMap;
import java.util.List;
@@ -67,6 +69,7 @@ public class ApplyRuleJob extends Job {
}
countJobExecutionTimesOfGroupExpressions(groupExpression);
+ List<DeriveStatsJob> deriveStatsJobs = Lists.newArrayList();
GroupExpressionMatching groupExpressionMatching
= new GroupExpressionMatching(rule.getPattern(),
groupExpression);
for (Plan plan : groupExpressionMatching) {
@@ -86,11 +89,22 @@ public class ApplyRuleJob extends Job {
pushJob(new CostAndEnforcerJob(newGroupExpression,
context));
}
// we should derive stats for new logical/physical plan if the
plan missing the stats
- pushJob(new DeriveStatsJob(newGroupExpression, context));
+ deriveStatsJobs.add(new DeriveStatsJob(newGroupExpression,
context));
NereidsTracer.logApplyRuleEvent(rule.toString(), plan,
newGroupExpression.getPlan());
APPLY_RULE_TRACER.log(TransformEvent.of(groupExpression, plan,
newPlans, rule.getRuleType()),
rule::isRewrite);
}
+ // we do derive stats job eager to avoid un derive stats due to
merge group and optimize group
+ // consider:
+ // we have two groups burned by order: G1 and G2
+ // then we have job by order derive G2, optimize group
expression in G2,
+ // derive G1, optimize group expression in G1
+ // if G1 merged into G2, then we maybe generated job optimize
group G2 before derive G1
+ // in this case, we will do get stats from G1's child before
derive G1's child stats
+ // then we will meet NPE in CostModel.
+ for (DeriveStatsJob deriveStatsJob : deriveStatsJobs) {
+ pushJob(deriveStatsJob);
+ }
}
groupExpression.setApplied(rule);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
