This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new c56d387d41a [opt](memo) reduce enforcer number in memo (#59600)
c56d387d41a is described below
commit c56d387d41a93c989b7bc552308af49abb25efcb
Author: morrySnow <[email protected]>
AuthorDate: Fri Jan 16 10:31:42 2026 +0800
[opt](memo) reduce enforcer number in memo (#59600)
This pull request refactors and simplifies the handling of job context
and enforcer management in the Cascades optimizer framework. The main
changes remove the cost upper bound from `JobContext`, improve enforcer
deduplication for distribution specs, and streamline how broadcast joins
are detected and handled. Additionally, the `PlanContext` class is
refactored for clarity and efficiency, and equality checks are improved
for distribution specs.
**Refactoring and Simplification of Job Context:**
- Removed the `costUpperBound` parameter and related logic from
`JobContext` and its usage throughout the optimizer, simplifying job
scheduling and context management.
**Enforcer Deduplication and Management:**
- Added a new `enforcerSpecs` map in `Group` to deduplicate enforcers
based on `DistributionSpec`, ensuring only one enforcer per spec and
improving efficiency during group merges and enforcer addition.
- Updated enforcer lookup and creation logic in both property
enforcement and regulation helpers to use the new deduplication
mechanism.
**PlanContext Refactoring:**
- Refactored `PlanContext` to remove redundant fields, directly use
`GroupExpression`, and determine broadcast join status based on child
properties, making the class more focused and reducing unnecessary
state.
**DistributionSpec Equality Improvements:**
- Implemented `equals` and `hashCode` methods for
`DistributionSpecHiveTableSinkHashPartitioned` to ensure correct
behavior in collections and when used as map keys, supporting the new
enforcer deduplication logic.
**Miscellaneous Improvements:**
- Ensured that ownership is properly set when merging groups and
updating cost plans, preventing inconsistencies in group expression
ownership.
- Minor code cleanups and import adjustments to support the above
changes.
**Test Result:**
- In TPC-DS Q72, this PR could reduce enforcer number from 89837 to
16344, reduce Memory usage more than 200MB.
---
.../org/apache/doris/nereids/CascadesContext.java | 4 +--
.../java/org/apache/doris/nereids/PlanContext.java | 41 +++++++++-------------
.../apache/doris/nereids/cost/CostCalculator.java | 8 +----
.../org/apache/doris/nereids/jobs/JobContext.java | 12 +------
.../nereids/jobs/cascades/CostAndEnforcerJob.java | 12 ++-----
.../hypergraph/receiver/PlanReceiver.java | 2 +-
.../java/org/apache/doris/nereids/memo/Group.java | 23 ++++++++++++
.../apache/doris/nereids/memo/GroupExpression.java | 2 +-
.../properties/ChildOutputPropertyDeriver.java | 4 ++-
.../properties/ChildrenPropertiesRegulator.java | 3 ++
...stributionSpecHiveTableSinkHashPartitioned.java | 18 ++++++++++
.../properties/EnforceMissingPropertiesHelper.java | 17 ++++++---
.../nereids/properties/RequestPropertyDeriver.java | 4 ++-
.../exploration/join/OuterJoinLAsscomProject.java | 8 +----
.../nereids/trees/expressions/Expression.java | 17 +++++----
.../trees/plans/physical/PhysicalHashJoin.java | 6 ++--
.../nereids/jobs/cascades/DeriveStatsJobTest.java | 2 +-
.../rules/rewrite/ConstantPropagationTest.java | 2 +-
.../org/apache/doris/nereids/util/PlanChecker.java | 3 +-
19 files changed, 103 insertions(+), 85 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java
index 1517d6f2a7a..845faa58db2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java
@@ -155,7 +155,7 @@ public class CascadesContext implements ScheduleContext {
this.ruleSet = new RuleSet();
this.jobPool = new JobStack();
this.jobScheduler = new SimpleJobScheduler();
- this.currentJobContext = new JobContext(this, requireProperties,
Double.MAX_VALUE);
+ this.currentJobContext = new JobContext(this, requireProperties);
this.subqueryExprIsAnalyzed = new HashMap<>();
IdGenerator<RuntimeFilterId> runtimeFilterIdGen =
RuntimeFilterId.createGenerator();
this.runtimeFilterContext = new
RuntimeFilterContext(getConnectContext().getSessionVariable(),
@@ -362,7 +362,7 @@ public class CascadesContext implements ScheduleContext {
}
public CascadesContext setJobContext(PhysicalProperties
physicalProperties) {
- this.currentJobContext = new JobContext(this, physicalProperties,
Double.MAX_VALUE);
+ this.currentJobContext = new JobContext(this, physicalProperties);
return this;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/PlanContext.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/PlanContext.java
index 3ab95423e24..09285a638de 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/PlanContext.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/PlanContext.java
@@ -18,12 +18,13 @@
package org.apache.doris.nereids;
import org.apache.doris.nereids.memo.GroupExpression;
+import org.apache.doris.nereids.properties.DistributionSpecReplicated;
+import org.apache.doris.nereids.properties.PhysicalProperties;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.SessionVariable;
import org.apache.doris.statistics.Statistics;
-import java.util.ArrayList;
import java.util.List;
/**
@@ -34,23 +35,21 @@ import java.util.List;
*/
public class PlanContext {
private final ConnectContext connectContext;
- private final List<Statistics> childrenStats;
- private final Statistics planStats;
- private final int arity;
- private boolean isBroadcastJoin = false;
- private final boolean isStatsReliable;
+ private final GroupExpression groupExpression;
+ private final boolean isBroadcastJoin;
/**
* Constructor for PlanContext.
*/
- public PlanContext(ConnectContext connectContext, GroupExpression
groupExpression) {
+ public PlanContext(ConnectContext connectContext, GroupExpression
groupExpression,
+ List<PhysicalProperties> childrenProperties) {
this.connectContext = connectContext;
- this.arity = groupExpression.arity();
- this.planStats = groupExpression.getOwnerGroup().getStatistics();
- this.isStatsReliable =
groupExpression.getOwnerGroup().isStatsReliable();
- this.childrenStats = new ArrayList<>(groupExpression.arity());
- for (int i = 0; i < groupExpression.arity(); i++) {
- childrenStats.add(groupExpression.childStatistics(i));
+ this.groupExpression = groupExpression;
+ if (childrenProperties.size() >= 2
+ && childrenProperties.get(1).getDistributionSpec() instanceof
DistributionSpecReplicated) {
+ isBroadcastJoin = true;
+ } else {
+ isBroadcastJoin = false;
}
}
@@ -58,35 +57,27 @@ public class PlanContext {
return connectContext.getSessionVariable();
}
- public void setBroadcastJoin() {
- isBroadcastJoin = true;
- }
-
public boolean isBroadcastJoin() {
return isBroadcastJoin;
}
public int arity() {
- return arity;
+ return groupExpression.arity();
}
public Statistics getStatisticsWithCheck() {
- return planStats;
+ return groupExpression.getOwnerGroup().getStatistics();
}
public boolean isStatsReliable() {
- return isStatsReliable;
+ return groupExpression.getOwnerGroup().isStatsReliable();
}
/**
* Get child statistics.
*/
public Statistics getChildStatistics(int index) {
- return childrenStats.get(index);
- }
-
- public List<Statistics> getChildrenStatistics() {
- return childrenStats;
+ return groupExpression.childStatistics(index);
}
public StatementContext getStatementContext() {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostCalculator.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostCalculator.java
index 7661e1aec51..8ec69391691 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostCalculator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostCalculator.java
@@ -19,7 +19,6 @@ package org.apache.doris.nereids.cost;
import org.apache.doris.nereids.PlanContext;
import org.apache.doris.nereids.memo.GroupExpression;
-import org.apache.doris.nereids.properties.DistributionSpecReplicated;
import org.apache.doris.nereids.properties.PhysicalProperties;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.qe.ConnectContext;
@@ -38,12 +37,7 @@ public class CostCalculator {
*/
public static Cost calculateCost(ConnectContext connectContext,
GroupExpression groupExpression,
List<PhysicalProperties> childrenProperties) {
- PlanContext planContext = new PlanContext(connectContext,
groupExpression);
- if (childrenProperties.size() >= 2
- && childrenProperties.get(1).getDistributionSpec() instanceof
DistributionSpecReplicated) {
- planContext.setBroadcastJoin();
- }
-
+ PlanContext planContext = new PlanContext(connectContext,
groupExpression, childrenProperties);
CostModel costModelV1 = new CostModel(connectContext);
return groupExpression.getPlan().accept(costModelV1, planContext);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/JobContext.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/JobContext.java
index 88ca252e537..24c3beadf6b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/JobContext.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/JobContext.java
@@ -37,7 +37,6 @@ public class JobContext {
// use for optimizer
protected final ScheduleContext scheduleContext;
protected final PhysicalProperties requiredProperties;
- protected double costUpperBound;
// use for rewriter
protected boolean rewritten = false;
@@ -46,10 +45,9 @@ public class JobContext {
// user for trace
protected Map<RuleType, Integer> ruleInvokeTimes = Maps.newLinkedHashMap();
- public JobContext(ScheduleContext scheduleContext, PhysicalProperties
requiredProperties, double costUpperBound) {
+ public JobContext(ScheduleContext scheduleContext, PhysicalProperties
requiredProperties) {
this.scheduleContext = scheduleContext;
this.requiredProperties = requiredProperties;
- this.costUpperBound = costUpperBound;
}
public ScheduleContext getScheduleContext() {
@@ -64,14 +62,6 @@ public class JobContext {
return requiredProperties;
}
- public double getCostUpperBound() {
- return costUpperBound;
- }
-
- public void setCostUpperBound(double costUpperBound) {
- this.costUpperBound = costUpperBound;
- }
-
public boolean isRewritten() {
return rewritten;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/CostAndEnforcerJob.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/CostAndEnforcerJob.java
index 76a659870d0..31206e85e38 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/CostAndEnforcerJob.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/CostAndEnforcerJob.java
@@ -174,9 +174,7 @@ public class CostAndEnforcerJob extends Job implements
Cloneable {
// Meaning that optimize recursively by derive job.
prevChildIndex = curChildIndex;
pushJob(clone());
- double newCostUpperBound = context.getCostUpperBound() -
curTotalCost.getValue();
- JobContext jobContext = new
JobContext(context.getCascadesContext(),
- requestChildProperty, newCostUpperBound);
+ JobContext jobContext = new
JobContext(context.getCascadesContext(), requestChildProperty);
pushJob(new OptimizeGroupJob(childGroup, jobContext));
return;
}
@@ -215,13 +213,7 @@ public class CostAndEnforcerJob extends Job implements
Cloneable {
// This mean that we successfully optimize all child groups.
// if break when running the loop above, the condition must be
false.
if (curChildIndex == groupExpression.arity()) {
- if (!calculateEnforce(requestChildrenProperties,
outputChildrenProperties)) {
- clear();
- continue; // if error exists, return
- }
- if (curTotalCost.getValue() < context.getCostUpperBound()) {
- context.setCostUpperBound(curTotalCost.getValue());
- }
+ calculateEnforce(requestChildrenProperties,
outputChildrenProperties);
}
clear();
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/receiver/PlanReceiver.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/receiver/PlanReceiver.java
index b4629ef9de6..9047a176322 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/receiver/PlanReceiver.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/receiver/PlanReceiver.java
@@ -188,7 +188,7 @@ public class PlanReceiver implements AbstractReceiver {
private void proposeAllDistributedPlans(GroupExpression groupExpression) {
jobContext.getCascadesContext().pushJob(new
OptimizeGroupExpressionJob(groupExpression,
- new JobContext(jobContext.getCascadesContext(),
PhysicalProperties.ANY, Double.MAX_VALUE)));
+ new JobContext(jobContext.getCascadesContext(),
PhysicalProperties.ANY)));
if (!groupExpression.isStatDerived()) {
jobContext.getCascadesContext().pushJob(new
DeriveStatsJob(groupExpression,
jobContext.getCascadesContext().getCurrentJobContext()));
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Group.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Group.java
index b3d45e6c80a..0b9c78bdec6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Group.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Group.java
@@ -19,6 +19,7 @@ package org.apache.doris.nereids.memo;
import org.apache.doris.common.Pair;
import org.apache.doris.nereids.cost.Cost;
+import org.apache.doris.nereids.properties.DistributionSpec;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.properties.PhysicalProperties;
import org.apache.doris.nereids.trees.expressions.literal.Literal;
@@ -27,6 +28,7 @@ import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.logical.LogicalJoin;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalDistribute;
import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan;
import org.apache.doris.nereids.util.TreeStringUtils;
import org.apache.doris.nereids.util.Utils;
@@ -60,6 +62,7 @@ public class Group {
private final List<GroupExpression> logicalExpressions =
Lists.newArrayList();
private final List<GroupExpression> physicalExpressions =
Lists.newArrayList();
private final Map<GroupExpression, GroupExpression> enforcers =
Maps.newHashMap();
+ private final Map<DistributionSpec, GroupExpression> enforcerSpecs =
Maps.newHashMap();
private boolean isStatsReliable = true;
private LogicalProperties logicalProperties;
@@ -243,8 +246,17 @@ public class Group {
return null;
}
+ /**
+ * add a new enforcer to this group.
+ */
public void addEnforcer(GroupExpression enforcer) {
enforcer.setOwnerGroup(this);
+ if (enforcer.getPlan() instanceof PhysicalDistribute) {
+ DistributionSpec distributionSpec = ((PhysicalDistribute)
enforcer.getPlan()).getDistributionSpec();
+ if (null != enforcerSpecs.put(distributionSpec, enforcer)) {
+ return;
+ }
+ }
enforcers.put(enforcer, enforcer);
}
@@ -252,6 +264,10 @@ public class Group {
return enforcers;
}
+ public Map<DistributionSpec, GroupExpression> getEnforcerSpecs() {
+ return enforcerSpecs;
+ }
+
/**
* Set or update lowestCostPlans: properties --> Pair.of(cost, expression)
*/
@@ -356,6 +372,7 @@ public class Group {
// TODO: dedup?
enforcers.forEach((k, v) -> target.addEnforcer(k));
enforcers.clear();
+ enforcerSpecs.clear();
// move LogicalExpression PhysicalExpression Ownership
Map<GroupExpression, GroupExpression> logicalSet =
target.getLogicalExpressions().stream()
@@ -390,10 +407,16 @@ public class Group {
lowestCostPlans.forEach((physicalProperties, costAndGroupExpr) -> {
// move lowestCostPlans Ownership
if (!target.lowestCostPlans.containsKey(physicalProperties)) {
+ // we must set owner group here, because the instance in
logical expression, physical expression
+ // and enforcer maybe not same with the instance in the
lowestCostPlans map
+ costAndGroupExpr.second.setOwnerGroup(target);
target.lowestCostPlans.put(physicalProperties,
costAndGroupExpr);
} else {
if (costAndGroupExpr.first.getValue()
<
target.lowestCostPlans.get(physicalProperties).first.getValue()) {
+ // we must set owner group here, because the instance in
logical expression, physical expression
+ // and enforcer maybe not same with the instance in the
lowestCostPlans map
+ costAndGroupExpr.second.setOwnerGroup(target);
target.lowestCostPlans.put(physicalProperties,
costAndGroupExpr);
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/GroupExpression.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/GroupExpression.java
index d6a9e29dd9e..96b4180db15 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/GroupExpression.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/GroupExpression.java
@@ -54,7 +54,7 @@ public class GroupExpression {
private static final EventProducer COST_STATE_TRACER = new
EventProducer(CostStateUpdateEvent.class,
EventChannel.getDefaultChannel().addConsumers(new
LogConsumer(CostStateUpdateEvent.class,
EventChannel.LOG)));
- private Cost cost;
+ private Cost cost = null;
private Group ownerGroup;
private final List<Group> children;
private final Plan plan;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriver.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriver.java
index a7dfc11046f..9416a4c3b91 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriver.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriver.java
@@ -76,6 +76,7 @@ import com.google.common.collect.Sets;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
@@ -101,7 +102,8 @@ public class ChildOutputPropertyDeriver extends
PlanVisitor<PhysicalProperties,
}
public PhysicalProperties getOutputProperties(ConnectContext
connectContext, GroupExpression groupExpression) {
- return groupExpression.getPlan().accept(this, new
PlanContext(connectContext, groupExpression));
+ return groupExpression.getPlan().accept(this,
+ new PlanContext(connectContext, groupExpression,
Collections.emptyList()));
}
@Override
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildrenPropertiesRegulator.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildrenPropertiesRegulator.java
index d6e0a4c8787..b415b1f8007 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildrenPropertiesRegulator.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildrenPropertiesRegulator.java
@@ -878,6 +878,9 @@ public class ChildrenPropertiesRegulator extends
PlanVisitor<List<List<PhysicalP
currentCost = newChildAndCost.first;
}
+ if (child.getOwnerGroup().getEnforcerSpecs().containsKey(target)) {
+ return;
+ }
PhysicalProperties newOutputProperty = new PhysicalProperties(target);
GroupExpression enforcer = target.addEnforcer(child.getOwnerGroup());
child.getOwnerGroup().addEnforcer(enforcer);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/DistributionSpecHiveTableSinkHashPartitioned.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/DistributionSpecHiveTableSinkHashPartitioned.java
index 4275bb53dd8..87a5eafc46f 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/DistributionSpecHiveTableSinkHashPartitioned.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/DistributionSpecHiveTableSinkHashPartitioned.java
@@ -20,6 +20,7 @@ package org.apache.doris.nereids.properties;
import org.apache.doris.nereids.trees.expressions.ExprId;
import java.util.List;
+import java.util.Objects;
/**
* use for shuffle data by partition keys before sink.
@@ -44,4 +45,21 @@ public class DistributionSpecHiveTableSinkHashPartitioned
extends DistributionSp
public boolean satisfy(DistributionSpec other) {
return other instanceof DistributionSpecHiveTableSinkHashPartitioned;
}
+
+ @Override
+ public boolean equals(Object o) {
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ if (!super.equals(o)) {
+ return false;
+ }
+ DistributionSpecHiveTableSinkHashPartitioned that =
(DistributionSpecHiveTableSinkHashPartitioned) o;
+ return Objects.equals(outputColExprIds, that.outputColExprIds);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(super.hashCode(), outputColExprIds);
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/EnforceMissingPropertiesHelper.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/EnforceMissingPropertiesHelper.java
index 6dc6fe565a2..81ce94e5f2f 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/EnforceMissingPropertiesHelper.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/EnforceMissingPropertiesHelper.java
@@ -123,7 +123,11 @@ public class EnforceMissingPropertiesHelper {
}
PhysicalProperties newOutputProperty = new
PhysicalProperties(outputDistributionSpec);
- GroupExpression enforcer =
outputDistributionSpec.addEnforcer(groupExpression.getOwnerGroup());
+ GroupExpression enforcer =
groupExpression.getOwnerGroup().getEnforcerSpecs().get(outputDistributionSpec);
+
+ if (enforcer == null) {
+ enforcer =
outputDistributionSpec.addEnforcer(groupExpression.getOwnerGroup());
+ }
addEnforcerUpdateCost(enforcer, oldOutputProperty, newOutputProperty);
return newOutputProperty;
}
@@ -160,10 +164,13 @@ public class EnforceMissingPropertiesHelper {
oldOutputProperty, newOutputProperty);
ENFORCER_TRACER.log(EnforcerEvent.of(groupExpression, ((PhysicalPlan)
enforcer.getPlan()),
oldOutputProperty, newOutputProperty));
-
enforcer.setEstOutputRowCount(enforcer.getOwnerGroup().getStatistics().getRowCount());
- Cost enforcerCost = CostCalculator.calculateCost(connectContext,
enforcer,
- Lists.newArrayList(oldOutputProperty));
- enforcer.setCost(enforcerCost);
+ Cost enforcerCost = enforcer.getCost();
+ if (enforcerCost == null) {
+
enforcer.setEstOutputRowCount(enforcer.getOwnerGroup().getStatistics().getRowCount());
+ enforcerCost = CostCalculator.calculateCost(connectContext,
enforcer,
+ Lists.newArrayList(oldOutputProperty));
+ enforcer.setCost(enforcerCost);
+ }
curTotalCost = CostCalculator.addChildCost(
connectContext,
enforcer.getPlan(),
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/RequestPropertyDeriver.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/RequestPropertyDeriver.java
index a5bf8e2fe92..4a39289c5f9 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/RequestPropertyDeriver.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/RequestPropertyDeriver.java
@@ -74,6 +74,7 @@ import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -114,7 +115,8 @@ public class RequestPropertyDeriver extends
PlanVisitor<Void, PlanContext> {
*/
public List<List<PhysicalProperties>>
getRequestChildrenPropertyList(GroupExpression groupExpression) {
requestPropertyToChildren = Lists.newArrayList();
- groupExpression.getPlan().accept(this, new PlanContext(connectContext,
groupExpression));
+ groupExpression.getPlan().accept(this,
+ new PlanContext(connectContext, groupExpression,
Collections.emptyList()));
return requestPropertyToChildren;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinLAsscomProject.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinLAsscomProject.java
index ac9787bed48..24fa7722970 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinLAsscomProject.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinLAsscomProject.java
@@ -23,7 +23,6 @@ import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.rules.exploration.CBOUtils;
import org.apache.doris.nereids.rules.exploration.OneExplorationRuleFactory;
import org.apache.doris.nereids.trees.expressions.ExprId;
-import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.plans.GroupPlan;
import org.apache.doris.nereids.trees.plans.JoinType;
import org.apache.doris.nereids.trees.plans.Plan;
@@ -36,7 +35,6 @@ import com.google.common.collect.ImmutableSet;
import java.util.HashSet;
import java.util.Set;
-import java.util.stream.Collectors;
import java.util.stream.Stream;
/**
@@ -116,11 +114,7 @@ public class OuterJoinLAsscomProject extends
OneExplorationRuleFactory {
topJoin.getHashJoinConjuncts().stream(),
topJoin.getOtherJoinConjuncts().stream())
.allMatch(expr -> {
- Set<ExprId> usedExprIdSet =
expr.<SlotReference>collect(SlotReference.class::isInstance)
- .stream()
- .map(SlotReference::getExprId)
- .collect(Collectors.toSet());
- return !Utils.isIntersecting(usedExprIdSet,
bOutputExprIdSet);
+ return !Utils.isIntersecting(expr.getInputSlotExprIds(),
bOutputExprIdSet);
});
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Expression.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Expression.java
index d6b091b8718..3f5dbca2468 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Expression.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Expression.java
@@ -71,6 +71,16 @@ public abstract class Expression extends
AbstractTreeNode<Expression> implements
private final boolean hasUnbound;
private final Supplier<Set<Slot>> inputSlots = LazyCompute.of(
() -> collect(e -> e instanceof Slot && !(e instanceof
ArrayItemSlot)));
+ private final Supplier<Set<ExprId>> inputExprIds = LazyCompute.of(
+ () -> {
+ Set<Slot> inputSlots = getInputSlots();
+ Builder<ExprId> exprIds =
ImmutableSet.builderWithExpectedSize(inputSlots.size());
+ for (Slot inputSlot : inputSlots) {
+ exprIds.add(inputSlot.getExprId());
+ }
+ return exprIds.build();
+ }
+ );
private final int fastChildrenHashCode;
private final Supplier<String> toSqlCache =
LazyCompute.of(this::computeToSql);
private final Supplier<Integer> hashCodeCache =
LazyCompute.of(this::computeHashCode);
@@ -394,12 +404,7 @@ public abstract class Expression extends
AbstractTreeNode<Expression> implements
* Note that the input slots of subquery's inner plan is not included.
*/
public final Set<ExprId> getInputSlotExprIds() {
- Set<Slot> inputSlots = getInputSlots();
- Builder<ExprId> exprIds =
ImmutableSet.builderWithExpectedSize(inputSlots.size());
- for (Slot inputSlot : inputSlots) {
- exprIds.add(inputSlot.getExprId());
- }
- return exprIds.build();
+ return inputExprIds.get();
}
public boolean isLiteral() {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashJoin.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashJoin.java
index ca84e1afc2f..d90a74f4199 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashJoin.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashJoin.java
@@ -131,13 +131,11 @@ public class PhysicalHashJoin<
// mark join with non-empty hash join conjuncts allow shuffle join by
hash join conjuncts
Preconditions.checkState(!(isMarkJoin() &&
hashJoinConjuncts.isEmpty()),
"shouldn't call mark join's getHashConjunctsExprIds method for
standalone mark join");
- int size = hashJoinConjuncts.size();
-
- List<ExprId> exprIds1 = new ArrayList<>(size);
- List<ExprId> exprIds2 = new ArrayList<>(size);
Set<ExprId> leftExprIds = left().getOutputExprIdSet();
Set<ExprId> rightExprIds = right().getOutputExprIdSet();
+ List<ExprId> exprIds1 = new ArrayList<>(leftExprIds.size());
+ List<ExprId> exprIds2 = new ArrayList<>(rightExprIds.size());
for (Expression expr : hashJoinConjuncts) {
for (ExprId exprId : expr.getInputSlotExprIds()) {
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/cascades/DeriveStatsJobTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/cascades/DeriveStatsJobTest.java
index 181ab5ba08c..aa5a21adcdc 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/cascades/DeriveStatsJobTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/cascades/DeriveStatsJobTest.java
@@ -68,7 +68,7 @@ public class DeriveStatsJobTest {
LogicalAggregate agg = constructAgg(olapScan);
CascadesContext cascadesContext =
MemoTestUtils.createCascadesContext(agg);
new
DeriveStatsJob(cascadesContext.getMemo().getRoot().getLogicalExpression(),
- new JobContext(cascadesContext, null,
Double.MAX_VALUE)).execute();
+ new JobContext(cascadesContext, null)).execute();
while (!cascadesContext.getJobPool().isEmpty()) {
cascadesContext.getJobPool().pop().execute();
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/ConstantPropagationTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/ConstantPropagationTest.java
index a1ef8f8d7b4..285e8313e6a 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/ConstantPropagationTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/ConstantPropagationTest.java
@@ -88,7 +88,7 @@ class ConstantPropagationTest {
ConstantPropagationTest() {
cascadesContext = MemoTestUtils.createCascadesContext(
new UnboundRelation(new RelationId(1),
ImmutableList.of("tbl")));
- jobContext = new JobContext(cascadesContext, null, Double.MAX_VALUE);
+ jobContext = new JobContext(cascadesContext, null);
student = new LogicalOlapScan(PlanConstructor.getNextRelationId(),
PlanConstructor.student, ImmutableList.of(""));
studentId = (SlotReference) student.getOutput().get(0);
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java
index c23ff24c868..5084ef09823 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java
@@ -700,8 +700,7 @@ public class PlanChecker {
public PlanChecker setMaxInvokeTimesPerRule(int maxInvokeTime) {
JobContext originJobContext = cascadesContext.getCurrentJobContext();
cascadesContext.setCurrentJobContext(
- new JobContext(cascadesContext,
- originJobContext.getRequiredProperties(),
originJobContext.getCostUpperBound()) {
+ new JobContext(cascadesContext,
originJobContext.getRequiredProperties()) {
@Override
public void onInvokeRule(RuleType ruleType) {
// add invoke times
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
