This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 6d6f46e20f6 [nereids](mtmv) Support rewrite by mv nested materialized
view (#33362)
6d6f46e20f6 is described below
commit 6d6f46e20f6bc7d2a1a60281864ac283e18aa501
Author: seawinde <[email protected]>
AuthorDate: Sun Apr 21 00:09:22 2024 +0800
[nereids](mtmv) Support rewrite by mv nested materialized view (#33362)
Support query rewritting by nested materialized view.
Such as `inner_mv` def is as following
> select
> l_linenumber,
> o_custkey,
> o_orderkey,
> o_orderstatus,
> l_partkey,
> l_suppkey,
> l_orderkey
> from lineitem
> inner join orders on lineitem.l_orderkey = orders.o_orderkey;
the mv1_0 def is as following:
> select
> l_linenumber,
> o_custkey,
> o_orderkey,
> o_orderstatus,
> l_partkey,
> l_suppkey,
> l_orderkey,
> ps_availqty
> from inner_mv
> inner join partsupp on l_partkey = ps_partkey AND l_suppkey =
ps_suppkey;
for the following query, both inner_mv and mv1_0 can be successful when
query rewritting by materialized view，and cbo will chose `mv1_0` finally.
> select lineitem.l_linenumber
> from lineitem
> inner join orders on l_orderkey = o_orderkey
> inner join partsupp on l_partkey = ps_partkey AND l_suppkey
= ps_suppkey
> where o_orderstatus = 'o' AND l_linenumber in (1, 2, 3, 4, 5)
---
.../jobs/joinorder/hypergraph/HyperGraph.java | 4 -
.../java/org/apache/doris/nereids/memo/Group.java | 20 +--
.../apache/doris/nereids/memo/StructInfoMap.java | 34 ++--
.../mv/AbstractMaterializedViewAggregateRule.java | 63 ++++---
.../mv/AbstractMaterializedViewJoinRule.java | 5 +-
.../mv/AbstractMaterializedViewRule.java | 148 +++++++----------
.../mv/InitMaterializationContextHook.java | 27 +--
.../mv/LogicalCompatibilityContext.java | 2 +-
.../exploration/mv/MaterializationContext.java | 71 ++++++--
.../mv/MaterializedViewAggregateRule.java | 3 +-
.../mv/MaterializedViewFilterAggregateRule.java | 10 +-
.../mv/MaterializedViewFilterJoinRule.java | 5 +-
...MaterializedViewFilterProjectAggregateRule.java | 10 +-
.../mv/MaterializedViewFilterProjectJoinRule.java | 10 +-
.../mv/MaterializedViewOnlyJoinRule.java | 5 +-
.../mv/MaterializedViewProjectAggregateRule.java | 10 +-
...MaterializedViewProjectFilterAggregateRule.java | 5 +-
.../mv/MaterializedViewProjectFilterJoinRule.java | 10 +-
.../mv/MaterializedViewProjectJoinRule.java | 10 +-
.../exploration/mv/MaterializedViewUtils.java | 55 +++++--
.../nereids/rules/exploration/mv/StructInfo.java | 101 +++++++-----
.../plans/visitor/ExpressionLineageReplacer.java | 37 +++--
.../apache/doris/nereids/util/ExpressionUtils.java | 15 +-
.../joinorder/hypergraph/CompareOuterJoinTest.java | 29 ++--
.../jobs/joinorder/hypergraph/InferJoinTest.java | 21 +--
.../joinorder/hypergraph/InferPredicateTest.java | 10 +-
.../joinorder/hypergraph/PullupExpressionTest.java | 22 +--
.../doris/nereids/memo/StructInfoMapTest.java | 2 +-
.../rules/exploration/mv/BuildStructInfoTest.java | 10 +-
.../rules/exploration/mv/EliminateJoinTest.java | 30 ++--
.../rules/exploration/mv/HyperGraphAggTest.java | 11 +-
.../exploration/mv/HyperGraphComparatorTest.java | 22 +--
.../doris/nereids/util/ExpressionUtilsTest.java | 8 +-
.../mv/nested/nested_materialized_view.out | 20 +++
.../agg_with_roll_up/aggregate_with_roll_up.groovy | 56 ++++---
.../mv/join/inner/inner_join.groovy | 20 ++-
.../mv/nested/nested_materialized_view.groovy | 182 +++++++++++++++++++++
37 files changed, 718 insertions(+), 385 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/HyperGraph.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/HyperGraph.java
index 5e45fc0bdb8..19ff555469c 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/HyperGraph.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/HyperGraph.java
@@ -356,10 +356,6 @@ public class HyperGraph {
return new HyperGraph(finalOutputs, joinEdges, nodes, filterEdges,
complexProject);
}
- public List<HyperGraph> buildAll() {
- return ImmutableList.of(build());
- }
-
public void updateNode(int idx, Group group) {
Preconditions.checkArgument(nodes.get(idx) instanceof DPhyperNode);
nodes.set(idx, ((DPhyperNode) nodes.get(idx)).withGroup(group));
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Group.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Group.java
index 01968a03bef..a9d3d881491 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Group.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Group.java
@@ -21,7 +21,6 @@ import org.apache.doris.common.Pair;
import org.apache.doris.nereids.cost.Cost;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.properties.PhysicalProperties;
-import org.apache.doris.nereids.rules.exploration.mv.StructInfo;
import org.apache.doris.nereids.trees.expressions.literal.Literal;
import org.apache.doris.nereids.trees.plans.JoinType;
import org.apache.doris.nereids.trees.plans.Plan;
@@ -76,8 +75,6 @@ public class Group {
private int chosenGroupExpressionId = -1;
- private List<StructInfo> structInfos = new ArrayList<>();
-
private StructInfoMap structInfoMap = new StructInfoMap();
/**
@@ -472,6 +469,7 @@ public class Group {
}
str.append(" stats").append("\n");
str.append(getStatistics() == null ? "" : getStatistics().detail("
"));
+
str.append(" lowest Plan(cost, properties, plan, childrenRequires)");
getAllProperties().forEach(
prop -> {
@@ -485,6 +483,10 @@ public class Group {
}
}
);
+
+ str.append("\n").append(" struct info map").append("\n");
+ str.append(structInfoMap);
+
return str.toString();
}
@@ -557,16 +559,4 @@ public class Group {
return TreeStringUtils.treeString(this, toString, getChildren,
getExtraPlans, displayExtraPlan);
}
-
- public List<StructInfo> getStructInfos() {
- return structInfos;
- }
-
- public void addStructInfo(StructInfo structInfo) {
- this.structInfos.add(structInfo);
- }
-
- public void addStructInfo(List<StructInfo> structInfos) {
- this.structInfos.addAll(structInfos);
- }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java
index d065c5cd3e2..a14c2070a8f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java
@@ -26,6 +26,7 @@ import com.google.common.collect.Sets;
import java.util.ArrayList;
import java.util.BitSet;
+import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -49,7 +50,7 @@ public class StructInfoMap {
* @param group the group that the mv matched
* @return struct info or null if not found
*/
- public @Nullable StructInfo getStructInfo(BitSet mvTableMap, BitSet
foldTableMap, Group group) {
+ public @Nullable StructInfo getStructInfo(BitSet mvTableMap, BitSet
foldTableMap, Group group, Plan originPlan) {
if (!infoMap.containsKey(mvTableMap)) {
if ((groupExpressionMap.containsKey(foldTableMap) ||
groupExpressionMap.isEmpty())
&& !groupExpressionMap.containsKey(mvTableMap)) {
@@ -59,11 +60,10 @@ public class StructInfoMap {
Pair<GroupExpression, List<BitSet>> groupExpressionBitSetPair
= getGroupExpressionWithChildren(
mvTableMap);
StructInfo structInfo =
constructStructInfo(groupExpressionBitSetPair.first,
- groupExpressionBitSetPair.second, mvTableMap);
+ groupExpressionBitSetPair.second, mvTableMap,
originPlan);
infoMap.put(mvTableMap, structInfo);
}
}
-
return infoMap.get(mvTableMap);
}
@@ -71,13 +71,19 @@ public class StructInfoMap {
return groupExpressionMap.keySet();
}
+ public Collection<StructInfo> getStructInfos() {
+ return infoMap.values();
+ }
+
public Pair<GroupExpression, List<BitSet>>
getGroupExpressionWithChildren(BitSet tableMap) {
return groupExpressionMap.get(tableMap);
}
- private StructInfo constructStructInfo(GroupExpression groupExpression,
List<BitSet> children, BitSet tableMap) {
+ private StructInfo constructStructInfo(GroupExpression groupExpression,
List<BitSet> children,
+ BitSet tableMap, Plan originPlan) {
+ // this plan is not origin plan, should record origin plan in struct
info
Plan plan = constructPlan(groupExpression, children, tableMap);
- return StructInfo.of(plan).get(0);
+ return originPlan == null ? StructInfo.of(plan) : StructInfo.of(plan,
originPlan);
}
private Plan constructPlan(GroupExpression groupExpression, List<BitSet>
children, BitSet tableMap) {
@@ -120,12 +126,11 @@ public class StructInfoMap {
refreshedGroup.add(child);
childrenTableMap.add(child.getstructInfoMap().getTableMaps());
}
-
- if (needRefresh) {
- Set<Pair<BitSet, List<BitSet>>> bitSetWithChildren =
cartesianProduct(childrenTableMap);
- for (Pair<BitSet, List<BitSet>> bitSetWithChild :
bitSetWithChildren) {
- groupExpressionMap.put(bitSetWithChild.first,
Pair.of(groupExpression, bitSetWithChild.second));
- }
+ // if cumulative child table map is different from current
+ // or current group expression map is empty, should update the
groupExpressionMap currently
+ Set<Pair<BitSet, List<BitSet>>> bitSetWithChildren =
cartesianProduct(childrenTableMap);
+ for (Pair<BitSet, List<BitSet>> bitSetWithChild :
bitSetWithChildren) {
+ groupExpressionMap.putIfAbsent(bitSetWithChild.first,
Pair.of(groupExpression, bitSetWithChild.second));
}
}
return originSize != groupExpressionMap.size();
@@ -135,7 +140,7 @@ public class StructInfoMap {
Plan plan = groupExpression.getPlan();
BitSet tableMap = new BitSet();
if (plan instanceof LogicalCatalogRelation) {
- // TODO: Bitmap is not compatible with long, use tree map instead
+ // TODO: Bitset is not compatible with long, use tree map instead
tableMap.set((int) ((LogicalCatalogRelation)
plan).getTable().getId());
}
// one row relation / CTE consumer
@@ -154,4 +159,9 @@ public class StructInfoMap {
})
.collect(Collectors.toSet());
}
+
+ @Override
+ public String toString() {
+ return "StructInfoMap{ groupExpressionMap = " + groupExpressionMap +
", infoMap = " + infoMap + '}';
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java
index 5cd2fb769b7..5c8c479986a 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java
@@ -56,6 +56,7 @@ import com.google.common.collect.Multimap;
import com.google.common.collect.Sets;
import java.util.ArrayList;
+import java.util.BitSet;
import java.util.Collection;
import java.util.HashSet;
import java.util.List;
@@ -180,18 +181,27 @@ public abstract class
AbstractMaterializedViewAggregateRule extends AbstractMate
}
// Firstly,if group by expression between query and view is equals,
try to rewrite expression directly
Plan queryTopPlan = queryTopPlanAndAggPair.key();
- if (isGroupByEquals(queryTopPlanAndAggPair, viewTopPlanAndAggPair,
viewToQuerySlotMapping)) {
+ if (isGroupByEquals(queryTopPlanAndAggPair, viewTopPlanAndAggPair,
viewToQuerySlotMapping, queryStructInfo,
+ viewStructInfo)) {
List<Expression> rewrittenQueryExpressions =
rewriteExpression(queryTopPlan.getOutput(),
queryTopPlan,
materializationContext.getMvExprToMvScanExprMapping(),
viewToQuerySlotMapping,
- true);
+ true,
+ queryStructInfo.getTableBitSet());
if (!rewrittenQueryExpressions.isEmpty()) {
- return new LogicalProject<>(
-
rewrittenQueryExpressions.stream().map(NamedExpression.class::cast)
- .collect(Collectors.toList()),
- tempRewritedPlan);
-
+ List<NamedExpression> projects = new ArrayList<>();
+ for (Expression expression : rewrittenQueryExpressions) {
+ if (expression.containsType(AggregateFunction.class)) {
+
materializationContext.recordFailReason(queryStructInfo,
+ "rewritten expression contains aggregate
functions when group equals aggregate rewrite",
+ () -> String.format("aggregate functions =
%s\n", rewrittenQueryExpressions));
+ return null;
+ }
+ projects.add(expression instanceof NamedExpression
+ ? (NamedExpression) expression : new
Alias(expression));
+ }
+ return new LogicalProject<>(projects, tempRewritedPlan);
}
// if fails, record the reason and then try to roll up aggregate
function
materializationContext.recordFailReason(queryStructInfo,
@@ -219,7 +229,7 @@ public abstract class AbstractMaterializedViewAggregateRule
extends AbstractMate
// try to roll up.
// split the query top plan expressions to group expressions and
functions, if can not, bail out.
Pair<Set<? extends Expression>, Set<? extends Expression>>
queryGroupAndFunctionPair
- = topPlanSplitToGroupAndFunction(queryTopPlanAndAggPair);
+ = topPlanSplitToGroupAndFunction(queryTopPlanAndAggPair,
queryStructInfo);
Set<? extends Expression> queryTopPlanFunctionSet =
queryGroupAndFunctionPair.value();
// try to rewrite, contains both roll up aggregate functions and
aggregate group expression
List<NamedExpression> finalOutputExpressions = new ArrayList<>();
@@ -234,9 +244,10 @@ public abstract class
AbstractMaterializedViewAggregateRule extends AbstractMate
if (queryTopPlanFunctionSet.contains(topExpression)) {
Expression queryFunctionShuttled =
ExpressionUtils.shuttleExpressionWithLineage(
topExpression,
- queryTopPlan);
+ queryTopPlan,
+ queryStructInfo.getTableBitSet());
AggregateExpressionRewriteContext context = new
AggregateExpressionRewriteContext(
- false, mvExprToMvScanExprQueryBased, queryTopPlan);
+ false, mvExprToMvScanExprQueryBased, queryTopPlan,
queryStructInfo.getTableBitSet());
// queryFunctionShuttled maybe sum(column) + count(*), so need
to use expression rewriter
Expression rollupedExpression =
queryFunctionShuttled.accept(AGGREGATE_EXPRESSION_REWRITER,
context);
@@ -250,10 +261,10 @@ public abstract class
AbstractMaterializedViewAggregateRule extends AbstractMate
finalOutputExpressions.add(new Alias(rollupedExpression));
} else {
// if group by expression, try to rewrite group by expression
- Expression queryGroupShuttledExpr =
-
ExpressionUtils.shuttleExpressionWithLineage(topExpression, queryTopPlan);
- AggregateExpressionRewriteContext context = new
AggregateExpressionRewriteContext(
- true, mvExprToMvScanExprQueryBased, queryTopPlan);
+ Expression queryGroupShuttledExpr =
ExpressionUtils.shuttleExpressionWithLineage(
+ topExpression, queryTopPlan,
queryStructInfo.getTableBitSet());
+ AggregateExpressionRewriteContext context = new
AggregateExpressionRewriteContext(true,
+ mvExprToMvScanExprQueryBased, queryTopPlan,
queryStructInfo.getTableBitSet());
// group by expression maybe group by a + b, so we need
expression rewriter
Expression rewrittenGroupByExpression =
queryGroupShuttledExpr.accept(AGGREGATE_EXPRESSION_REWRITER,
context);
@@ -302,16 +313,18 @@ public abstract class
AbstractMaterializedViewAggregateRule extends AbstractMate
private boolean isGroupByEquals(Pair<Plan, LogicalAggregate<Plan>>
queryTopPlanAndAggPair,
Pair<Plan, LogicalAggregate<Plan>> viewTopPlanAndAggPair,
- SlotMapping viewToQuerySlotMapping) {
+ SlotMapping viewToQuerySlotMapping,
+ StructInfo queryStructInfo,
+ StructInfo viewStructInfo) {
Plan queryTopPlan = queryTopPlanAndAggPair.key();
Plan viewTopPlan = viewTopPlanAndAggPair.key();
LogicalAggregate<Plan> queryAggregate = queryTopPlanAndAggPair.value();
LogicalAggregate<Plan> viewAggregate = viewTopPlanAndAggPair.value();
Set<? extends Expression> queryGroupShuttledExpression = new HashSet<>(
ExpressionUtils.shuttleExpressionWithLineage(
- queryAggregate.getGroupByExpressions(), queryTopPlan));
+ queryAggregate.getGroupByExpressions(), queryTopPlan,
queryStructInfo.getTableBitSet()));
Set<? extends Expression> viewGroupShuttledExpressionQueryBased =
ExpressionUtils.shuttleExpressionWithLineage(
- viewAggregate.getGroupByExpressions(), viewTopPlan)
+ viewAggregate.getGroupByExpressions(), viewTopPlan,
viewStructInfo.getTableBitSet())
.stream()
.map(expr -> ExpressionUtils.replace(expr,
viewToQuerySlotMapping.toSlotReferenceMap()))
.collect(Collectors.toSet());
@@ -384,7 +397,7 @@ public abstract class AbstractMaterializedViewAggregateRule
extends AbstractMate
}
private Pair<Set<? extends Expression>, Set<? extends Expression>>
topPlanSplitToGroupAndFunction(
- Pair<Plan, LogicalAggregate<Plan>> topPlanAndAggPair) {
+ Pair<Plan, LogicalAggregate<Plan>> topPlanAndAggPair, StructInfo
queryStructInfo) {
LogicalAggregate<Plan> bottomQueryAggregate =
topPlanAndAggPair.value();
Set<Expression> groupByExpressionSet = new
HashSet<>(bottomQueryAggregate.getGroupByExpressions());
// when query is bitmap_count(bitmap_union), the plan is as following:
@@ -403,7 +416,7 @@ public abstract class AbstractMaterializedViewAggregateRule
extends AbstractMate
queryTopPlan.getOutput().forEach(expression -> {
ExpressionLineageReplacer.ExpressionReplaceContext replaceContext =
new
ExpressionLineageReplacer.ExpressionReplaceContext(ImmutableList.of(expression),
- ImmutableSet.of(), ImmutableSet.of());
+ ImmutableSet.of(), ImmutableSet.of(),
queryStructInfo.getTableBitSet());
queryTopPlan.accept(ExpressionLineageReplacer.INSTANCE,
replaceContext);
if (!Sets.intersection(bottomAggregateFunctionExprIdSet,
replaceContext.getExprIdExpressionMap().keySet()).isEmpty()) {
@@ -509,7 +522,8 @@ public abstract class AbstractMaterializedViewAggregateRule
extends AbstractMate
}
Expression queryFunctionShuttled =
ExpressionUtils.shuttleExpressionWithLineage(
aggregateFunction,
- rewriteContext.getQueryTopPlan());
+ rewriteContext.getQueryTopPlan(),
+ rewriteContext.getQueryTableBitSet());
Function rollupAggregateFunction = rollup(aggregateFunction,
queryFunctionShuttled,
rewriteContext.getMvExprToMvScanExprQueryBasedMapping());
if (rollupAggregateFunction == null) {
@@ -565,12 +579,15 @@ public abstract class
AbstractMaterializedViewAggregateRule extends AbstractMate
private final boolean onlyContainGroupByExpression;
private final Map<Expression, Expression>
mvExprToMvScanExprQueryBasedMapping;
private final Plan queryTopPlan;
+ private final BitSet queryTableBitSet;
public AggregateExpressionRewriteContext(boolean
onlyContainGroupByExpression,
- Map<Expression, Expression>
mvExprToMvScanExprQueryBasedMapping, Plan queryTopPlan) {
+ Map<Expression, Expression>
mvExprToMvScanExprQueryBasedMapping, Plan queryTopPlan,
+ BitSet queryTableBitSet) {
this.onlyContainGroupByExpression = onlyContainGroupByExpression;
this.mvExprToMvScanExprQueryBasedMapping =
mvExprToMvScanExprQueryBasedMapping;
this.queryTopPlan = queryTopPlan;
+ this.queryTableBitSet = queryTableBitSet;
}
public boolean isValid() {
@@ -592,5 +609,9 @@ public abstract class AbstractMaterializedViewAggregateRule
extends AbstractMate
public Plan getQueryTopPlan() {
return queryTopPlan;
}
+
+ public BitSet getQueryTableBitSet() {
+ return queryTableBitSet;
+ }
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewJoinRule.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewJoinRule.java
index df58345405a..2a05fecd33f 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewJoinRule.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewJoinRule.java
@@ -44,10 +44,11 @@ public abstract class AbstractMaterializedViewJoinRule
extends AbstractMateriali
// Rewrite top projects, represent the query projects by view
List<Expression> expressionsRewritten = rewriteExpression(
queryStructInfo.getExpressions(),
- queryStructInfo.getOriginalPlan(),
+ queryStructInfo.getTopPlan(),
materializationContext.getMvExprToMvScanExprMapping(),
targetToSourceMapping,
- true
+ true,
+ queryStructInfo.getTableBitSet()
);
// Can not rewrite, bail out
if (expressionsRewritten.isEmpty()) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
index 90e0f8ed1db..b596408851f 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
@@ -38,6 +38,7 @@ import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.Not;
import org.apache.doris.nereids.trees.expressions.Slot;
+import
org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction;
import org.apache.doris.nereids.trees.expressions.functions.scalar.NonNullable;
import org.apache.doris.nereids.trees.expressions.functions.scalar.Nullable;
import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral;
@@ -57,6 +58,7 @@ import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import java.util.ArrayList;
+import java.util.BitSet;
import java.util.Collection;
import java.util.List;
import java.util.Map;
@@ -85,52 +87,62 @@ public abstract class AbstractMaterializedViewRule
implements ExplorationRuleFac
*/
public List<Plan> rewrite(Plan queryPlan, CascadesContext cascadesContext)
{
List<Plan> rewrittenPlans = new ArrayList<>();
- // already rewrite or query is invalid, bail out
- List<StructInfo> queryStructInfos = checkQuery(queryPlan,
cascadesContext);
- if (queryStructInfos.isEmpty()) {
+ // if available materialization list is empty, bail out
+ if (cascadesContext.getMaterializationContexts().isEmpty()) {
return rewrittenPlans;
}
for (MaterializationContext context :
cascadesContext.getMaterializationContexts()) {
if (checkIfRewritten(queryPlan, context)) {
continue;
}
- // TODO Just support only one query struct info, support multi
later.
- StructInfo queryStructInfo = queryStructInfos.get(0);
- try {
- if (rewrittenPlans.size() < cascadesContext.getConnectContext()
-
.getSessionVariable().getMaterializedViewRewriteSuccessCandidateNum()) {
- rewrittenPlans.addAll(doRewrite(queryStructInfo,
cascadesContext, context));
+ context.tryReGenerateMvScanPlan(cascadesContext);
+ // check mv plan is valid or not
+ if (!checkPattern(context.getStructInfo())) {
+ context.recordFailReason(context.getStructInfo(),
+ "View struct info is invalid", () -> String.format(",
view plan is %s",
+
context.getStructInfo().getOriginalPlan().treeString()));
+ continue;
+ }
+ // get query struct infos according to the view strut info, if
valid query struct infos is empty, bail out
+ List<StructInfo> queryStructInfos =
getValidQueryStructInfos(queryPlan, cascadesContext,
+ context.getStructInfo().getTableBitSet());
+ if (queryStructInfos.isEmpty()) {
+ continue;
+ }
+ for (StructInfo queryStructInfo : queryStructInfos) {
+ try {
+ if (rewrittenPlans.size() <
cascadesContext.getConnectContext()
+
.getSessionVariable().getMaterializedViewRewriteSuccessCandidateNum()) {
+ rewrittenPlans.addAll(doRewrite(queryStructInfo,
cascadesContext, context));
+ }
+ } catch (Exception exception) {
+ context.recordFailReason(queryStructInfo,
+ "Materialized view rule exec fail",
exception::toString);
}
- } catch (Exception exception) {
- context.recordFailReason(queryStructInfo,
- "Materialized view rule exec fail",
exception::toString);
}
}
return rewrittenPlans;
}
/**
- * Check query is valid or not, if valid return the query struct infos, if
invalid return empty list.
+ * Get valid query struct infos, if invalid record the invalid reason
*/
- protected List<StructInfo> checkQuery(Plan queryPlan, CascadesContext
cascadesContext) {
- List<StructInfo> validQueryStructInfos = new ArrayList<>();
- List<MaterializationContext> materializationContexts =
cascadesContext.getMaterializationContexts();
- if (materializationContexts.isEmpty()) {
- return validQueryStructInfos;
- }
- List<StructInfo> queryStructInfos =
MaterializedViewUtils.extractStructInfo(queryPlan, cascadesContext);
- // TODO Just Check query queryPlan firstly, support multi later.
- StructInfo queryStructInfo = queryStructInfos.get(0);
- if (!checkPattern(queryStructInfo)) {
- for (MaterializationContext ctx :
cascadesContext.getMaterializationContexts()) {
- ctx.recordFailReason(queryStructInfo, "Query struct info is
invalid",
- () -> String.format("queryPlan is %s",
queryPlan.treeString())
- );
- }
- return validQueryStructInfos;
- }
- validQueryStructInfos.add(queryStructInfo);
- return validQueryStructInfos;
+ protected List<StructInfo> getValidQueryStructInfos(Plan queryPlan,
CascadesContext cascadesContext,
+ BitSet materializedViewTableSet) {
+ return MaterializedViewUtils.extractStructInfo(queryPlan,
cascadesContext, materializedViewTableSet)
+ .stream()
+ .filter(queryStructInfo -> {
+ boolean valid = checkPattern(queryStructInfo);
+ if (!valid) {
+
cascadesContext.getMaterializationContexts().forEach(ctx ->
+ ctx.recordFailReason(queryStructInfo, "Query
struct info is invalid",
+ () -> String.format("query table
bitmap is %s, plan is %s",
+
queryStructInfo.getTableBitSet(), queryPlan.treeString())
+ ));
+ }
+ return valid;
+ })
+ .collect(Collectors.toList());
}
/**
@@ -140,22 +152,7 @@ public abstract class AbstractMaterializedViewRule
implements ExplorationRuleFac
protected List<Plan> doRewrite(StructInfo queryStructInfo, CascadesContext
cascadesContext,
MaterializationContext materializationContext) {
List<Plan> rewriteResults = new ArrayList<>();
- List<StructInfo> viewStructInfos =
MaterializedViewUtils.extractStructInfo(
- materializationContext.getMvPlan(), cascadesContext);
- if (viewStructInfos.size() > 1) {
- // view struct info should only have one
- materializationContext.recordFailReason(queryStructInfo,
- "The num of view struct info is more then one",
- () -> String.format("mv plan is %s",
materializationContext.getMvPlan().treeString()));
- return rewriteResults;
- }
- StructInfo viewStructInfo = viewStructInfos.get(0);
- if (!checkPattern(viewStructInfo)) {
- materializationContext.recordFailReason(queryStructInfo,
- "View struct info is invalid",
- () -> String.format(", view plan is %s",
viewStructInfo.getOriginalPlan().treeString()));
- return rewriteResults;
- }
+ StructInfo viewStructInfo = materializationContext.getStructInfo();
MatchMode matchMode = decideMatchMode(queryStructInfo.getRelations(),
viewStructInfo.getRelations());
if (MatchMode.COMPLETE != matchMode) {
materializationContext.recordFailReason(queryStructInfo, "Match
mode is invalid",
@@ -178,15 +175,6 @@ public abstract class AbstractMaterializedViewRule
implements ExplorationRuleFac
continue;
}
SlotMapping viewToQuerySlotMapping =
queryToViewSlotMapping.inverse();
- // check the column used in query is in mv or not
- if (!checkColumnUsedValid(queryStructInfo, viewStructInfo,
queryToViewSlotMapping)) {
- materializationContext.recordFailReason(queryStructInfo,
- "The columns used by query are not in view",
- () -> String.format("query struct info is %s, view
struct info is %s",
- queryStructInfo.getTopPlan().treeString(),
- viewStructInfo.getTopPlan().treeString()));
- continue;
- }
LogicalCompatibilityContext compatibilityContext =
LogicalCompatibilityContext.from(
queryToViewTableMapping, queryToViewSlotMapping,
queryStructInfo, viewStructInfo);
ComparisonResult comparisonResult =
StructInfo.isGraphLogicalEquals(queryStructInfo, viewStructInfo,
@@ -212,14 +200,14 @@ public abstract class AbstractMaterializedViewRule
implements ExplorationRuleFac
}
Plan rewrittenPlan;
Plan mvScan = materializationContext.getMvScanPlan();
- Plan originalPlan = queryStructInfo.getOriginalPlan();
+ Plan topPlan = queryStructInfo.getTopPlan();
if (compensatePredicates.isAlwaysTrue()) {
rewrittenPlan = mvScan;
} else {
// Try to rewrite compensate predicates by using mv scan
List<Expression> rewriteCompensatePredicates =
rewriteExpression(compensatePredicates.toList(),
- originalPlan,
materializationContext.getMvExprToMvScanExprMapping(),
- viewToQuerySlotMapping, true);
+ topPlan,
materializationContext.getMvExprToMvScanExprMapping(),
+ viewToQuerySlotMapping, true,
queryStructInfo.getTableBitSet());
if (rewriteCompensatePredicates.isEmpty()) {
materializationContext.recordFailReason(queryStructInfo,
"Rewrite compensate predicate by view fail",
@@ -237,13 +225,13 @@ public abstract class AbstractMaterializedViewRule
implements ExplorationRuleFac
if (rewrittenPlan == null) {
continue;
}
- final Plan finalRewrittenPlan = rewriteByRules(cascadesContext,
rewrittenPlan, originalPlan);
- if (!isOutputValid(originalPlan, finalRewrittenPlan)) {
+ final Plan finalRewrittenPlan = rewriteByRules(cascadesContext,
rewrittenPlan, topPlan);
+ if (!isOutputValid(topPlan, finalRewrittenPlan)) {
materializationContext.recordFailReason(queryStructInfo,
"RewrittenPlan output logical properties is different
with target group",
() -> String.format("planOutput logical"
+ " properties = %s,\n groupOutput
logical properties = %s",
- finalRewrittenPlan.getLogicalProperties(),
originalPlan.getLogicalProperties()));
+ finalRewrittenPlan.getLogicalProperties(),
topPlan.getLogicalProperties()));
continue;
}
// check the partitions used by rewritten plan is valid or not
@@ -260,32 +248,12 @@ public abstract class AbstractMaterializedViewRule
implements ExplorationRuleFac
.collect(Collectors.toSet())));
continue;
}
- recordIfRewritten(originalPlan, materializationContext);
+ recordIfRewritten(queryStructInfo.getOriginalPlan(),
materializationContext);
rewriteResults.add(finalRewrittenPlan);
}
return rewriteResults;
}
- /**
- * Check the column used by query is in materialized view output or not
- */
- protected boolean checkColumnUsedValid(StructInfo queryInfo, StructInfo
mvInfo,
- SlotMapping queryToViewSlotMapping) {
- Set<ExprId> queryUsedSlotSetViewBased =
ExpressionUtils.shuttleExpressionWithLineage(
- queryInfo.getTopPlan().getOutput(),
queryInfo.getTopPlan()).stream()
- .flatMap(expr -> ExpressionUtils.replace(expr,
queryToViewSlotMapping.toSlotReferenceMap())
- .collectToSet(each -> each instanceof Slot).stream())
- .map(each -> ((Slot) each).getExprId())
- .collect(Collectors.toSet());
-
- Set<ExprId> viewUsedSlotSet =
ExpressionUtils.shuttleExpressionWithLineage(mvInfo.getTopPlan().getOutput(),
- mvInfo.getTopPlan()).stream()
- .flatMap(expr -> expr.collectToSet(each -> each instanceof
Slot).stream())
- .map(each -> ((Slot) each).getExprId())
- .collect(Collectors.toSet());
- return viewUsedSlotSet.containsAll(queryUsedSlotSetViewBased);
- }
-
/**
* Rewrite by rules and try to make output is the same after optimize by
rules
*/
@@ -398,13 +366,13 @@ public abstract class AbstractMaterializedViewRule
implements ExplorationRuleFac
*/
protected List<Expression> rewriteExpression(List<? extends Expression>
sourceExpressionsToWrite, Plan sourcePlan,
ExpressionMapping targetExpressionMapping, SlotMapping
targetToSourceMapping,
- boolean targetExpressionNeedSourceBased) {
+ boolean targetExpressionNeedSourceBased, BitSet sourcePlanBitSet) {
// Firstly, rewrite the target expression using source with inverse
mapping
// then try to use the target expression to represent the query. if
any of source expressions
// can not be represented by target expressions, return null.
// generate target to target replacement expression mapping, and
change target expression to source based
List<? extends Expression> sourceShuttledExpressions =
ExpressionUtils.shuttleExpressionWithLineage(
- sourceExpressionsToWrite, sourcePlan);
+ sourceExpressionsToWrite, sourcePlan, sourcePlanBitSet);
ExpressionMapping expressionMappingKeySourceBased =
targetExpressionNeedSourceBased
? targetExpressionMapping.keyPermute(targetToSourceMapping) :
targetExpressionMapping;
// target to target replacement expression mapping, because mv is 1:1
so get first element
@@ -516,6 +484,12 @@ public abstract class AbstractMaterializedViewRule
implements ExplorationRuleFac
|| residualCompensatePredicates == null) {
return SplitPredicate.INVALID_INSTANCE;
}
+ if (equalCompensateConjunctions.stream().anyMatch(expr ->
expr.containsType(AggregateFunction.class))
+ || rangeCompensatePredicates.stream().anyMatch(expr ->
expr.containsType(AggregateFunction.class))
+ || residualCompensatePredicates.stream().anyMatch(expr ->
+ expr.containsType(AggregateFunction.class))) {
+ return SplitPredicate.INVALID_INSTANCE;
+ }
return SplitPredicate.of(equalCompensateConjunctions.isEmpty() ?
BooleanLiteral.TRUE
: ExpressionUtils.and(equalCompensateConjunctions),
rangeCompensatePredicates.isEmpty() ? BooleanLiteral.TRUE
@@ -592,7 +566,7 @@ public abstract class AbstractMaterializedViewRule
implements ExplorationRuleFac
protected void recordIfRewritten(Plan plan, MaterializationContext
context) {
context.setSuccess(true);
if (plan.getGroupExpression().isPresent()) {
-
context.addMatchedGroup(plan.getGroupExpression().get().getOwnerGroup().getGroupId());
+
context.addMatchedGroup(plan.getGroupExpression().get().getOwnerGroup().getGroupId(),
true);
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java
index 2d5c9bf377b..08c312b4737 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java
@@ -24,23 +24,16 @@ import org.apache.doris.mtmv.BaseTableInfo;
import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.NereidsPlanner;
import org.apache.doris.nereids.PlannerHook;
-import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.plans.Plan;
-import org.apache.doris.nereids.trees.plans.PreAggStatus;
-import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
-import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
import org.apache.doris.nereids.trees.plans.visitor.TableCollector;
import
org.apache.doris.nereids.trees.plans.visitor.TableCollector.TableCollectorContext;
import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.List;
-import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
@@ -77,25 +70,15 @@ public class InitMaterializationContextHook implements
PlannerHook {
Set<MTMV> availableMTMVs =
Env.getCurrentEnv().getMtmvService().getRelationManager()
.getAvailableMTMVs(usedBaseTables,
cascadesContext.getConnectContext());
if (availableMTMVs.isEmpty()) {
+ LOG.warn(String.format("enable materialized view rewrite but
availableMTMVs is empty, current queryId "
+ + "is %s",
cascadesContext.getConnectContext().getQueryIdentifier()));
return;
}
for (MTMV materializedView : availableMTMVs) {
- // generate outside, maybe add partition filter in the future
- LogicalOlapScan mvScan = new LogicalOlapScan(
- cascadesContext.getStatementContext().getNextRelationId(),
- materializedView,
- ImmutableList.of(materializedView.getQualifiedDbName()),
- // this must be empty, or it will be used to sample
- Lists.newArrayList(),
- Lists.newArrayList(),
- Optional.empty());
- mvScan = mvScan.withMaterializedIndexSelected(PreAggStatus.on(),
materializedView.getBaseIndexId());
- List<NamedExpression> mvProjects =
mvScan.getOutput().stream().map(NamedExpression.class::cast)
- .collect(Collectors.toList());
- // todo should force keep consistency to mv sql plan output
- Plan projectScan = new LogicalProject<Plan>(mvProjects, mvScan);
cascadesContext.addMaterializationContext(
-
MaterializationContext.fromMaterializedView(materializedView, projectScan,
cascadesContext));
+
MaterializationContext.fromMaterializedView(materializedView,
+
MaterializedViewUtils.generateMvScanPlan(materializedView, cascadesContext),
+ cascadesContext));
}
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/LogicalCompatibilityContext.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/LogicalCompatibilityContext.java
index 3cd3000a670..7f92735da64 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/LogicalCompatibilityContext.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/LogicalCompatibilityContext.java
@@ -56,7 +56,7 @@ public class LogicalCompatibilityContext {
this.queryToViewNodeMapping = queryToViewNodeMapping;
this.queryToViewEdgeExpressionMapping =
queryToViewEdgeExpressionMapping;
this.queryToViewNodeIDMapping = HashBiMap.create();
- this.planNodeId =
queryStructInfo.getOriginalPlan().getGroupExpression()
+ this.planNodeId = queryStructInfo.getTopPlan().getGroupExpression()
.map(GroupExpression::getId).orElseGet(() -> new ObjectId(-1));
queryToViewNodeMapping.forEach((k, v) ->
queryToViewNodeIDMapping.put(k.getIndex(), v.getIndex()));
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java
index db9f58ae070..1af63dab21f 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java
@@ -26,6 +26,7 @@ import org.apache.doris.mtmv.MTMVCache;
import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.memo.GroupId;
import org.apache.doris.nereids.rules.exploration.mv.mapping.ExpressionMapping;
+import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.plans.ObjectId;
import org.apache.doris.nereids.trees.plans.Plan;
import
org.apache.doris.nereids.trees.plans.commands.ExplainCommand.ExplainLevel;
@@ -37,6 +38,7 @@ import com.google.common.collect.ImmutableSet;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
+import java.util.BitSet;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
@@ -53,23 +55,27 @@ public class MaterializationContext {
private static final Logger LOG =
LogManager.getLogger(MaterializationContext.class);
private final MTMV mtmv;
- // Should use stmt id generator in query context
- private final Plan mvScanPlan;
private final List<Table> baseTables;
private final List<Table> baseViews;
// Group ids that are rewritten by this mv to reduce rewrite times
- private final Set<GroupId> matchedGroups = new HashSet<>();
- // generate form mv scan plan
+ private final Set<GroupId> matchedFailGroups = new HashSet<>();
+ private final Set<GroupId> matchedSuccessGroups = new HashSet<>();
+ // if rewrite by mv fail, record the reason, if success the failReason
should be empty.
+ // The key is the query belonged group expression objectId, the value is
the fail reason
+ private final Map<ObjectId, Pair<String, String>> failReason = new
LinkedHashMap<>();
+ // Should regenerate when materialization is already rewritten
successfully because one query may hit repeatedly
+ // make sure output is different in multi using
+ private Plan mvScanPlan;
+ // generated expressions form mv scan plan
private ExpressionMapping mvExprToMvScanExprMapping;
+ private List<? extends Expression> mvPlanOutputShuttledExpressions;
private boolean available = true;
// the mv plan from cache at present, record it to make sure query rewrite
by mv is right when cache change.
private Plan mvPlan;
// mark rewrite success or not
private boolean success = false;
- // if rewrite by mv fail, record the reason, if success the failReason
should be empty.
- // The key is the query belonged group expression objectId, the value is
the fail reason
- private final Map<ObjectId, Pair<String, String>> failReason = new
LinkedHashMap<>();
private boolean enableRecordFailureDetail = false;
+ private StructInfo structInfo;
/**
* MaterializationContext, this contains necessary info for query
rewriting by mv
@@ -93,22 +99,50 @@ public class MaterializationContext {
this.available = false;
return;
}
+ this.mvPlanOutputShuttledExpressions =
ExpressionUtils.shuttleExpressionWithLineage(
+ mtmvCache.getOriginalPlan().getOutput(),
+ mtmvCache.getOriginalPlan(),
+ new BitSet());
// mv output expression shuttle, this will be used to expression
rewrite
- this.mvExprToMvScanExprMapping = ExpressionMapping.generate(
- ExpressionUtils.shuttleExpressionWithLineage(
- mtmvCache.getOriginalPlan().getOutput(),
- mtmvCache.getOriginalPlan()),
- mvScanPlan.getExpressions());
+ this.mvExprToMvScanExprMapping =
ExpressionMapping.generate(this.mvPlanOutputShuttledExpressions,
+ this.mvScanPlan.getExpressions());
// copy the plan from cache, which the plan in cache may change
this.mvPlan = mtmvCache.getLogicalPlan();
+ List<StructInfo> viewStructInfos =
MaterializedViewUtils.extractStructInfo(
+ mtmvCache.getLogicalPlan(), cascadesContext, new BitSet());
+ if (viewStructInfos.size() > 1) {
+ // view struct info should only have one, log error and use the
first struct info
+ LOG.warn(String.format("view strut info is more than one, mv name
is %s, mv plan is %s",
+ mtmv.getName(), mvPlan.treeString()));
+ }
+ this.structInfo = viewStructInfos.get(0);
}
public boolean alreadyRewrite(GroupId groupId) {
- return this.matchedGroups.contains(groupId);
+ return this.matchedFailGroups.contains(groupId) ||
this.matchedSuccessGroups.contains(groupId);
}
- public void addMatchedGroup(GroupId groupId) {
- matchedGroups.add(groupId);
+ public void addMatchedGroup(GroupId groupId, boolean rewriteSuccess) {
+ if (rewriteSuccess) {
+ this.matchedSuccessGroups.add(groupId);
+ } else {
+ this.matchedFailGroups.add(groupId);
+ }
+ }
+
+ /**
+ * Try to generate scan plan for materialized view
+ * if MaterializationContext is already rewritten by materialized view,
then should generate in real time
+ * when query rewrite, because one plan may hit the materialized view
repeatedly and the mv scan output
+ * should be different
+ */
+ public void tryReGenerateMvScanPlan(CascadesContext cascadesContext) {
+ if (!this.matchedSuccessGroups.isEmpty()) {
+ this.mvScanPlan =
MaterializedViewUtils.generateMvScanPlan(this.mtmv, cascadesContext);
+ // mv output expression shuttle, this will be used to expression
rewrite
+ this.mvExprToMvScanExprMapping =
ExpressionMapping.generate(this.mvPlanOutputShuttledExpressions,
+ this.mvScanPlan.getExpressions());
+ }
}
public MTMV getMTMV() {
@@ -152,13 +186,18 @@ public class MaterializationContext {
this.failReason.clear();
}
+ public StructInfo getStructInfo() {
+ return structInfo;
+ }
+
/**
* recordFailReason
*/
public void recordFailReason(StructInfo structInfo, String summary,
Supplier<String> failureReasonSupplier) {
// record it's rewritten
if (structInfo.getTopPlan().getGroupExpression().isPresent()) {
-
this.addMatchedGroup(structInfo.getTopPlan().getGroupExpression().get().getOwnerGroup().getGroupId());
+
this.addMatchedGroup(structInfo.getTopPlan().getGroupExpression().get().getOwnerGroup().getGroupId(),
+ false);
}
// once success, do not record the fail reason
if (this.success) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewAggregateRule.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewAggregateRule.java
index 8e0f8d6f717..20a7965f08b 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewAggregateRule.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewAggregateRule.java
@@ -21,6 +21,7 @@ import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate;
+import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import com.google.common.collect.ImmutableList;
@@ -36,7 +37,7 @@ public class MaterializedViewAggregateRule extends
AbstractMaterializedViewAggre
@Override
public List<Rule> buildRules() {
return ImmutableList.of(
- logicalAggregate(any()).thenApplyMultiNoThrow(ctx -> {
+
logicalAggregate(any().when(LogicalPlan.class::isInstance)).thenApplyMultiNoThrow(ctx
-> {
LogicalAggregate<Plan> root = ctx.root;
return rewrite(root, ctx.cascadesContext);
}).toRule(RuleType.MATERIALIZED_VIEW_ONLY_AGGREGATE));
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewFilterAggregateRule.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewFilterAggregateRule.java
index 40c8b6d681b..10b7aa5e84d 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewFilterAggregateRule.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewFilterAggregateRule.java
@@ -22,6 +22,7 @@ import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate;
import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
+import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import com.google.common.collect.ImmutableList;
@@ -37,9 +38,10 @@ public class MaterializedViewFilterAggregateRule extends
AbstractMaterializedVie
@Override
public List<Rule> buildRules() {
return ImmutableList.of(
-
logicalFilter(logicalAggregate(any())).thenApplyMultiNoThrow(ctx -> {
- LogicalFilter<LogicalAggregate<Plan>> root = ctx.root;
- return rewrite(root, ctx.cascadesContext);
- }).toRule(RuleType.MATERIALIZED_VIEW_FILTER_AGGREGATE));
+
logicalFilter(logicalAggregate(any().when(LogicalPlan.class::isInstance))).thenApplyMultiNoThrow(
+ ctx -> {
+ LogicalFilter<LogicalAggregate<Plan>> root =
ctx.root;
+ return rewrite(root, ctx.cascadesContext);
+
}).toRule(RuleType.MATERIALIZED_VIEW_FILTER_AGGREGATE));
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewFilterJoinRule.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewFilterJoinRule.java
index d1ffa522bb0..e7b12e6bb1f 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewFilterJoinRule.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewFilterJoinRule.java
@@ -22,6 +22,7 @@ import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
import org.apache.doris.nereids.trees.plans.logical.LogicalJoin;
+import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import com.google.common.collect.ImmutableList;
@@ -36,8 +37,8 @@ public class MaterializedViewFilterJoinRule extends
AbstractMaterializedViewJoin
@Override
public List<Rule> buildRules() {
- return ImmutableList.of(
- logicalFilter(logicalJoin(any(),
any())).thenApplyMultiNoThrow(ctx -> {
+ return
ImmutableList.of(logicalFilter(logicalJoin(any().when(LogicalPlan.class::isInstance),
+
any().when(LogicalPlan.class::isInstance))).thenApplyMultiNoThrow(ctx -> {
LogicalFilter<LogicalJoin<Plan, Plan>> root = ctx.root;
return rewrite(root, ctx.cascadesContext);
}).toRule(RuleType.MATERIALIZED_VIEW_FILTER_JOIN));
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewFilterProjectAggregateRule.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewFilterProjectAggregateRule.java
index 40655e97394..bd7437a7d90 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewFilterProjectAggregateRule.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewFilterProjectAggregateRule.java
@@ -22,6 +22,7 @@ import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate;
import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
+import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
import com.google.common.collect.ImmutableList;
@@ -39,9 +40,10 @@ public class MaterializedViewFilterProjectAggregateRule
extends AbstractMaterial
@Override
public List<Rule> buildRules() {
return ImmutableList.of(
-
logicalFilter(logicalProject(logicalAggregate(any()))).thenApplyMultiNoThrow(ctx
-> {
- LogicalFilter<LogicalProject<LogicalAggregate<Plan>>> root
= ctx.root;
- return rewrite(root, ctx.cascadesContext);
- }).toRule(RuleType.MATERIALIZED_VIEW_FILTER_AGGREGATE));
+
logicalFilter(logicalProject(logicalAggregate(any().when(LogicalPlan.class::isInstance))))
+ .thenApplyMultiNoThrow(ctx -> {
+
LogicalFilter<LogicalProject<LogicalAggregate<Plan>>> root = ctx.root;
+ return rewrite(root, ctx.cascadesContext);
+
}).toRule(RuleType.MATERIALIZED_VIEW_FILTER_AGGREGATE));
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewFilterProjectJoinRule.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewFilterProjectJoinRule.java
index 2c9cca199de..c18347dce08 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewFilterProjectJoinRule.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewFilterProjectJoinRule.java
@@ -22,6 +22,7 @@ import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
import org.apache.doris.nereids.trees.plans.logical.LogicalJoin;
+import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
import com.google.common.collect.ImmutableList;
@@ -38,9 +39,10 @@ public class MaterializedViewFilterProjectJoinRule extends
AbstractMaterializedV
@Override
public List<Rule> buildRules() {
return ImmutableList.of(
- logicalFilter(logicalProject(logicalJoin(any(),
any()))).thenApplyMultiNoThrow(ctx -> {
- LogicalFilter<LogicalProject<LogicalJoin<Plan, Plan>>>
root = ctx.root;
- return rewrite(root, ctx.cascadesContext);
- }).toRule(RuleType.MATERIALIZED_VIEW_FILTER_PROJECT_JOIN));
+
logicalFilter(logicalProject(logicalJoin(any().when(LogicalPlan.class::isInstance),
+
any().when(LogicalPlan.class::isInstance)))).thenApplyMultiNoThrow(ctx -> {
+ LogicalFilter<LogicalProject<LogicalJoin<Plan,
Plan>>> root = ctx.root;
+ return rewrite(root, ctx.cascadesContext);
+
}).toRule(RuleType.MATERIALIZED_VIEW_FILTER_PROJECT_JOIN));
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewOnlyJoinRule.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewOnlyJoinRule.java
index a9f8333f4df..2735ca87fe9 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewOnlyJoinRule.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewOnlyJoinRule.java
@@ -21,6 +21,7 @@ import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.logical.LogicalJoin;
+import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import com.google.common.collect.ImmutableList;
@@ -35,8 +36,8 @@ public class MaterializedViewOnlyJoinRule extends
AbstractMaterializedViewJoinRu
@Override
public List<Rule> buildRules() {
- return ImmutableList.of(
- logicalJoin(any(), any()).thenApplyMultiNoThrow(ctx -> {
+ return
ImmutableList.of(logicalJoin(any().when(LogicalPlan.class::isInstance),
+
any().when(LogicalPlan.class::isInstance)).thenApplyMultiNoThrow(ctx -> {
LogicalJoin<Plan, Plan> root = ctx.root;
return rewrite(root, ctx.cascadesContext);
}).toRule(RuleType.MATERIALIZED_VIEW_ONLY_JOIN));
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectAggregateRule.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectAggregateRule.java
index 106de2a7f9c..d857d4eab0b 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectAggregateRule.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectAggregateRule.java
@@ -21,6 +21,7 @@ import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate;
+import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
import com.google.common.collect.ImmutableList;
@@ -35,9 +36,10 @@ public class MaterializedViewProjectAggregateRule extends
AbstractMaterializedVi
@Override
public List<Rule> buildRules() {
return ImmutableList.of(
-
logicalProject(logicalAggregate(any())).thenApplyMultiNoThrow(ctx -> {
- LogicalProject<LogicalAggregate<Plan>> root = ctx.root;
- return rewrite(root, ctx.cascadesContext);
- }).toRule(RuleType.MATERIALIZED_VIEW_PROJECT_AGGREGATE));
+
logicalProject(logicalAggregate(any().when(LogicalPlan.class::isInstance))).thenApplyMultiNoThrow(
+ ctx -> {
+ LogicalProject<LogicalAggregate<Plan>> root =
ctx.root;
+ return rewrite(root, ctx.cascadesContext);
+
}).toRule(RuleType.MATERIALIZED_VIEW_PROJECT_AGGREGATE));
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectFilterAggregateRule.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectFilterAggregateRule.java
index b841862d5b2..906ca31b4c2 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectFilterAggregateRule.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectFilterAggregateRule.java
@@ -22,6 +22,7 @@ import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate;
import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
+import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
import com.google.common.collect.ImmutableList;
@@ -38,8 +39,8 @@ public class MaterializedViewProjectFilterAggregateRule
extends AbstractMaterial
@Override
public List<Rule> buildRules() {
- return ImmutableList.of(
-
logicalProject(logicalFilter(logicalAggregate(any()))).thenApplyMultiNoThrow(ctx
-> {
+ return ImmutableList.of(logicalProject(logicalFilter(logicalAggregate(
+
any().when(LogicalPlan.class::isInstance)))).thenApplyMultiNoThrow(ctx -> {
LogicalProject<LogicalFilter<LogicalAggregate<Plan>>> root
= ctx.root;
return rewrite(root, ctx.cascadesContext);
}).toRule(RuleType.MATERIALIZED_VIEW_FILTER_AGGREGATE));
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectFilterJoinRule.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectFilterJoinRule.java
index e97a5378b83..d82f838ea6b 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectFilterJoinRule.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectFilterJoinRule.java
@@ -22,6 +22,7 @@ import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
import org.apache.doris.nereids.trees.plans.logical.LogicalJoin;
+import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
import com.google.common.collect.ImmutableList;
@@ -38,9 +39,10 @@ public class MaterializedViewProjectFilterJoinRule extends
AbstractMaterializedV
@Override
public List<Rule> buildRules() {
return ImmutableList.of(
- logicalProject(logicalFilter(logicalJoin(any(),
any()))).thenApplyMultiNoThrow(ctx -> {
- LogicalProject<LogicalFilter<LogicalJoin<Plan, Plan>>>
root = ctx.root;
- return rewrite(root, ctx.cascadesContext);
- }).toRule(RuleType.MATERIALIZED_VIEW_PROJECT_FILTER_JOIN));
+
logicalProject(logicalFilter(logicalJoin(any().when(LogicalPlan.class::isInstance),
+
any().when(LogicalPlan.class::isInstance)))).thenApplyMultiNoThrow(ctx -> {
+ LogicalProject<LogicalFilter<LogicalJoin<Plan,
Plan>>> root = ctx.root;
+ return rewrite(root, ctx.cascadesContext);
+
}).toRule(RuleType.MATERIALIZED_VIEW_PROJECT_FILTER_JOIN));
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectJoinRule.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectJoinRule.java
index c9e309c0096..bbeda87191a 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectJoinRule.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectJoinRule.java
@@ -21,6 +21,7 @@ import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.logical.LogicalJoin;
+import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
import com.google.common.collect.ImmutableList;
@@ -37,9 +38,10 @@ public class MaterializedViewProjectJoinRule extends
AbstractMaterializedViewJoi
@Override
public List<Rule> buildRules() {
return ImmutableList.of(
- logicalProject(logicalJoin(any(),
any())).thenApplyMultiNoThrow(ctx -> {
- LogicalProject<LogicalJoin<Plan, Plan>> root = ctx.root;
- return rewrite(root, ctx.cascadesContext);
- }).toRule(RuleType.MATERIALIZED_VIEW_PROJECT_JOIN));
+
logicalProject(logicalJoin(any().when(LogicalPlan.class::isInstance),
+
any().when(LogicalPlan.class::isInstance))).thenApplyMultiNoThrow(ctx -> {
+ LogicalProject<LogicalJoin<Plan, Plan>> root =
ctx.root;
+ return rewrite(root, ctx.cascadesContext);
+ }).toRule(RuleType.MATERIALIZED_VIEW_PROJECT_JOIN));
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java
index ce12e059b77..ac4b9ccad38 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java
@@ -18,12 +18,15 @@
package org.apache.doris.nereids.rules.exploration.mv;
import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.MTMV;
import org.apache.doris.catalog.PartitionType;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.common.Pair;
import org.apache.doris.mtmv.BaseTableInfo;
import org.apache.doris.mtmv.MTMVRelatedTableIf;
import org.apache.doris.nereids.CascadesContext;
+import org.apache.doris.nereids.memo.Group;
+import org.apache.doris.nereids.memo.StructInfoMap;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.Slot;
@@ -31,6 +34,7 @@ import
org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.expressions.WindowExpression;
import org.apache.doris.nereids.trees.plans.JoinType;
import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.PreAggStatus;
import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate;
import org.apache.doris.nereids.trees.plans.logical.LogicalCatalogRelation;
import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan;
@@ -44,9 +48,12 @@ import
org.apache.doris.nereids.trees.plans.logical.LogicalWindow;
import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanVisitor;
import com.google.common.collect.HashMultimap;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
import com.google.common.collect.Multimap;
import java.util.ArrayList;
+import java.util.BitSet;
import java.util.HashSet;
import java.util.List;
import java.util.Optional;
@@ -132,18 +139,46 @@ public class MaterializedViewUtils {
/**
* Extract struct info from plan, support to get struct info from logical
plan or plan in group.
*/
- public static List<StructInfo> extractStructInfo(Plan plan,
CascadesContext cascadesContext) {
- if (plan.getGroupExpression().isPresent() &&
!plan.getGroupExpression().get().getOwnerGroup().getStructInfos()
- .isEmpty()) {
- return
plan.getGroupExpression().get().getOwnerGroup().getStructInfos();
- } else {
- // build struct info and add them to current group
- List<StructInfo> structInfos = StructInfo.of(plan);
- if (plan.getGroupExpression().isPresent()) {
-
plan.getGroupExpression().get().getOwnerGroup().addStructInfo(structInfos);
+ public static List<StructInfo> extractStructInfo(Plan plan,
CascadesContext cascadesContext,
+ BitSet materializedViewTableSet) {
+ // If plan belong to some group, construct it with group struct info
+ if (plan.getGroupExpression().isPresent()) {
+ Group ownerGroup = plan.getGroupExpression().get().getOwnerGroup();
+ StructInfoMap structInfoMap = ownerGroup.getstructInfoMap();
+ structInfoMap.refresh(ownerGroup);
+ Set<BitSet> queryTableSets = structInfoMap.getTableMaps();
+ if (!queryTableSets.isEmpty()) {
+ return queryTableSets.stream()
+ // Just construct the struct info which mv table set
contains all the query table set
+ .filter(queryTableSet ->
materializedViewTableSet.isEmpty()
+ ||
StructInfo.containsAll(materializedViewTableSet, queryTableSet))
+ .map(tableMap -> structInfoMap.getStructInfo(tableMap,
tableMap, ownerGroup, plan))
+ .collect(Collectors.toList());
}
- return structInfos;
}
+ // if plan doesn't belong to any group, construct it directly
+ return ImmutableList.of(StructInfo.of(plan));
+ }
+
+ /**
+ * Generate scan plan for materialized view
+ * if MaterializationContext is already rewritten by materialized view,
then should generate in real time
+ * when query rewrite, because one plan may hit the materialized view
repeatedly and the mv scan output
+ * should be different
+ */
+ public static Plan generateMvScanPlan(MTMV materializedView,
CascadesContext cascadesContext) {
+ LogicalOlapScan mvScan = new LogicalOlapScan(
+ cascadesContext.getStatementContext().getNextRelationId(),
+ materializedView,
+ ImmutableList.of(materializedView.getQualifiedDbName()),
+ // this must be empty, or it will be used to sample
+ Lists.newArrayList(),
+ Lists.newArrayList(),
+ Optional.empty());
+ mvScan = mvScan.withMaterializedIndexSelected(PreAggStatus.on(),
materializedView.getBaseIndexId());
+ List<NamedExpression> mvProjects =
mvScan.getOutput().stream().map(NamedExpression.class::cast)
+ .collect(Collectors.toList());
+ return new LogicalProject<Plan>(mvProjects, mvScan);
}
private static final class TableQueryOperatorChecker extends
DefaultPlanVisitor<Boolean, Void> {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java
index 0d280bb8340..604e7853d48 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java
@@ -47,12 +47,12 @@ import
org.apache.doris.nereids.trees.plans.visitor.DefaultPlanVisitor;
import org.apache.doris.nereids.trees.plans.visitor.ExpressionLineageReplacer;
import org.apache.doris.nereids.util.ExpressionUtils;
-import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import java.util.ArrayList;
+import java.util.BitSet;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
@@ -83,6 +83,7 @@ public class StructInfo {
// bottom plan which top plan only contain join or scan. this is needed by
hyper graph
private final Plan bottomPlan;
private final List<CatalogRelation> relations;
+ private final BitSet tableBitSet = new BitSet();
// this is for LogicalCompatibilityContext later
private final Map<RelationId, StructInfoNode> relationIdStructInfoNodeMap;
// this recorde the predicates which can pull up, not shuttled
@@ -112,6 +113,7 @@ public class StructInfo {
this.topPlan = topPlan;
this.bottomPlan = bottomPlan;
this.relations = relations;
+ relations.forEach(relation -> this.tableBitSet.set((int)
(relation.getTable().getId())));
this.relationIdStructInfoNodeMap = relationIdStructInfoNodeMap;
this.predicates = predicates;
if (predicates == null) {
@@ -120,7 +122,8 @@ public class StructInfo {
topPlan.accept(PREDICATE_COLLECTOR, topPlanPredicates);
this.predicates = Predicates.of(topPlanPredicates);
}
- Pair<SplitPredicate, EquivalenceClass> derivedPredicates =
predicatesDerive(this.predicates, originalPlan);
+ Pair<SplitPredicate, EquivalenceClass> derivedPredicates =
+ predicatesDerive(this.predicates, topPlan, tableBitSet);
this.splitPredicate = derivedPredicates.key();
this.equivalenceClass = derivedPredicates.value();
this.shuttledHashConjunctsToConjunctsMap =
shuttledHashConjunctsToConjunctsMap;
@@ -140,8 +143,24 @@ public class StructInfo {
Plan topPlan,
Map<Expression, Expression> shuttledHashConjunctsToConjunctsMap,
Map<ExprId, Expression> namedExprIdAndExprMapping,
- ImmutableList.Builder<CatalogRelation> relationBuilder,
+ List<CatalogRelation> relations,
Map<RelationId, StructInfoNode> relationIdStructInfoNodeMap) {
+
+ // Collect relations from hyper graph which in the bottom plan firstly
+ BitSet hyperTableBitSet = new BitSet();
+ hyperGraph.getNodes().forEach(node -> {
+ // plan relation collector and set to map
+ Plan nodePlan = node.getPlan();
+ List<CatalogRelation> nodeRelations = new ArrayList<>();
+ nodePlan.accept(RELATION_COLLECTOR, nodeRelations);
+ relations.addAll(nodeRelations);
+ nodeRelations.forEach(relation -> hyperTableBitSet.set((int)
relation.getTable().getId()));
+ // every node should only have one relation, this is for
LogicalCompatibilityContext
+ if (!nodeRelations.isEmpty()) {
+
relationIdStructInfoNodeMap.put(nodeRelations.get(0).getRelationId(),
(StructInfoNode) node);
+ }
+ });
+
// Collect expression from join condition in hyper graph
for (JoinEdge edge : hyperGraph.getJoinEdges()) {
List<Expression> hashJoinConjuncts = edge.getHashJoinConjuncts();
@@ -151,9 +170,8 @@ public class StructInfo {
hashJoinConjuncts.forEach(conjunctExpr -> {
ExpressionLineageReplacer.ExpressionReplaceContext
replaceContext =
new ExpressionLineageReplacer.ExpressionReplaceContext(
- Lists.newArrayList(conjunctExpr),
- ImmutableSet.of(),
- ImmutableSet.of());
+ Lists.newArrayList(conjunctExpr),
ImmutableSet.of(),
+ ImmutableSet.of(), hyperTableBitSet);
topPlan.accept(ExpressionLineageReplacer.INSTANCE,
replaceContext);
// Replace expressions by expression map
List<Expression> replacedExpressions =
replaceContext.getReplacedExpressions();
@@ -167,27 +185,17 @@ public class StructInfo {
return false;
}
}
- // Collect relations from hyper graph which in the bottom plan
+ // Record expressions in node
hyperGraph.getNodes().forEach(node -> {
// plan relation collector and set to map
StructInfoNode structInfoNode = (StructInfoNode) node;
- // plan relation collector and set to map
- Plan nodePlan = node.getPlan();
- List<CatalogRelation> nodeRelations = new ArrayList<>();
- nodePlan.accept(RELATION_COLLECTOR, nodeRelations);
- relationBuilder.addAll(nodeRelations);
- // every node should only have one relation, this is for
LogicalCompatibilityContext
- if (!nodeRelations.isEmpty()) {
-
relationIdStructInfoNodeMap.put(nodeRelations.get(0).getRelationId(),
(StructInfoNode) node);
- }
// record expressions in node
if (structInfoNode.getExpressions() != null) {
structInfoNode.getExpressions().forEach(expression -> {
ExpressionLineageReplacer.ExpressionReplaceContext
replaceContext =
new
ExpressionLineageReplacer.ExpressionReplaceContext(
- Lists.newArrayList(expression),
- ImmutableSet.of(),
- ImmutableSet.of());
+ Lists.newArrayList(expression),
ImmutableSet.of(),
+ ImmutableSet.of(), hyperTableBitSet);
structInfoNode.getPlan().accept(ExpressionLineageReplacer.INSTANCE,
replaceContext);
// Replace expressions by expression map
List<Expression> replacedExpressions =
replaceContext.getReplacedExpressions();
@@ -204,18 +212,19 @@ public class StructInfo {
filterExpressions.forEach(predicate -> {
// this is used for LogicalCompatibilityContext
ExpressionUtils.extractConjunction(predicate).forEach(expr ->
- shuttledHashConjunctsToConjunctsMap.put(
-
ExpressionUtils.shuttleExpressionWithLineage(predicate, topPlan), predicate));
+
shuttledHashConjunctsToConjunctsMap.put(ExpressionUtils.shuttleExpressionWithLineage(
+ predicate, topPlan, hyperTableBitSet),
predicate));
});
});
return true;
}
// derive some useful predicate by predicates
- private Pair<SplitPredicate, EquivalenceClass> predicatesDerive(Predicates
predicates, Plan originalPlan) {
+ private Pair<SplitPredicate, EquivalenceClass> predicatesDerive(Predicates
predicates, Plan originalPlan,
+ BitSet tableBitSet) {
// construct equivalenceClass according to equals predicates
List<Expression> shuttledExpression =
ExpressionUtils.shuttleExpressionWithLineage(
- new ArrayList<>(predicates.getPulledUpPredicates()),
originalPlan).stream()
+ new ArrayList<>(predicates.getPulledUpPredicates()),
originalPlan, tableBitSet).stream()
.map(Expression.class::cast)
.collect(Collectors.toList());
SplitPredicate splitPredicate =
Predicates.splitPredicates(ExpressionUtils.and(shuttledExpression));
@@ -238,20 +247,23 @@ public class StructInfo {
* Build Struct info from plan.
* Maybe return multi structInfo when original plan already be rewritten
by mv
*/
- public static List<StructInfo> of(Plan originalPlan) {
- // TODO only consider the inner join currently, Should support outer
join
+ public static StructInfo of(Plan originalPlan) {
+ return of(originalPlan, originalPlan);
+ }
+
+ /**
+ * Build Struct info from plan.
+ * Maybe return multi structInfo when original plan already be rewritten
by mv
+ */
+ public static StructInfo of(Plan derivedPlan, Plan originalPlan) {
// Split plan by the boundary which contains multi child
LinkedHashSet<Class<? extends Plan>> set = Sets.newLinkedHashSet();
set.add(LogicalJoin.class);
PlanSplitContext planSplitContext = new PlanSplitContext(set);
// if single table without join, the bottom is
- originalPlan.accept(PLAN_SPLITTER, planSplitContext);
-
- List<HyperGraph> structInfos =
HyperGraph.builderForMv(planSplitContext.getBottomPlan()).buildAll();
- return structInfos.stream()
- .map(hyperGraph -> StructInfo.of(originalPlan,
planSplitContext.getTopPlan(),
- planSplitContext.getBottomPlan(), hyperGraph))
- .collect(Collectors.toList());
+ derivedPlan.accept(PLAN_SPLITTER, planSplitContext);
+ return StructInfo.of(originalPlan, planSplitContext.getTopPlan(),
planSplitContext.getBottomPlan(),
+
HyperGraph.builderForMv(planSplitContext.getBottomPlan()).build());
}
/**
@@ -271,16 +283,16 @@ public class StructInfo {
topPlan = planSplitContext.getTopPlan();
}
// collect struct info fromGraph
- ImmutableList.Builder<CatalogRelation> relationBuilder =
ImmutableList.builder();
+ List<CatalogRelation> relationList = new ArrayList<>();
Map<RelationId, StructInfoNode> relationIdStructInfoNodeMap = new
LinkedHashMap<>();
Map<Expression, Expression> shuttledHashConjunctsToConjunctsMap = new
LinkedHashMap<>();
Map<ExprId, Expression> namedExprIdAndExprMapping = new
LinkedHashMap<>();
boolean valid = collectStructInfoFromGraph(hyperGraph, topPlan,
shuttledHashConjunctsToConjunctsMap,
namedExprIdAndExprMapping,
- relationBuilder,
+ relationList,
relationIdStructInfoNodeMap);
return new StructInfo(originalPlan, originalPlanId, hyperGraph, valid,
topPlan, bottomPlan,
- relationBuilder.build(), relationIdStructInfoNodeMap, null,
shuttledHashConjunctsToConjunctsMap,
+ relationList, relationIdStructInfoNodeMap, null,
shuttledHashConjunctsToConjunctsMap,
namedExprIdAndExprMapping);
}
@@ -338,8 +350,8 @@ public class StructInfo {
}
public List<? extends Expression> getExpressions() {
- return originalPlan instanceof LogicalProject
- ? ((LogicalProject<Plan>) originalPlan).getProjects() :
originalPlan.getOutput();
+ return topPlan instanceof LogicalProject
+ ? ((LogicalProject<Plan>) topPlan).getProjects() :
topPlan.getOutput();
}
public ObjectId getOriginalPlanId() {
@@ -350,6 +362,10 @@ public class StructInfo {
return namedExprIdAndExprMapping;
}
+ public BitSet getTableBitSet() {
+ return tableBitSet;
+ }
+
/**
* Judge the source graph logical is whether the same as target
* For inner join should judge only the join tables,
@@ -361,6 +377,11 @@ public class StructInfo {
.isLogicCompatible(queryStructInfo.hyperGraph,
viewStructInfo.hyperGraph, compatibilityContext);
}
+ @Override
+ public String toString() {
+ return "StructInfo{ originalPlanId = " + originalPlanId + ", relations
= " + relations + '}';
+ }
+
private static class RelationCollector extends DefaultPlanVisitor<Void,
List<CatalogRelation>> {
@Override
public Void visit(Plan plan, List<CatalogRelation> collectedRelations)
{
@@ -409,6 +430,12 @@ public class StructInfo {
}
}
+ public static boolean containsAll(BitSet source, BitSet target) {
+ BitSet intersection = (BitSet) source.clone();
+ intersection.and(target);
+ return intersection.equals(target);
+ }
+
/**
* Plan split context, this hold bottom and top plan, and boundary plan
setting
*/
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/ExpressionLineageReplacer.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/ExpressionLineageReplacer.java
index b7cc0dff80b..2060718ec13 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/ExpressionLineageReplacer.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/ExpressionLineageReplacer.java
@@ -32,9 +32,11 @@ import org.apache.doris.nereids.trees.plans.Plan;
import
org.apache.doris.nereids.trees.plans.visitor.ExpressionLineageReplacer.ExpressionReplaceContext;
import java.util.ArrayList;
+import java.util.BitSet;
import java.util.Collection;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
@@ -43,12 +45,12 @@ import java.util.stream.Collectors;
* Get from rewrite plan and can also get from plan struct info, if from plan
struct info it depends on
* the nodes from graph.
*/
-public class ExpressionLineageReplacer extends DefaultPlanVisitor<Expression,
ExpressionReplaceContext> {
+public class ExpressionLineageReplacer extends DefaultPlanVisitor<Void,
ExpressionReplaceContext> {
public static final ExpressionLineageReplacer INSTANCE = new
ExpressionLineageReplacer();
@Override
- public Expression visit(Plan plan, ExpressionReplaceContext context) {
+ public Void visit(Plan plan, ExpressionReplaceContext context) {
List<? extends Expression> expressions = plan.getExpressions();
Map<ExprId, Expression> targetExpressionMap =
context.getExprIdExpressionMap();
// Filter the namedExpression used by target and collect the
namedExpression
@@ -60,19 +62,27 @@ public class ExpressionLineageReplacer extends
DefaultPlanVisitor<Expression, Ex
}
@Override
- public Expression visitGroupPlan(GroupPlan groupPlan,
ExpressionReplaceContext context) {
+ public Void visitGroupPlan(GroupPlan groupPlan, ExpressionReplaceContext
context) {
Group group = groupPlan.getGroup();
if (group == null) {
return visit(groupPlan, context);
}
- List<StructInfo> structInfos = group.getStructInfos();
+ Collection<StructInfo> structInfos =
group.getstructInfoMap().getStructInfos();
if (structInfos.isEmpty()) {
return visit(groupPlan, context);
}
- // TODO only support group has one struct info, will support more
struct info later
- StructInfo structInfo = structInfos.get(0);
-
context.getExprIdExpressionMap().putAll(structInfo.getNamedExprIdAndExprMapping());
- return visit(groupPlan, context);
+ // Find first info which the context's bitmap contains all to make
sure that
+ // the expression lineage is correct
+ Optional<StructInfo> structInfoOptional = structInfos.stream()
+ .filter(info -> (context.getTableBitSet().isEmpty()
+ || StructInfo.containsAll(context.getTableBitSet(),
info.getTableBitSet()))
+ && !info.getNamedExprIdAndExprMapping().isEmpty())
+ .findFirst();
+ if (!structInfoOptional.isPresent()) {
+ return visit(groupPlan, context);
+ }
+
context.getExprIdExpressionMap().putAll(structInfoOptional.get().getNamedExprIdAndExprMapping());
+ return null;
}
/**
@@ -144,7 +154,8 @@ public class ExpressionLineageReplacer extends
DefaultPlanVisitor<Expression, Ex
private final List<Expression> targetExpressions;
private final Set<TableType> targetTypes;
private final Set<String> tableIdentifiers;
- private Map<ExprId, Expression> exprIdExpressionMap;
+ private final Map<ExprId, Expression> exprIdExpressionMap;
+ private final BitSet tableBitSet;
private List<Expression> replacedExpressions;
/**
@@ -152,10 +163,12 @@ public class ExpressionLineageReplacer extends
DefaultPlanVisitor<Expression, Ex
*/
public ExpressionReplaceContext(List<Expression> targetExpressions,
Set<TableType> targetTypes,
- Set<String> tableIdentifiers) {
+ Set<String> tableIdentifiers,
+ BitSet tableBitSet) {
this.targetExpressions = targetExpressions;
this.targetTypes = targetTypes;
this.tableIdentifiers = tableIdentifiers;
+ this.tableBitSet = tableBitSet;
// collect the named expressions used in target expression and
will be replaced later
this.exprIdExpressionMap = targetExpressions.stream()
.map(each ->
each.collectToList(NamedExpression.class::isInstance))
@@ -181,6 +194,10 @@ public class ExpressionLineageReplacer extends
DefaultPlanVisitor<Expression, Ex
return exprIdExpressionMap;
}
+ public BitSet getTableBitSet() {
+ return tableBitSet;
+ }
+
/**
* getReplacedExpressions
*/
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java
index e5e79f7b725..51eef31de2f 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java
@@ -65,6 +65,7 @@ import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import java.util.Arrays;
+import java.util.BitSet;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
@@ -225,14 +226,14 @@ public class ExpressionUtils {
return result;
}
- public static Expression shuttleExpressionWithLineage(Expression
expression, Plan plan) {
+ public static Expression shuttleExpressionWithLineage(Expression
expression, Plan plan, BitSet tableBitSet) {
return shuttleExpressionWithLineage(Lists.newArrayList(expression),
- plan, ImmutableSet.of(), ImmutableSet.of()).get(0);
+ plan, ImmutableSet.of(), ImmutableSet.of(),
tableBitSet).get(0);
}
public static List<? extends Expression>
shuttleExpressionWithLineage(List<? extends Expression> expressions,
- Plan plan) {
- return shuttleExpressionWithLineage(expressions, plan,
ImmutableSet.of(), ImmutableSet.of());
+ Plan plan, BitSet tableBitSet) {
+ return shuttleExpressionWithLineage(expressions, plan,
ImmutableSet.of(), ImmutableSet.of(), tableBitSet);
}
/**
@@ -247,7 +248,8 @@ public class ExpressionUtils {
public static List<? extends Expression>
shuttleExpressionWithLineage(List<? extends Expression> expressions,
Plan plan,
Set<TableType> targetTypes,
- Set<String> tableIdentifiers) {
+ Set<String> tableIdentifiers,
+ BitSet tableBitSet) {
if (expressions.isEmpty()) {
return ImmutableList.of();
}
@@ -255,7 +257,8 @@ public class ExpressionUtils {
new ExpressionLineageReplacer.ExpressionReplaceContext(
expressions.stream().map(Expression.class::cast).collect(Collectors.toList()),
targetTypes,
- tableIdentifiers);
+ tableIdentifiers,
+ tableBitSet);
plan.accept(ExpressionLineageReplacer.INSTANCE, replaceContext);
// Replace expressions by expression map
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/CompareOuterJoinTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/CompareOuterJoinTest.java
index afdce7ca08d..9ebb04e6ba3 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/CompareOuterJoinTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/CompareOuterJoinTest.java
@@ -37,6 +37,7 @@ import com.google.common.collect.Sets;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
+import java.util.BitSet;
import java.util.List;
class CompareOuterJoinTest extends SqlTestBase {
@@ -63,8 +64,8 @@ class CompareOuterJoinTest extends SqlTestBase {
.rewrite()
.applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER)
.getAllPlan().get(0).child(0);
- HyperGraph h1 = HyperGraph.builderForMv(p1).buildAll().get(0);
- HyperGraph h2 = HyperGraph.builderForMv(p2).buildAll().get(0);
+ HyperGraph h1 = HyperGraph.builderForMv(p1).build();
+ HyperGraph h2 = HyperGraph.builderForMv(p2).build();
Assertions.assertFalse(
HyperGraphComparator.isLogicCompatible(h1, h2,
constructContext(p1, p2)).isInvalid());
}
@@ -82,8 +83,8 @@ class CompareOuterJoinTest extends SqlTestBase {
.rewrite()
.applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER)
.getAllPlan().get(0);
- HyperGraph h1 = HyperGraph.builderForMv(p1).buildAll().get(0);
- HyperGraph h2 = HyperGraph.builderForMv(p2).buildAll().get(0);
+ HyperGraph h1 = HyperGraph.builderForMv(p1).build();
+ HyperGraph h2 = HyperGraph.builderForMv(p2).build();
Assertions.assertFalse(
HyperGraphComparator.isLogicCompatible(h1, h2,
constructContext(p1, p2)).isInvalid());
}
@@ -108,8 +109,8 @@ class CompareOuterJoinTest extends SqlTestBase {
.rewrite()
.applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER)
.getAllPlan().get(0).child(0);
- HyperGraph h1 = HyperGraph.builderForMv(p1).buildAll().get(0);
- HyperGraph h2 = HyperGraph.builderForMv(p2).buildAll().get(0);
+ HyperGraph h1 = HyperGraph.builderForMv(p1).build();
+ HyperGraph h2 = HyperGraph.builderForMv(p2).build();
ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2,
constructContext(p1, p2));
Assertions.assertEquals(1, res.getQueryExpressions().size());
Assertions.assertEquals("(id = 0)",
res.getQueryExpressions().get(0).toSql());
@@ -135,8 +136,8 @@ class CompareOuterJoinTest extends SqlTestBase {
.rewrite()
.applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER)
.getAllPlan().get(0).child(0);
- HyperGraph h1 = HyperGraph.builderForMv(p1).buildAll().get(0);
- HyperGraph h2 = HyperGraph.builderForMv(p2).buildAll().get(0);
+ HyperGraph h1 = HyperGraph.builderForMv(p1).build();
+ HyperGraph h2 = HyperGraph.builderForMv(p2).build();
List<Expression> exprList = HyperGraphComparator.isLogicCompatible(h1,
h2, constructContext(p1, p2)).getQueryExpressions();
Assertions.assertEquals(0, exprList.size());
}
@@ -162,8 +163,8 @@ class CompareOuterJoinTest extends SqlTestBase {
.rewrite()
.applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER)
.getAllPlan().get(0).child(0);
- HyperGraph h1 = HyperGraph.builderForMv(p1).buildAll().get(0);
- HyperGraph h2 = HyperGraph.builderForMv(p2).buildAll().get(0);
+ HyperGraph h1 = HyperGraph.builderForMv(p1).build();
+ HyperGraph h2 = HyperGraph.builderForMv(p2).build();
ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2,
constructContext(p1, p2));
Assertions.assertEquals(1, res.getQueryExpressions().size());
Assertions.assertEquals("(id = 0)",
res.getQueryExpressions().get(0).toSql());
@@ -190,17 +191,17 @@ class CompareOuterJoinTest extends SqlTestBase {
.rewrite()
.applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER)
.getAllPlan().get(0).child(0);
- HyperGraph h1 = HyperGraph.builderForMv(p1).buildAll().get(0);
- HyperGraph h2 = HyperGraph.builderForMv(p2).buildAll().get(0);
+ HyperGraph h1 = HyperGraph.builderForMv(p1).build();
+ HyperGraph h2 = HyperGraph.builderForMv(p2).build();
ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2,
constructContext(p1, p2));
Assertions.assertTrue(res.isInvalid());
}
LogicalCompatibilityContext constructContext(Plan p1, Plan p2) {
StructInfo st1 = MaterializedViewUtils.extractStructInfo(p1,
- null).get(0);
+ null, new BitSet()).get(0);
StructInfo st2 = MaterializedViewUtils.extractStructInfo(p2,
- null).get(0);
+ null, new BitSet()).get(0);
RelationMapping rm = RelationMapping.generate(st1.getRelations(),
st2.getRelations()).get(0);
SlotMapping sm = SlotMapping.generate(rm);
return LogicalCompatibilityContext.from(rm, sm, st1, st2);
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/InferJoinTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/InferJoinTest.java
index 05f56c4de20..d61224c1433 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/InferJoinTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/InferJoinTest.java
@@ -35,6 +35,7 @@ import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
+import java.util.BitSet;
import java.util.stream.Collectors;
class InferJoinTest extends SqlTestBase {
@@ -58,8 +59,8 @@ class InferJoinTest extends SqlTestBase {
.rewrite()
.applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER)
.getAllPlan().get(0).child(0);
- HyperGraph h1 = HyperGraph.builderForMv(p1).buildAll().get(0);
- HyperGraph h2 = HyperGraph.builderForMv(p2).buildAll().get(0);
+ HyperGraph h1 = HyperGraph.builderForMv(p1).build();
+ HyperGraph h2 = HyperGraph.builderForMv(p2).build();
ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2,
constructContext(p1, p2));
Assertions.assertFalse(res.isInvalid());
Assertions.assertEquals(1, res.getViewNoNullableSlot().size());
@@ -87,8 +88,8 @@ class InferJoinTest extends SqlTestBase {
.rewrite()
.applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER)
.getAllPlan().get(0).child(0);
- HyperGraph h1 = HyperGraph.builderForMv(p1).buildAll().get(0);
- HyperGraph h2 = HyperGraph.builderForMv(p2).buildAll().get(0);
+ HyperGraph h1 = HyperGraph.builderForMv(p1).build();
+ HyperGraph h2 = HyperGraph.builderForMv(p2).build();
ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2,
constructContext(p1, p2));
Assertions.assertFalse(res.isInvalid());
Assertions.assertEquals(1, res.getViewNoNullableSlot().size());
@@ -124,8 +125,8 @@ class InferJoinTest extends SqlTestBase {
.rewrite()
.applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER)
.getAllPlan().get(0).child(0);
- HyperGraph h1 = HyperGraph.builderForMv(p1).buildAll().get(0);
- HyperGraph h2 = HyperGraph.builderForMv(p2).buildAll().get(0);
+ HyperGraph h1 = HyperGraph.builderForMv(p1).build();
+ HyperGraph h2 = HyperGraph.builderForMv(p2).build();
ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2,
constructContext(p1, p2));
Assertions.assertFalse(res.isInvalid());
Assertions.assertEquals(1, res.getViewNoNullableSlot().size());
@@ -155,17 +156,17 @@ class InferJoinTest extends SqlTestBase {
.rewrite()
.applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER)
.getAllPlan().get(0).child(0);
- HyperGraph h1 = HyperGraph.builderForMv(p1).buildAll().get(0);
- HyperGraph h2 = HyperGraph.builderForMv(p2).buildAll().get(0);
+ HyperGraph h1 = HyperGraph.builderForMv(p1).build();
+ HyperGraph h2 = HyperGraph.builderForMv(p2).build();
ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2,
constructContext(p1, p2));
Assertions.assertTrue(res.isInvalid());
}
LogicalCompatibilityContext constructContext(Plan p1, Plan p2) {
StructInfo st1 = MaterializedViewUtils.extractStructInfo(p1,
- null).get(0);
+ null, new BitSet()).get(0);
StructInfo st2 = MaterializedViewUtils.extractStructInfo(p2,
- null).get(0);
+ null, new BitSet()).get(0);
RelationMapping rm = RelationMapping.generate(st1.getRelations(),
st2.getRelations()).get(0);
SlotMapping sm = SlotMapping.generate(rm);
return LogicalCompatibilityContext.from(rm, sm, st1, st2);
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/InferPredicateTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/InferPredicateTest.java
index 8bb1ede8048..fe705fda72c 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/InferPredicateTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/InferPredicateTest.java
@@ -33,6 +33,8 @@ import org.apache.doris.nereids.util.PlanChecker;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
+import java.util.BitSet;
+
class InferPredicateTest extends SqlTestBase {
@Test
void testPullUpQueryFilter() {
@@ -53,8 +55,8 @@ class InferPredicateTest extends SqlTestBase {
.rewrite()
.applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER)
.getAllPlan().get(0).child(0);
- HyperGraph h1 = HyperGraph.builderForMv(p1).buildAll().get(0);
- HyperGraph h2 = HyperGraph.builderForMv(p2).buildAll().get(0);
+ HyperGraph h1 = HyperGraph.builderForMv(p1).build();
+ HyperGraph h2 = HyperGraph.builderForMv(p2).build();
ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2,
constructContext(p1, p2));
Assertions.assertFalse(res.isInvalid());
Assertions.assertEquals("(id = 1)",
res.getQueryExpressions().get(0).toSql());
@@ -62,9 +64,9 @@ class InferPredicateTest extends SqlTestBase {
LogicalCompatibilityContext constructContext(Plan p1, Plan p2) {
StructInfo st1 = MaterializedViewUtils.extractStructInfo(p1,
- null).get(0);
+ null, new BitSet()).get(0);
StructInfo st2 = MaterializedViewUtils.extractStructInfo(p2,
- null).get(0);
+ null, new BitSet()).get(0);
RelationMapping rm = RelationMapping.generate(st1.getRelations(),
st2.getRelations()).get(0);
SlotMapping sm = SlotMapping.generate(rm);
return LogicalCompatibilityContext.from(rm, sm, st1, st2);
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/PullupExpressionTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/PullupExpressionTest.java
index 6e65dba4f03..b564f166ed5 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/PullupExpressionTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/PullupExpressionTest.java
@@ -33,6 +33,8 @@ import org.apache.doris.nereids.util.PlanChecker;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
+import java.util.BitSet;
+
class PullupExpressionTest extends SqlTestBase {
@Test
void testPullUpQueryFilter() {
@@ -53,8 +55,8 @@ class PullupExpressionTest extends SqlTestBase {
.rewrite()
.applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER)
.getAllPlan().get(0).child(0);
- HyperGraph h1 = HyperGraph.builderForMv(p1).buildAll().get(0);
- HyperGraph h2 = HyperGraph.builderForMv(p2).buildAll().get(0);
+ HyperGraph h1 = HyperGraph.builderForMv(p1).build();
+ HyperGraph h2 = HyperGraph.builderForMv(p2).build();
ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2,
constructContext(p1, p2));
Assertions.assertEquals(1, res.getQueryExpressions().size());
Assertions.assertEquals("(id = 1)",
res.getQueryExpressions().get(0).toSql());
@@ -79,8 +81,8 @@ class PullupExpressionTest extends SqlTestBase {
.rewrite()
.applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER)
.getAllPlan().get(0).child(0);
- HyperGraph h1 = HyperGraph.builderForMv(p1).buildAll().get(0);
- HyperGraph h2 = HyperGraph.builderForMv(p2).buildAll().get(0);
+ HyperGraph h1 = HyperGraph.builderForMv(p1).build();
+ HyperGraph h2 = HyperGraph.builderForMv(p2).build();
ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2,
constructContext(p1, p2));
Assertions.assertEquals(1, res.getQueryExpressions().size());
Assertions.assertEquals("(score = score)",
res.getQueryExpressions().get(0).toSql());
@@ -105,8 +107,8 @@ class PullupExpressionTest extends SqlTestBase {
.rewrite()
.applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER)
.getAllPlan().get(0).child(0);
- HyperGraph h1 = HyperGraph.builderForMv(p1).buildAll().get(0);
- HyperGraph h2 = HyperGraph.builderForMv(p2).buildAll().get(0);
+ HyperGraph h1 = HyperGraph.builderForMv(p1).build();
+ HyperGraph h2 = HyperGraph.builderForMv(p2).build();
ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2,
constructContext(p1, p2));
Assertions.assertEquals(2, res.getViewExpressions().size());
Assertions.assertEquals("(id = 1)",
res.getViewExpressions().get(0).toSql());
@@ -132,8 +134,8 @@ class PullupExpressionTest extends SqlTestBase {
.rewrite()
.applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER)
.getAllPlan().get(0).child(0);
- HyperGraph h1 = HyperGraph.builderForMv(p1).buildAll().get(0);
- HyperGraph h2 = HyperGraph.builderForMv(p2).buildAll().get(0);
+ HyperGraph h1 = HyperGraph.builderForMv(p1).build();
+ HyperGraph h2 = HyperGraph.builderForMv(p2).build();
ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2,
constructContext(p1, p2));
Assertions.assertEquals(1, res.getViewExpressions().size());
Assertions.assertEquals("(score = score)",
res.getViewExpressions().get(0).toSql());
@@ -141,9 +143,9 @@ class PullupExpressionTest extends SqlTestBase {
LogicalCompatibilityContext constructContext(Plan p1, Plan p2) {
StructInfo st1 = MaterializedViewUtils.extractStructInfo(p1,
- null).get(0);
+ null, new BitSet()).get(0);
StructInfo st2 = MaterializedViewUtils.extractStructInfo(p2,
- null).get(0);
+ null, new BitSet()).get(0);
RelationMapping rm = RelationMapping.generate(st1.getRelations(),
st2.getRelations()).get(0);
SlotMapping sm = SlotMapping.generate(rm);
return LogicalCompatibilityContext.from(rm, sm, st1, st2);
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java
index c2d9d072ac3..13bdf35252e 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java
@@ -172,7 +172,7 @@ class StructInfoMapTest extends SqlTestBase {
BitSet mvMap = structInfoMap.getTableMaps().stream()
.filter(b -> b.cardinality() == 2)
.collect(Collectors.toList()).get(0);
- StructInfo structInfo = structInfoMap.getStructInfo(mvMap, mvMap,
root);
+ StructInfo structInfo = structInfoMap.getStructInfo(mvMap, mvMap,
root, null);
System.out.println(structInfo.getOriginalPlan().treeString());
BitSet bitSet = new BitSet();
structInfo.getRelations().forEach(r -> bitSet.set((int)
r.getTable().getId()));
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/BuildStructInfoTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/BuildStructInfoTest.java
index 20324ff8739..a3fc234f983 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/BuildStructInfoTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/BuildStructInfoTest.java
@@ -25,8 +25,6 @@ import org.apache.doris.nereids.util.PlanChecker;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
-import java.util.List;
-
class BuildStructInfoTest extends SqlTestBase {
@Test
void testSimpleSQL() {
@@ -58,8 +56,8 @@ class BuildStructInfoTest extends SqlTestBase {
.deriveStats()
.matches(logicalJoin()
.when(j -> {
- List<HyperGraph> hyperGraph =
HyperGraph.builderForMv(j).buildAll();
-
Assertions.assertTrue(hyperGraph.get(0).getNodes().stream()
+ HyperGraph hyperGraph =
HyperGraph.builderForMv(j).build();
+
Assertions.assertTrue(hyperGraph.getNodes().stream()
.allMatch(n -> n.getPlan()
.collectToList(GroupPlan.class::isInstance).isEmpty()));
return true;
@@ -77,7 +75,7 @@ class BuildStructInfoTest extends SqlTestBase {
.rewrite()
.matches(logicalJoin()
.when(j -> {
- HyperGraph structInfo =
HyperGraph.builderForMv(j).buildAll().get(0);
+ HyperGraph structInfo =
HyperGraph.builderForMv(j).build();
Assertions.assertTrue(structInfo.getJoinEdge(0).getJoinType().isLeftOuterJoin());
Assertions.assertEquals(0,
structInfo.getFilterEdge(0).getLeftRejectEdge().size());
Assertions.assertEquals(1,
structInfo.getFilterEdge(0).getRightRejectEdge().size());
@@ -91,7 +89,7 @@ class BuildStructInfoTest extends SqlTestBase {
.rewrite()
.matches(logicalJoin()
.when(j -> {
- HyperGraph structInfo =
HyperGraph.builderForMv(j).buildAll().get(0);
+ HyperGraph structInfo =
HyperGraph.builderForMv(j).build();
Assertions.assertTrue(structInfo.getJoinEdge(0).getJoinType().isLeftOuterJoin());
return true;
}));
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/EliminateJoinTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/EliminateJoinTest.java
index cc8b7147423..08faf589390 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/EliminateJoinTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/EliminateJoinTest.java
@@ -30,6 +30,8 @@ import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
+import java.util.BitSet;
+
class EliminateJoinTest extends SqlTestBase {
@Test
void testLOJWithGroupBy() {
@@ -62,9 +64,9 @@ class EliminateJoinTest extends SqlTestBase {
.rewrite()
.applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER)
.getAllPlan().get(0).child(0);
- HyperGraph h1 = HyperGraph.builderForMv(p1).buildAll().get(0);
- HyperGraph h2 = HyperGraph.builderForMv(p2).buildAll().get(0);
- HyperGraph h3 = HyperGraph.builderForMv(p3).buildAll().get(0);
+ HyperGraph h1 = HyperGraph.builderForMv(p1).build();
+ HyperGraph h2 = HyperGraph.builderForMv(p2).build();
+ HyperGraph h3 = HyperGraph.builderForMv(p3).build();
ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2,
constructContext(p1, p2));
Assertions.assertTrue(!res.isInvalid());
Assertions.assertTrue(!HyperGraphComparator.isLogicCompatible(h1, h3,
constructContext(p1, p2)).isInvalid());
@@ -93,8 +95,8 @@ class EliminateJoinTest extends SqlTestBase {
.rewrite()
.applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER)
.getAllPlan().get(0).child(0);
- HyperGraph h1 = HyperGraph.builderForMv(p1).buildAll().get(0);
- HyperGraph h2 = HyperGraph.builderForMv(p2).buildAll().get(0);
+ HyperGraph h1 = HyperGraph.builderForMv(p1).build();
+ HyperGraph h2 = HyperGraph.builderForMv(p2).build();
ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2,
constructContext(p1, p2));
Assertions.assertTrue(!res.isInvalid());
Assertions.assertTrue(res.getViewExpressions().isEmpty());
@@ -134,9 +136,9 @@ class EliminateJoinTest extends SqlTestBase {
.rewrite()
.applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER)
.getAllPlan().get(0).child(0);
- HyperGraph h1 = HyperGraph.builderForMv(p1).buildAll().get(0);
- HyperGraph h2 = HyperGraph.builderForMv(p2).buildAll().get(0);
- HyperGraph h3 = HyperGraph.builderForMv(p3).buildAll().get(0);
+ HyperGraph h1 = HyperGraph.builderForMv(p1).build();
+ HyperGraph h2 = HyperGraph.builderForMv(p2).build();
+ HyperGraph h3 = HyperGraph.builderForMv(p3).build();
ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2,
constructContext(p1, p2));
Assertions.assertTrue(!res.isInvalid());
Assertions.assertTrue(res.getViewExpressions().isEmpty());
@@ -169,8 +171,8 @@ class EliminateJoinTest extends SqlTestBase {
.rewrite()
.applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER)
.getAllPlan().get(0).child(0);
- HyperGraph h1 = HyperGraph.builderForMv(p1).buildAll().get(0);
- HyperGraph h2 = HyperGraph.builderForMv(p2).buildAll().get(0);
+ HyperGraph h1 = HyperGraph.builderForMv(p1).build();
+ HyperGraph h2 = HyperGraph.builderForMv(p2).build();
ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2,
constructContext(p1, p2));
Assertions.assertTrue(!res.isInvalid());
Assertions.assertTrue(res.getViewExpressions().isEmpty());
@@ -202,8 +204,8 @@ class EliminateJoinTest extends SqlTestBase {
.rewrite()
.applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER)
.getAllPlan().get(0).child(0);
- HyperGraph h1 = HyperGraph.builderForMv(p1).buildAll().get(0);
- HyperGraph h2 = HyperGraph.builderForMv(p2).buildAll().get(0);
+ HyperGraph h1 = HyperGraph.builderForMv(p1).build();
+ HyperGraph h2 = HyperGraph.builderForMv(p2).build();
ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2,
constructContext(p1, p2));
Assertions.assertTrue(!res.isInvalid());
Assertions.assertTrue(res.getViewExpressions().isEmpty());
@@ -213,9 +215,9 @@ class EliminateJoinTest extends SqlTestBase {
LogicalCompatibilityContext constructContext(Plan p1, Plan p2) {
StructInfo st1 = MaterializedViewUtils.extractStructInfo(p1,
- null).get(0);
+ null, new BitSet()).get(0);
StructInfo st2 = MaterializedViewUtils.extractStructInfo(p2,
- null).get(0);
+ null, new BitSet()).get(0);
RelationMapping rm = RelationMapping.generate(st1.getRelations(),
st2.getRelations()).get(0);
SlotMapping sm = SlotMapping.generate(rm);
return LogicalCompatibilityContext.from(rm, sm, st1, st2);
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphAggTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphAggTest.java
index 38ebc99c479..be4ce1cd054 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphAggTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphAggTest.java
@@ -31,6 +31,7 @@ import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
+import java.util.BitSet;
import java.util.Objects;
class HyperGraphAggTest extends SqlTestBase {
@@ -49,7 +50,7 @@ class HyperGraphAggTest extends SqlTestBase {
.rewrite()
.applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER)
.getAllPlan().get(0).child(0);
- HyperGraph h1 = HyperGraph.builderForMv(p2).buildAll().get(0);
+ HyperGraph h1 = HyperGraph.builderForMv(p2).build();
Assertions.assertEquals("id", Objects.requireNonNull(((StructInfoNode)
h1.getNode(1)).getExpressions()).get(0).toSql());
}
@@ -79,8 +80,8 @@ class HyperGraphAggTest extends SqlTestBase {
.rewrite()
.applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER)
.getAllPlan().get(0).child(0);
- HyperGraph h1 = HyperGraph.builderForMv(p1).buildAll().get(0);
- HyperGraph h2 = HyperGraph.builderForMv(p2).buildAll().get(0);
+ HyperGraph h1 = HyperGraph.builderForMv(p1).build();
+ HyperGraph h2 = HyperGraph.builderForMv(p2).build();
ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2,
constructContext(p1, p2));
Assertions.assertTrue(!res.isInvalid());
Assertions.assertEquals(2, res.getViewNoNullableSlot().size());
@@ -88,9 +89,9 @@ class HyperGraphAggTest extends SqlTestBase {
LogicalCompatibilityContext constructContext(Plan p1, Plan p2) {
StructInfo st1 = MaterializedViewUtils.extractStructInfo(p1,
- null).get(0);
+ null, new BitSet()).get(0);
StructInfo st2 = MaterializedViewUtils.extractStructInfo(p2,
- null).get(0);
+ null, new BitSet()).get(0);
RelationMapping rm = RelationMapping.generate(st1.getRelations(),
st2.getRelations()).get(0);
SlotMapping sm = SlotMapping.generate(rm);
return LogicalCompatibilityContext.from(rm, sm, st1, st2);
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparatorTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparatorTest.java
index 1fd2ac86ab6..066f9dd0881 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparatorTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparatorTest.java
@@ -30,6 +30,8 @@ import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
+import java.util.BitSet;
+
class HyperGraphComparatorTest extends SqlTestBase {
@Test
void testInnerJoinAndLOJ() {
@@ -55,8 +57,8 @@ class HyperGraphComparatorTest extends SqlTestBase {
.rewrite()
.applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER)
.getAllPlan().get(0).child(0);
- HyperGraph h1 = HyperGraph.builderForMv(p1).buildAll().get(0);
- HyperGraph h2 = HyperGraph.builderForMv(p2).buildAll().get(0);
+ HyperGraph h1 = HyperGraph.builderForMv(p1).build();
+ HyperGraph h2 = HyperGraph.builderForMv(p2).build();
ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2,
constructContext(p1, p2));
Assertions.assertTrue(!res.isInvalid());
Assertions.assertEquals(2, res.getViewNoNullableSlot().size());
@@ -86,8 +88,8 @@ class HyperGraphComparatorTest extends SqlTestBase {
.rewrite()
.applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER)
.getAllPlan().get(0).child(0);
- HyperGraph h1 = HyperGraph.builderForMv(p1).buildAll().get(0);
- HyperGraph h2 = HyperGraph.builderForMv(p2).buildAll().get(0);
+ HyperGraph h1 = HyperGraph.builderForMv(p1).build();
+ HyperGraph h2 = HyperGraph.builderForMv(p2).build();
ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2,
constructContext(p1, p2));
Assertions.assertTrue(!res.isInvalid());
Assertions.assertEquals(2, res.getViewNoNullableSlot().size());
@@ -118,8 +120,8 @@ class HyperGraphComparatorTest extends SqlTestBase {
.rewrite()
.applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER)
.getAllPlan().get(0).child(0);
- HyperGraph h1 = HyperGraph.builderForMv(p1).buildAll().get(0);
- HyperGraph h2 = HyperGraph.builderForMv(p2).buildAll().get(0);
+ HyperGraph h1 = HyperGraph.builderForMv(p1).build();
+ HyperGraph h2 = HyperGraph.builderForMv(p2).build();
ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2,
constructContext(p1, p2));
Assertions.assertTrue(!res.isInvalid());
Assertions.assertEquals(2, res.getViewNoNullableSlot().size());
@@ -153,8 +155,8 @@ class HyperGraphComparatorTest extends SqlTestBase {
.rewrite()
.applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER)
.getAllPlan().get(0).child(0);
- HyperGraph h1 = HyperGraph.builderForMv(p1).buildAll().get(0);
- HyperGraph h2 = HyperGraph.builderForMv(p2).buildAll().get(0);
+ HyperGraph h1 = HyperGraph.builderForMv(p1).build();
+ HyperGraph h2 = HyperGraph.builderForMv(p2).build();
ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2,
constructContext(p1, p2));
Assertions.assertTrue(!res.isInvalid());
Assertions.assertEquals(2, res.getViewNoNullableSlot().size());
@@ -162,9 +164,9 @@ class HyperGraphComparatorTest extends SqlTestBase {
LogicalCompatibilityContext constructContext(Plan p1, Plan p2) {
StructInfo st1 = MaterializedViewUtils.extractStructInfo(p1,
- null).get(0);
+ null, new BitSet()).get(0);
StructInfo st2 = MaterializedViewUtils.extractStructInfo(p2,
- null).get(0);
+ null, new BitSet()).get(0);
RelationMapping rm = RelationMapping.generate(st1.getRelations(),
st2.getRelations()).get(0);
SlotMapping sm = SlotMapping.generate(rm);
return LogicalCompatibilityContext.from(rm, sm, st1, st2);
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/ExpressionUtilsTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/ExpressionUtilsTest.java
index f68d37a24db..83a1faace1e 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/ExpressionUtilsTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/ExpressionUtilsTest.java
@@ -26,6 +26,7 @@ import com.google.common.collect.Sets;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
+import java.util.BitSet;
import java.util.List;
/**
@@ -173,11 +174,8 @@ public class ExpressionUtilsTest extends TestWithFeService
{
Plan rewrittenPlan =
nereidsPlanner.getRewrittenPlan();
List<? extends Expression> originalExpressions =
rewrittenPlan.getExpressions();
List<? extends Expression> shuttledExpressions
- =
ExpressionUtils.shuttleExpressionWithLineage(
- originalExpressions,
- rewrittenPlan,
- Sets.newHashSet(),
- Sets.newHashSet());
+ =
ExpressionUtils.shuttleExpressionWithLineage(originalExpressions, rewrittenPlan,
+ Sets.newHashSet(), Sets.newHashSet(), new
BitSet());
assertExpect(originalExpressions,
shuttledExpressions,
"(cast(abs((cast(O_TOTALPRICE as
DECIMALV3(16, 2)) + 10.00)) as "
+ "DOUBLE) +
abs(sqrt(cast(PS_SUPPLYCOST as DOUBLE))))",
diff --git
a/regression-test/data/nereids_rules_p0/mv/nested/nested_materialized_view.out
b/regression-test/data/nereids_rules_p0/mv/nested/nested_materialized_view.out
new file mode 100644
index 00000000000..09c11bcee6a
--- /dev/null
+++
b/regression-test/data/nereids_rules_p0/mv/nested/nested_materialized_view.out
@@ -0,0 +1,20 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !query1_0_before --
+4
+4
+4
+4
+6
+6
+6
+6
+
+-- !query1_0_after --
+4
+4
+4
+4
+6
+6
+6
+6
\ No newline at end of file
diff --git
a/regression-test/suites/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.groovy
b/regression-test/suites/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.groovy
index 01672af1be0..d8037301c04 100644
---
a/regression-test/suites/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.groovy
+++
b/regression-test/suites/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.groovy
@@ -176,31 +176,37 @@ suite("aggregate_with_roll_up") {
// multi table
// filter inside + left + use roll up dimension
- def mv13_0 = "select l_shipdate, o_orderdate, l_partkey, l_suppkey, " +
- "sum(o_totalprice) as sum_total, " +
- "max(o_totalprice) as max_total, " +
- "min(o_totalprice) as min_total, " +
- "count(*) as count_all, " +
- "bitmap_union(to_bitmap(case when o_shippriority > 1 and
o_orderkey IN (1, 3) then o_custkey else null end)) as bitmap_union_basic " +
- "from lineitem " +
- "left join orders on lineitem.l_orderkey = orders.o_orderkey and
l_shipdate = o_orderdate " +
- "group by " +
- "l_shipdate, " +
- "o_orderdate, " +
- "l_partkey, " +
- "l_suppkey"
- def query13_0 = "select t1.l_partkey, t1.l_suppkey, o_orderdate, " +
- "sum(o_totalprice), " +
- "max(o_totalprice), " +
- "min(o_totalprice), " +
- "count(*), " +
- "count(distinct case when o_shippriority > 1 and o_orderkey IN (1,
3) then o_custkey else null end) " +
- "from (select * from lineitem where l_shipdate = '2023-12-11') t1
" +
- "left join orders on t1.l_orderkey = orders.o_orderkey and
t1.l_shipdate = o_orderdate " +
- "group by " +
- "o_orderdate, " +
- "l_partkey, " +
- "l_suppkey"
+ def mv13_0 =
+ """
+ select l_shipdate, o_orderdate, l_partkey, l_suppkey,
+ sum(o_totalprice) as sum_total,
+ max(o_totalprice) as max_total,
+ min(o_totalprice) as min_total,
+ count(*) as count_all,
+ bitmap_union(to_bitmap(case when o_shippriority > 1 and o_orderkey
IN (1, 3) then o_custkey else null end)) as bitmap_union_basic
+ from lineitem
+ left join orders on lineitem.l_orderkey = orders.o_orderkey and
l_shipdate = o_orderdate
+ group by
+ l_shipdate,
+ o_orderdate,
+ l_partkey,
+ l_suppkey
+ """
+ def query13_0 =
+ """
+ select t1.l_partkey, t1.l_suppkey, o_orderdate,
+ sum(o_totalprice),
+ max(o_totalprice),
+ min(o_totalprice),
+ count(*),
+ count(distinct case when o_shippriority > 1 and o_orderkey IN (1,
3) then o_custkey else null end)
+ from (select * from lineitem where l_shipdate = '2023-12-11') t1
+ left join orders on t1.l_orderkey = orders.o_orderkey and
t1.l_shipdate = o_orderdate
+ group by
+ o_orderdate,
+ l_partkey,
+ l_suppkey
+ """
order_qt_query13_0_before "${query13_0}"
check_mv_rewrite_success(db, mv13_0, query13_0, "mv13_0")
order_qt_query13_0_after "${query13_0}"
diff --git
a/regression-test/suites/nereids_rules_p0/mv/join/inner/inner_join.groovy
b/regression-test/suites/nereids_rules_p0/mv/join/inner/inner_join.groovy
index 118fa959c9a..9f0b66fc66a 100644
--- a/regression-test/suites/nereids_rules_p0/mv/join/inner/inner_join.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/join/inner/inner_join.groovy
@@ -355,13 +355,19 @@ suite("inner_join") {
// filter outside + left + right
- def mv4_0 = "select l_linenumber, o_custkey, o_orderkey, o_orderstatus " +
- "from lineitem " +
- "inner join orders on lineitem.l_orderkey = orders.o_orderkey "
- def query4_0 = "select lineitem.l_linenumber " +
- "from lineitem " +
- "inner join orders on lineitem.l_orderkey = orders.o_orderkey " +
- "where o_orderstatus = 'o' AND l_linenumber in (1, 2, 3, 4, 5) "
+ def mv4_0 =
+ """
+ select l_linenumber, o_custkey, o_orderkey, o_orderstatus
+ from lineitem
+ inner join orders on lineitem.l_orderkey = orders.o_orderkey
+ """
+ def query4_0 =
+ """
+ select lineitem.l_linenumber
+ from lineitem
+ inner join orders on lineitem.l_orderkey = orders.o_orderkey
+ where o_orderstatus = 'o' AND l_linenumber in (1, 2, 3, 4, 5)
+ """
order_qt_query4_0_before "${query4_0}"
check_mv_rewrite_success(db, mv4_0, query4_0, "mv4_0")
order_qt_query4_0_after "${query4_0}"
diff --git
a/regression-test/suites/nereids_rules_p0/mv/nested/nested_materialized_view.groovy
b/regression-test/suites/nereids_rules_p0/mv/nested/nested_materialized_view.groovy
new file mode 100644
index 00000000000..6ad175e85ce
--- /dev/null
+++
b/regression-test/suites/nereids_rules_p0/mv/nested/nested_materialized_view.groovy
@@ -0,0 +1,182 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("nested_materialized_view") {
+ String db = context.config.getDbNameByFile(context.file)
+ sql "use ${db}"
+ sql "SET enable_nereids_planner=true"
+ sql "set runtime_filter_mode=OFF"
+ sql "SET enable_fallback_to_original_planner=false"
+ sql "SET enable_materialized_view_rewrite=true"
+ sql "SET enable_nereids_timeout = false"
+
+ def create_mtmv = { db_name, mv_name, mv_sql ->
+ sql """DROP MATERIALIZED VIEW IF EXISTS ${mv_name}"""
+ sql"""
+ CREATE MATERIALIZED VIEW ${mv_name}
+ BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL
+ DISTRIBUTED BY RANDOM BUCKETS 2
+ PROPERTIES ('replication_num' = '1')
+ AS ${mv_sql}
+ """
+
+ def job_name = getJobName(db_name, mv_name);
+ waitingMTMVTaskFinished(job_name)
+ }
+
+ sql """
+ drop table if exists orders
+ """
+
+ sql """
+ CREATE TABLE IF NOT EXISTS orders (
+ o_orderkey INTEGER NOT NULL,
+ o_custkey INTEGER NOT NULL,
+ o_orderstatus CHAR(1) NOT NULL,
+ o_totalprice DECIMALV3(15,2) NOT NULL,
+ o_orderdate DATE NOT NULL,
+ o_orderpriority CHAR(15) NOT NULL,
+ o_clerk CHAR(15) NOT NULL,
+ o_shippriority INTEGER NOT NULL,
+ o_comment VARCHAR(79) NOT NULL
+ )
+ DUPLICATE KEY(o_orderkey, o_custkey)
+ PARTITION BY RANGE(o_orderdate) (PARTITION `day_2` VALUES LESS THAN
('2023-12-30'))
+ DISTRIBUTED BY HASH(o_orderkey) BUCKETS 3
+ PROPERTIES (
+ "replication_num" = "1"
+ )
+ """
+
+ sql """
+ drop table if exists lineitem
+ """
+
+ sql"""
+ CREATE TABLE IF NOT EXISTS lineitem (
+ l_orderkey INTEGER NOT NULL,
+ l_partkey INTEGER NOT NULL,
+ l_suppkey INTEGER NOT NULL,
+ l_linenumber INTEGER NOT NULL,
+ l_quantity DECIMALV3(15,2) NOT NULL,
+ l_extendedprice DECIMALV3(15,2) NOT NULL,
+ l_discount DECIMALV3(15,2) NOT NULL,
+ l_tax DECIMALV3(15,2) NOT NULL,
+ l_returnflag CHAR(1) NOT NULL,
+ l_linestatus CHAR(1) NOT NULL,
+ l_shipdate DATE NOT NULL,
+ l_commitdate DATE NOT NULL,
+ l_receiptdate DATE NOT NULL,
+ l_shipinstruct CHAR(25) NOT NULL,
+ l_shipmode CHAR(10) NOT NULL,
+ l_comment VARCHAR(44) NOT NULL
+ )
+ DUPLICATE KEY(l_orderkey, l_partkey, l_suppkey, l_linenumber)
+ PARTITION BY RANGE(l_shipdate) (PARTITION `day_1` VALUES LESS THAN
('2023-12-30'))
+ DISTRIBUTED BY HASH(l_orderkey) BUCKETS 3
+ PROPERTIES (
+ "replication_num" = "1"
+ )
+ """
+
+ sql """
+ drop table if exists partsupp
+ """
+
+ sql """
+ CREATE TABLE IF NOT EXISTS partsupp (
+ ps_partkey INTEGER NOT NULL,
+ ps_suppkey INTEGER NOT NULL,
+ ps_availqty INTEGER NOT NULL,
+ ps_supplycost DECIMALV3(15,2) NOT NULL,
+ ps_comment VARCHAR(199) NOT NULL
+ )
+ DUPLICATE KEY(ps_partkey, ps_suppkey)
+ DISTRIBUTED BY HASH(ps_partkey) BUCKETS 3
+ PROPERTIES (
+ "replication_num" = "1"
+ )
+ """
+
+ sql """ insert into lineitem values
+ (1, 2, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-08', '2023-12-09',
'2023-12-10', 'a', 'b', 'yyyyyyyyy'),
+ (2, 4, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-09', '2023-12-09',
'2023-12-10', 'a', 'b', 'yyyyyyyyy'),
+ (3, 2, 4, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-10', '2023-12-09',
'2023-12-10', 'a', 'b', 'yyyyyyyyy'),
+ (4, 3, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-11', '2023-12-09',
'2023-12-10', 'a', 'b', 'yyyyyyyyy'),
+ (5, 2, 3, 6, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-12-12', '2023-12-12',
'2023-12-13', 'c', 'd', 'xxxxxxxxx');
+ """
+
+ sql """
+ insert into orders values
+ (1, 1, 'o', 9.5, '2023-12-08', 'a', 'b', 1, 'yy'),
+ (1, 1, 'o', 10.5, '2023-12-08', 'a', 'b', 1, 'yy'),
+ (2, 1, 'o', 11.5, '2023-12-09', 'a', 'b', 1, 'yy'),
+ (3, 1, 'o', 12.5, '2023-12-10', 'a', 'b', 1, 'yy'),
+ (3, 1, 'o', 33.5, '2023-12-10', 'a', 'b', 1, 'yy'),
+ (4, 2, 'o', 43.2, '2023-12-11', 'c','d',2, 'mm'),
+ (5, 2, 'o', 56.2, '2023-12-12', 'c','d',2, 'mi'),
+ (5, 2, 'o', 1.2, '2023-12-12', 'c','d',2, 'mi');
+ """
+
+ sql """
+ insert into partsupp values
+ (2, 3, 9, 10.01, 'supply1'),
+ (2, 3, 10, 11.01, 'supply2');
+ """
+
+ // simple nested materialized view
+ def mv1_0_inner_mv = """
+ select
+ l_linenumber,
+ o_custkey,
+ o_orderkey,
+ o_orderstatus,
+ l_partkey,
+ l_suppkey,
+ l_orderkey
+ from lineitem
+ inner join orders on lineitem.l_orderkey = orders.o_orderkey;
+ """
+
+ def mv1_0 =
+ """
+ select
+ l_linenumber,
+ o_custkey,
+ o_orderkey,
+ o_orderstatus,
+ l_partkey,
+ l_suppkey,
+ l_orderkey,
+ ps_availqty
+ from mv1_0_inner_mv
+ inner join partsupp on l_partkey = ps_partkey AND l_suppkey =
ps_suppkey;
+ """
+ def query1_0 = """
+ select lineitem.l_linenumber
+ from lineitem
+ inner join orders on l_orderkey = o_orderkey
+ inner join partsupp on l_partkey = ps_partkey AND l_suppkey =
ps_suppkey
+ where o_orderstatus = 'o'
+ """
+ order_qt_query1_0_before "${query1_0}"
+ create_mtmv(db, "mv1_0_inner_mv", mv1_0_inner_mv)
+ check_mv_rewrite_success(db, mv1_0, query1_0, "mv1_0")
+ order_qt_query1_0_after "${query1_0}"
+ sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_0_inner_mv"""
+ sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_0"""
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
