This is an automated email from the ASF dual-hosted git repository. yiguolei pushed a commit to branch branch-2.1 in repository https://gitbox.apache.org/repos/asf/doris.git
commit e6a6b8220123719f1fad2337c1a3e0e74e9000e0 Author: seawinde <[email protected]> AuthorDate: Sun Apr 21 00:09:22 2024 +0800 [nereids](mtmv) Support rewrite by mv nested materialized view (#33362) Support query rewritting by nested materialized view. Such as `inner_mv` def is as following > select > l_linenumber, > o_custkey, > o_orderkey, > o_orderstatus, > l_partkey, > l_suppkey, > l_orderkey > from lineitem > inner join orders on lineitem.l_orderkey = orders.o_orderkey; the mv1_0 def is as following: > select > l_linenumber, > o_custkey, > o_orderkey, > o_orderstatus, > l_partkey, > l_suppkey, > l_orderkey, > ps_availqty > from inner_mv > inner join partsupp on l_partkey = ps_partkey AND l_suppkey = ps_suppkey; for the following query, both inner_mv and mv1_0 can be successful when query rewritting by materialized view，and cbo will chose `mv1_0` finally. > select lineitem.l_linenumber > from lineitem > inner join orders on l_orderkey = o_orderkey > inner join partsupp on l_partkey = ps_partkey AND l_suppkey = ps_suppkey > where o_orderstatus = 'o' AND l_linenumber in (1, 2, 3, 4, 5) --- .../jobs/joinorder/hypergraph/HyperGraph.java | 4 - .../java/org/apache/doris/nereids/memo/Group.java | 20 +-- .../apache/doris/nereids/memo/StructInfoMap.java | 34 ++-- .../mv/AbstractMaterializedViewAggregateRule.java | 63 ++++--- .../mv/AbstractMaterializedViewJoinRule.java | 5 +- .../mv/AbstractMaterializedViewRule.java | 148 +++++++---------- .../mv/InitMaterializationContextHook.java | 27 +-- .../mv/LogicalCompatibilityContext.java | 2 +- .../exploration/mv/MaterializationContext.java | 71 ++++++-- .../mv/MaterializedViewAggregateRule.java | 3 +- .../mv/MaterializedViewFilterAggregateRule.java | 10 +- .../mv/MaterializedViewFilterJoinRule.java | 5 +- ...MaterializedViewFilterProjectAggregateRule.java | 10 +- .../mv/MaterializedViewFilterProjectJoinRule.java | 10 +- .../mv/MaterializedViewOnlyJoinRule.java | 5 +- .../mv/MaterializedViewProjectAggregateRule.java | 10 +- ...MaterializedViewProjectFilterAggregateRule.java | 5 +- .../mv/MaterializedViewProjectFilterJoinRule.java | 10 +- .../mv/MaterializedViewProjectJoinRule.java | 10 +- .../exploration/mv/MaterializedViewUtils.java | 55 +++++-- .../nereids/rules/exploration/mv/StructInfo.java | 101 +++++++----- .../plans/visitor/ExpressionLineageReplacer.java | 37 +++-- .../apache/doris/nereids/util/ExpressionUtils.java | 15 +- .../joinorder/hypergraph/CompareOuterJoinTest.java | 29 ++-- .../jobs/joinorder/hypergraph/InferJoinTest.java | 21 +-- .../joinorder/hypergraph/InferPredicateTest.java | 10 +- .../joinorder/hypergraph/PullupExpressionTest.java | 22 +-- .../doris/nereids/memo/StructInfoMapTest.java | 2 +- .../rules/exploration/mv/BuildStructInfoTest.java | 10 +- .../rules/exploration/mv/EliminateJoinTest.java | 30 ++-- .../rules/exploration/mv/HyperGraphAggTest.java | 11 +- .../exploration/mv/HyperGraphComparatorTest.java | 22 +-- .../doris/nereids/util/ExpressionUtilsTest.java | 8 +- .../mv/nested/nested_materialized_view.out | 20 +++ .../agg_with_roll_up/aggregate_with_roll_up.groovy | 56 ++++--- .../mv/join/inner/inner_join.groovy | 20 ++- .../mv/nested/nested_materialized_view.groovy | 182 +++++++++++++++++++++ 37 files changed, 718 insertions(+), 385 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/HyperGraph.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/HyperGraph.java index 5e45fc0bdb8..19ff555469c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/HyperGraph.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/HyperGraph.java @@ -356,10 +356,6 @@ public class HyperGraph { return new HyperGraph(finalOutputs, joinEdges, nodes, filterEdges, complexProject); } - public List<HyperGraph> buildAll() { - return ImmutableList.of(build()); - } - public void updateNode(int idx, Group group) { Preconditions.checkArgument(nodes.get(idx) instanceof DPhyperNode); nodes.set(idx, ((DPhyperNode) nodes.get(idx)).withGroup(group)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Group.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Group.java index 01968a03bef..a9d3d881491 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Group.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Group.java @@ -21,7 +21,6 @@ import org.apache.doris.common.Pair; import org.apache.doris.nereids.cost.Cost; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.properties.PhysicalProperties; -import org.apache.doris.nereids.rules.exploration.mv.StructInfo; import org.apache.doris.nereids.trees.expressions.literal.Literal; import org.apache.doris.nereids.trees.plans.JoinType; import org.apache.doris.nereids.trees.plans.Plan; @@ -76,8 +75,6 @@ public class Group { private int chosenGroupExpressionId = -1; - private List<StructInfo> structInfos = new ArrayList<>(); - private StructInfoMap structInfoMap = new StructInfoMap(); /** @@ -472,6 +469,7 @@ public class Group { } str.append(" stats").append("\n"); str.append(getStatistics() == null ? "" : getStatistics().detail(" ")); + str.append(" lowest Plan(cost, properties, plan, childrenRequires)"); getAllProperties().forEach( prop -> { @@ -485,6 +483,10 @@ public class Group { } } ); + + str.append("\n").append(" struct info map").append("\n"); + str.append(structInfoMap); + return str.toString(); } @@ -557,16 +559,4 @@ public class Group { return TreeStringUtils.treeString(this, toString, getChildren, getExtraPlans, displayExtraPlan); } - - public List<StructInfo> getStructInfos() { - return structInfos; - } - - public void addStructInfo(StructInfo structInfo) { - this.structInfos.add(structInfo); - } - - public void addStructInfo(List<StructInfo> structInfos) { - this.structInfos.addAll(structInfos); - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java index d065c5cd3e2..a14c2070a8f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java @@ -26,6 +26,7 @@ import com.google.common.collect.Sets; import java.util.ArrayList; import java.util.BitSet; +import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -49,7 +50,7 @@ public class StructInfoMap { * @param group the group that the mv matched * @return struct info or null if not found */ - public @Nullable StructInfo getStructInfo(BitSet mvTableMap, BitSet foldTableMap, Group group) { + public @Nullable StructInfo getStructInfo(BitSet mvTableMap, BitSet foldTableMap, Group group, Plan originPlan) { if (!infoMap.containsKey(mvTableMap)) { if ((groupExpressionMap.containsKey(foldTableMap) || groupExpressionMap.isEmpty()) && !groupExpressionMap.containsKey(mvTableMap)) { @@ -59,11 +60,10 @@ public class StructInfoMap { Pair<GroupExpression, List<BitSet>> groupExpressionBitSetPair = getGroupExpressionWithChildren( mvTableMap); StructInfo structInfo = constructStructInfo(groupExpressionBitSetPair.first, - groupExpressionBitSetPair.second, mvTableMap); + groupExpressionBitSetPair.second, mvTableMap, originPlan); infoMap.put(mvTableMap, structInfo); } } - return infoMap.get(mvTableMap); } @@ -71,13 +71,19 @@ public class StructInfoMap { return groupExpressionMap.keySet(); } + public Collection<StructInfo> getStructInfos() { + return infoMap.values(); + } + public Pair<GroupExpression, List<BitSet>> getGroupExpressionWithChildren(BitSet tableMap) { return groupExpressionMap.get(tableMap); } - private StructInfo constructStructInfo(GroupExpression groupExpression, List<BitSet> children, BitSet tableMap) { + private StructInfo constructStructInfo(GroupExpression groupExpression, List<BitSet> children, + BitSet tableMap, Plan originPlan) { + // this plan is not origin plan, should record origin plan in struct info Plan plan = constructPlan(groupExpression, children, tableMap); - return StructInfo.of(plan).get(0); + return originPlan == null ? StructInfo.of(plan) : StructInfo.of(plan, originPlan); } private Plan constructPlan(GroupExpression groupExpression, List<BitSet> children, BitSet tableMap) { @@ -120,12 +126,11 @@ public class StructInfoMap { refreshedGroup.add(child); childrenTableMap.add(child.getstructInfoMap().getTableMaps()); } - - if (needRefresh) { - Set<Pair<BitSet, List<BitSet>>> bitSetWithChildren = cartesianProduct(childrenTableMap); - for (Pair<BitSet, List<BitSet>> bitSetWithChild : bitSetWithChildren) { - groupExpressionMap.put(bitSetWithChild.first, Pair.of(groupExpression, bitSetWithChild.second)); - } + // if cumulative child table map is different from current + // or current group expression map is empty, should update the groupExpressionMap currently + Set<Pair<BitSet, List<BitSet>>> bitSetWithChildren = cartesianProduct(childrenTableMap); + for (Pair<BitSet, List<BitSet>> bitSetWithChild : bitSetWithChildren) { + groupExpressionMap.putIfAbsent(bitSetWithChild.first, Pair.of(groupExpression, bitSetWithChild.second)); } } return originSize != groupExpressionMap.size(); @@ -135,7 +140,7 @@ public class StructInfoMap { Plan plan = groupExpression.getPlan(); BitSet tableMap = new BitSet(); if (plan instanceof LogicalCatalogRelation) { - // TODO: Bitmap is not compatible with long, use tree map instead + // TODO: Bitset is not compatible with long, use tree map instead tableMap.set((int) ((LogicalCatalogRelation) plan).getTable().getId()); } // one row relation / CTE consumer @@ -154,4 +159,9 @@ public class StructInfoMap { }) .collect(Collectors.toSet()); } + + @Override + public String toString() { + return "StructInfoMap{ groupExpressionMap = " + groupExpressionMap + ", infoMap = " + infoMap + '}'; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java index 5cd2fb769b7..5c8c479986a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java @@ -56,6 +56,7 @@ import com.google.common.collect.Multimap; import com.google.common.collect.Sets; import java.util.ArrayList; +import java.util.BitSet; import java.util.Collection; import java.util.HashSet; import java.util.List; @@ -180,18 +181,27 @@ public abstract class AbstractMaterializedViewAggregateRule extends AbstractMate } // Firstly,if group by expression between query and view is equals, try to rewrite expression directly Plan queryTopPlan = queryTopPlanAndAggPair.key(); - if (isGroupByEquals(queryTopPlanAndAggPair, viewTopPlanAndAggPair, viewToQuerySlotMapping)) { + if (isGroupByEquals(queryTopPlanAndAggPair, viewTopPlanAndAggPair, viewToQuerySlotMapping, queryStructInfo, + viewStructInfo)) { List<Expression> rewrittenQueryExpressions = rewriteExpression(queryTopPlan.getOutput(), queryTopPlan, materializationContext.getMvExprToMvScanExprMapping(), viewToQuerySlotMapping, - true); + true, + queryStructInfo.getTableBitSet()); if (!rewrittenQueryExpressions.isEmpty()) { - return new LogicalProject<>( - rewrittenQueryExpressions.stream().map(NamedExpression.class::cast) - .collect(Collectors.toList()), - tempRewritedPlan); - + List<NamedExpression> projects = new ArrayList<>(); + for (Expression expression : rewrittenQueryExpressions) { + if (expression.containsType(AggregateFunction.class)) { + materializationContext.recordFailReason(queryStructInfo, + "rewritten expression contains aggregate functions when group equals aggregate rewrite", + () -> String.format("aggregate functions = %s\n", rewrittenQueryExpressions)); + return null; + } + projects.add(expression instanceof NamedExpression + ? (NamedExpression) expression : new Alias(expression)); + } + return new LogicalProject<>(projects, tempRewritedPlan); } // if fails, record the reason and then try to roll up aggregate function materializationContext.recordFailReason(queryStructInfo, @@ -219,7 +229,7 @@ public abstract class AbstractMaterializedViewAggregateRule extends AbstractMate // try to roll up. // split the query top plan expressions to group expressions and functions, if can not, bail out. Pair<Set<? extends Expression>, Set<? extends Expression>> queryGroupAndFunctionPair - = topPlanSplitToGroupAndFunction(queryTopPlanAndAggPair); + = topPlanSplitToGroupAndFunction(queryTopPlanAndAggPair, queryStructInfo); Set<? extends Expression> queryTopPlanFunctionSet = queryGroupAndFunctionPair.value(); // try to rewrite, contains both roll up aggregate functions and aggregate group expression List<NamedExpression> finalOutputExpressions = new ArrayList<>(); @@ -234,9 +244,10 @@ public abstract class AbstractMaterializedViewAggregateRule extends AbstractMate if (queryTopPlanFunctionSet.contains(topExpression)) { Expression queryFunctionShuttled = ExpressionUtils.shuttleExpressionWithLineage( topExpression, - queryTopPlan); + queryTopPlan, + queryStructInfo.getTableBitSet()); AggregateExpressionRewriteContext context = new AggregateExpressionRewriteContext( - false, mvExprToMvScanExprQueryBased, queryTopPlan); + false, mvExprToMvScanExprQueryBased, queryTopPlan, queryStructInfo.getTableBitSet()); // queryFunctionShuttled maybe sum(column) + count(*), so need to use expression rewriter Expression rollupedExpression = queryFunctionShuttled.accept(AGGREGATE_EXPRESSION_REWRITER, context); @@ -250,10 +261,10 @@ public abstract class AbstractMaterializedViewAggregateRule extends AbstractMate finalOutputExpressions.add(new Alias(rollupedExpression)); } else { // if group by expression, try to rewrite group by expression - Expression queryGroupShuttledExpr = - ExpressionUtils.shuttleExpressionWithLineage(topExpression, queryTopPlan); - AggregateExpressionRewriteContext context = new AggregateExpressionRewriteContext( - true, mvExprToMvScanExprQueryBased, queryTopPlan); + Expression queryGroupShuttledExpr = ExpressionUtils.shuttleExpressionWithLineage( + topExpression, queryTopPlan, queryStructInfo.getTableBitSet()); + AggregateExpressionRewriteContext context = new AggregateExpressionRewriteContext(true, + mvExprToMvScanExprQueryBased, queryTopPlan, queryStructInfo.getTableBitSet()); // group by expression maybe group by a + b, so we need expression rewriter Expression rewrittenGroupByExpression = queryGroupShuttledExpr.accept(AGGREGATE_EXPRESSION_REWRITER, context); @@ -302,16 +313,18 @@ public abstract class AbstractMaterializedViewAggregateRule extends AbstractMate private boolean isGroupByEquals(Pair<Plan, LogicalAggregate<Plan>> queryTopPlanAndAggPair, Pair<Plan, LogicalAggregate<Plan>> viewTopPlanAndAggPair, - SlotMapping viewToQuerySlotMapping) { + SlotMapping viewToQuerySlotMapping, + StructInfo queryStructInfo, + StructInfo viewStructInfo) { Plan queryTopPlan = queryTopPlanAndAggPair.key(); Plan viewTopPlan = viewTopPlanAndAggPair.key(); LogicalAggregate<Plan> queryAggregate = queryTopPlanAndAggPair.value(); LogicalAggregate<Plan> viewAggregate = viewTopPlanAndAggPair.value(); Set<? extends Expression> queryGroupShuttledExpression = new HashSet<>( ExpressionUtils.shuttleExpressionWithLineage( - queryAggregate.getGroupByExpressions(), queryTopPlan)); + queryAggregate.getGroupByExpressions(), queryTopPlan, queryStructInfo.getTableBitSet())); Set<? extends Expression> viewGroupShuttledExpressionQueryBased = ExpressionUtils.shuttleExpressionWithLineage( - viewAggregate.getGroupByExpressions(), viewTopPlan) + viewAggregate.getGroupByExpressions(), viewTopPlan, viewStructInfo.getTableBitSet()) .stream() .map(expr -> ExpressionUtils.replace(expr, viewToQuerySlotMapping.toSlotReferenceMap())) .collect(Collectors.toSet()); @@ -384,7 +397,7 @@ public abstract class AbstractMaterializedViewAggregateRule extends AbstractMate } private Pair<Set<? extends Expression>, Set<? extends Expression>> topPlanSplitToGroupAndFunction( - Pair<Plan, LogicalAggregate<Plan>> topPlanAndAggPair) { + Pair<Plan, LogicalAggregate<Plan>> topPlanAndAggPair, StructInfo queryStructInfo) { LogicalAggregate<Plan> bottomQueryAggregate = topPlanAndAggPair.value(); Set<Expression> groupByExpressionSet = new HashSet<>(bottomQueryAggregate.getGroupByExpressions()); // when query is bitmap_count(bitmap_union), the plan is as following: @@ -403,7 +416,7 @@ public abstract class AbstractMaterializedViewAggregateRule extends AbstractMate queryTopPlan.getOutput().forEach(expression -> { ExpressionLineageReplacer.ExpressionReplaceContext replaceContext = new ExpressionLineageReplacer.ExpressionReplaceContext(ImmutableList.of(expression), - ImmutableSet.of(), ImmutableSet.of()); + ImmutableSet.of(), ImmutableSet.of(), queryStructInfo.getTableBitSet()); queryTopPlan.accept(ExpressionLineageReplacer.INSTANCE, replaceContext); if (!Sets.intersection(bottomAggregateFunctionExprIdSet, replaceContext.getExprIdExpressionMap().keySet()).isEmpty()) { @@ -509,7 +522,8 @@ public abstract class AbstractMaterializedViewAggregateRule extends AbstractMate } Expression queryFunctionShuttled = ExpressionUtils.shuttleExpressionWithLineage( aggregateFunction, - rewriteContext.getQueryTopPlan()); + rewriteContext.getQueryTopPlan(), + rewriteContext.getQueryTableBitSet()); Function rollupAggregateFunction = rollup(aggregateFunction, queryFunctionShuttled, rewriteContext.getMvExprToMvScanExprQueryBasedMapping()); if (rollupAggregateFunction == null) { @@ -565,12 +579,15 @@ public abstract class AbstractMaterializedViewAggregateRule extends AbstractMate private final boolean onlyContainGroupByExpression; private final Map<Expression, Expression> mvExprToMvScanExprQueryBasedMapping; private final Plan queryTopPlan; + private final BitSet queryTableBitSet; public AggregateExpressionRewriteContext(boolean onlyContainGroupByExpression, - Map<Expression, Expression> mvExprToMvScanExprQueryBasedMapping, Plan queryTopPlan) { + Map<Expression, Expression> mvExprToMvScanExprQueryBasedMapping, Plan queryTopPlan, + BitSet queryTableBitSet) { this.onlyContainGroupByExpression = onlyContainGroupByExpression; this.mvExprToMvScanExprQueryBasedMapping = mvExprToMvScanExprQueryBasedMapping; this.queryTopPlan = queryTopPlan; + this.queryTableBitSet = queryTableBitSet; } public boolean isValid() { @@ -592,5 +609,9 @@ public abstract class AbstractMaterializedViewAggregateRule extends AbstractMate public Plan getQueryTopPlan() { return queryTopPlan; } + + public BitSet getQueryTableBitSet() { + return queryTableBitSet; + } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewJoinRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewJoinRule.java index df58345405a..2a05fecd33f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewJoinRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewJoinRule.java @@ -44,10 +44,11 @@ public abstract class AbstractMaterializedViewJoinRule extends AbstractMateriali // Rewrite top projects, represent the query projects by view List<Expression> expressionsRewritten = rewriteExpression( queryStructInfo.getExpressions(), - queryStructInfo.getOriginalPlan(), + queryStructInfo.getTopPlan(), materializationContext.getMvExprToMvScanExprMapping(), targetToSourceMapping, - true + true, + queryStructInfo.getTableBitSet() ); // Can not rewrite, bail out if (expressionsRewritten.isEmpty()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java index 90e0f8ed1db..b596408851f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java @@ -38,6 +38,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Not; import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction; import org.apache.doris.nereids.trees.expressions.functions.scalar.NonNullable; import org.apache.doris.nereids.trees.expressions.functions.scalar.Nullable; import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral; @@ -57,6 +58,7 @@ import com.google.common.collect.Maps; import com.google.common.collect.Sets; import java.util.ArrayList; +import java.util.BitSet; import java.util.Collection; import java.util.List; import java.util.Map; @@ -85,52 +87,62 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac */ public List<Plan> rewrite(Plan queryPlan, CascadesContext cascadesContext) { List<Plan> rewrittenPlans = new ArrayList<>(); - // already rewrite or query is invalid, bail out - List<StructInfo> queryStructInfos = checkQuery(queryPlan, cascadesContext); - if (queryStructInfos.isEmpty()) { + // if available materialization list is empty, bail out + if (cascadesContext.getMaterializationContexts().isEmpty()) { return rewrittenPlans; } for (MaterializationContext context : cascadesContext.getMaterializationContexts()) { if (checkIfRewritten(queryPlan, context)) { continue; } - // TODO Just support only one query struct info, support multi later. - StructInfo queryStructInfo = queryStructInfos.get(0); - try { - if (rewrittenPlans.size() < cascadesContext.getConnectContext() - .getSessionVariable().getMaterializedViewRewriteSuccessCandidateNum()) { - rewrittenPlans.addAll(doRewrite(queryStructInfo, cascadesContext, context)); + context.tryReGenerateMvScanPlan(cascadesContext); + // check mv plan is valid or not + if (!checkPattern(context.getStructInfo())) { + context.recordFailReason(context.getStructInfo(), + "View struct info is invalid", () -> String.format(", view plan is %s", + context.getStructInfo().getOriginalPlan().treeString())); + continue; + } + // get query struct infos according to the view strut info, if valid query struct infos is empty, bail out + List<StructInfo> queryStructInfos = getValidQueryStructInfos(queryPlan, cascadesContext, + context.getStructInfo().getTableBitSet()); + if (queryStructInfos.isEmpty()) { + continue; + } + for (StructInfo queryStructInfo : queryStructInfos) { + try { + if (rewrittenPlans.size() < cascadesContext.getConnectContext() + .getSessionVariable().getMaterializedViewRewriteSuccessCandidateNum()) { + rewrittenPlans.addAll(doRewrite(queryStructInfo, cascadesContext, context)); + } + } catch (Exception exception) { + context.recordFailReason(queryStructInfo, + "Materialized view rule exec fail", exception::toString); } - } catch (Exception exception) { - context.recordFailReason(queryStructInfo, - "Materialized view rule exec fail", exception::toString); } } return rewrittenPlans; } /** - * Check query is valid or not, if valid return the query struct infos, if invalid return empty list. + * Get valid query struct infos, if invalid record the invalid reason */ - protected List<StructInfo> checkQuery(Plan queryPlan, CascadesContext cascadesContext) { - List<StructInfo> validQueryStructInfos = new ArrayList<>(); - List<MaterializationContext> materializationContexts = cascadesContext.getMaterializationContexts(); - if (materializationContexts.isEmpty()) { - return validQueryStructInfos; - } - List<StructInfo> queryStructInfos = MaterializedViewUtils.extractStructInfo(queryPlan, cascadesContext); - // TODO Just Check query queryPlan firstly, support multi later. - StructInfo queryStructInfo = queryStructInfos.get(0); - if (!checkPattern(queryStructInfo)) { - for (MaterializationContext ctx : cascadesContext.getMaterializationContexts()) { - ctx.recordFailReason(queryStructInfo, "Query struct info is invalid", - () -> String.format("queryPlan is %s", queryPlan.treeString()) - ); - } - return validQueryStructInfos; - } - validQueryStructInfos.add(queryStructInfo); - return validQueryStructInfos; + protected List<StructInfo> getValidQueryStructInfos(Plan queryPlan, CascadesContext cascadesContext, + BitSet materializedViewTableSet) { + return MaterializedViewUtils.extractStructInfo(queryPlan, cascadesContext, materializedViewTableSet) + .stream() + .filter(queryStructInfo -> { + boolean valid = checkPattern(queryStructInfo); + if (!valid) { + cascadesContext.getMaterializationContexts().forEach(ctx -> + ctx.recordFailReason(queryStructInfo, "Query struct info is invalid", + () -> String.format("query table bitmap is %s, plan is %s", + queryStructInfo.getTableBitSet(), queryPlan.treeString()) + )); + } + return valid; + }) + .collect(Collectors.toList()); } /** @@ -140,22 +152,7 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac protected List<Plan> doRewrite(StructInfo queryStructInfo, CascadesContext cascadesContext, MaterializationContext materializationContext) { List<Plan> rewriteResults = new ArrayList<>(); - List<StructInfo> viewStructInfos = MaterializedViewUtils.extractStructInfo( - materializationContext.getMvPlan(), cascadesContext); - if (viewStructInfos.size() > 1) { - // view struct info should only have one - materializationContext.recordFailReason(queryStructInfo, - "The num of view struct info is more then one", - () -> String.format("mv plan is %s", materializationContext.getMvPlan().treeString())); - return rewriteResults; - } - StructInfo viewStructInfo = viewStructInfos.get(0); - if (!checkPattern(viewStructInfo)) { - materializationContext.recordFailReason(queryStructInfo, - "View struct info is invalid", - () -> String.format(", view plan is %s", viewStructInfo.getOriginalPlan().treeString())); - return rewriteResults; - } + StructInfo viewStructInfo = materializationContext.getStructInfo(); MatchMode matchMode = decideMatchMode(queryStructInfo.getRelations(), viewStructInfo.getRelations()); if (MatchMode.COMPLETE != matchMode) { materializationContext.recordFailReason(queryStructInfo, "Match mode is invalid", @@ -178,15 +175,6 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac continue; } SlotMapping viewToQuerySlotMapping = queryToViewSlotMapping.inverse(); - // check the column used in query is in mv or not - if (!checkColumnUsedValid(queryStructInfo, viewStructInfo, queryToViewSlotMapping)) { - materializationContext.recordFailReason(queryStructInfo, - "The columns used by query are not in view", - () -> String.format("query struct info is %s, view struct info is %s", - queryStructInfo.getTopPlan().treeString(), - viewStructInfo.getTopPlan().treeString())); - continue; - } LogicalCompatibilityContext compatibilityContext = LogicalCompatibilityContext.from( queryToViewTableMapping, queryToViewSlotMapping, queryStructInfo, viewStructInfo); ComparisonResult comparisonResult = StructInfo.isGraphLogicalEquals(queryStructInfo, viewStructInfo, @@ -212,14 +200,14 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac } Plan rewrittenPlan; Plan mvScan = materializationContext.getMvScanPlan(); - Plan originalPlan = queryStructInfo.getOriginalPlan(); + Plan topPlan = queryStructInfo.getTopPlan(); if (compensatePredicates.isAlwaysTrue()) { rewrittenPlan = mvScan; } else { // Try to rewrite compensate predicates by using mv scan List<Expression> rewriteCompensatePredicates = rewriteExpression(compensatePredicates.toList(), - originalPlan, materializationContext.getMvExprToMvScanExprMapping(), - viewToQuerySlotMapping, true); + topPlan, materializationContext.getMvExprToMvScanExprMapping(), + viewToQuerySlotMapping, true, queryStructInfo.getTableBitSet()); if (rewriteCompensatePredicates.isEmpty()) { materializationContext.recordFailReason(queryStructInfo, "Rewrite compensate predicate by view fail", @@ -237,13 +225,13 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac if (rewrittenPlan == null) { continue; } - final Plan finalRewrittenPlan = rewriteByRules(cascadesContext, rewrittenPlan, originalPlan); - if (!isOutputValid(originalPlan, finalRewrittenPlan)) { + final Plan finalRewrittenPlan = rewriteByRules(cascadesContext, rewrittenPlan, topPlan); + if (!isOutputValid(topPlan, finalRewrittenPlan)) { materializationContext.recordFailReason(queryStructInfo, "RewrittenPlan output logical properties is different with target group", () -> String.format("planOutput logical" + " properties = %s,\n groupOutput logical properties = %s", - finalRewrittenPlan.getLogicalProperties(), originalPlan.getLogicalProperties())); + finalRewrittenPlan.getLogicalProperties(), topPlan.getLogicalProperties())); continue; } // check the partitions used by rewritten plan is valid or not @@ -260,32 +248,12 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac .collect(Collectors.toSet()))); continue; } - recordIfRewritten(originalPlan, materializationContext); + recordIfRewritten(queryStructInfo.getOriginalPlan(), materializationContext); rewriteResults.add(finalRewrittenPlan); } return rewriteResults; } - /** - * Check the column used by query is in materialized view output or not - */ - protected boolean checkColumnUsedValid(StructInfo queryInfo, StructInfo mvInfo, - SlotMapping queryToViewSlotMapping) { - Set<ExprId> queryUsedSlotSetViewBased = ExpressionUtils.shuttleExpressionWithLineage( - queryInfo.getTopPlan().getOutput(), queryInfo.getTopPlan()).stream() - .flatMap(expr -> ExpressionUtils.replace(expr, queryToViewSlotMapping.toSlotReferenceMap()) - .collectToSet(each -> each instanceof Slot).stream()) - .map(each -> ((Slot) each).getExprId()) - .collect(Collectors.toSet()); - - Set<ExprId> viewUsedSlotSet = ExpressionUtils.shuttleExpressionWithLineage(mvInfo.getTopPlan().getOutput(), - mvInfo.getTopPlan()).stream() - .flatMap(expr -> expr.collectToSet(each -> each instanceof Slot).stream()) - .map(each -> ((Slot) each).getExprId()) - .collect(Collectors.toSet()); - return viewUsedSlotSet.containsAll(queryUsedSlotSetViewBased); - } - /** * Rewrite by rules and try to make output is the same after optimize by rules */ @@ -398,13 +366,13 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac */ protected List<Expression> rewriteExpression(List<? extends Expression> sourceExpressionsToWrite, Plan sourcePlan, ExpressionMapping targetExpressionMapping, SlotMapping targetToSourceMapping, - boolean targetExpressionNeedSourceBased) { + boolean targetExpressionNeedSourceBased, BitSet sourcePlanBitSet) { // Firstly, rewrite the target expression using source with inverse mapping // then try to use the target expression to represent the query. if any of source expressions // can not be represented by target expressions, return null. // generate target to target replacement expression mapping, and change target expression to source based List<? extends Expression> sourceShuttledExpressions = ExpressionUtils.shuttleExpressionWithLineage( - sourceExpressionsToWrite, sourcePlan); + sourceExpressionsToWrite, sourcePlan, sourcePlanBitSet); ExpressionMapping expressionMappingKeySourceBased = targetExpressionNeedSourceBased ? targetExpressionMapping.keyPermute(targetToSourceMapping) : targetExpressionMapping; // target to target replacement expression mapping, because mv is 1:1 so get first element @@ -516,6 +484,12 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac || residualCompensatePredicates == null) { return SplitPredicate.INVALID_INSTANCE; } + if (equalCompensateConjunctions.stream().anyMatch(expr -> expr.containsType(AggregateFunction.class)) + || rangeCompensatePredicates.stream().anyMatch(expr -> expr.containsType(AggregateFunction.class)) + || residualCompensatePredicates.stream().anyMatch(expr -> + expr.containsType(AggregateFunction.class))) { + return SplitPredicate.INVALID_INSTANCE; + } return SplitPredicate.of(equalCompensateConjunctions.isEmpty() ? BooleanLiteral.TRUE : ExpressionUtils.and(equalCompensateConjunctions), rangeCompensatePredicates.isEmpty() ? BooleanLiteral.TRUE @@ -592,7 +566,7 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac protected void recordIfRewritten(Plan plan, MaterializationContext context) { context.setSuccess(true); if (plan.getGroupExpression().isPresent()) { - context.addMatchedGroup(plan.getGroupExpression().get().getOwnerGroup().getGroupId()); + context.addMatchedGroup(plan.getGroupExpression().get().getOwnerGroup().getGroupId(), true); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java index 2d5c9bf377b..08c312b4737 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java @@ -24,23 +24,16 @@ import org.apache.doris.mtmv.BaseTableInfo; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.NereidsPlanner; import org.apache.doris.nereids.PlannerHook; -import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.plans.Plan; -import org.apache.doris.nereids.trees.plans.PreAggStatus; -import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; -import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import org.apache.doris.nereids.trees.plans.visitor.TableCollector; import org.apache.doris.nereids.trees.plans.visitor.TableCollector.TableCollectorContext; import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Lists; import com.google.common.collect.Sets; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import java.util.List; -import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -77,25 +70,15 @@ public class InitMaterializationContextHook implements PlannerHook { Set<MTMV> availableMTMVs = Env.getCurrentEnv().getMtmvService().getRelationManager() .getAvailableMTMVs(usedBaseTables, cascadesContext.getConnectContext()); if (availableMTMVs.isEmpty()) { + LOG.warn(String.format("enable materialized view rewrite but availableMTMVs is empty, current queryId " + + "is %s", cascadesContext.getConnectContext().getQueryIdentifier())); return; } for (MTMV materializedView : availableMTMVs) { - // generate outside, maybe add partition filter in the future - LogicalOlapScan mvScan = new LogicalOlapScan( - cascadesContext.getStatementContext().getNextRelationId(), - materializedView, - ImmutableList.of(materializedView.getQualifiedDbName()), - // this must be empty, or it will be used to sample - Lists.newArrayList(), - Lists.newArrayList(), - Optional.empty()); - mvScan = mvScan.withMaterializedIndexSelected(PreAggStatus.on(), materializedView.getBaseIndexId()); - List<NamedExpression> mvProjects = mvScan.getOutput().stream().map(NamedExpression.class::cast) - .collect(Collectors.toList()); - // todo should force keep consistency to mv sql plan output - Plan projectScan = new LogicalProject<Plan>(mvProjects, mvScan); cascadesContext.addMaterializationContext( - MaterializationContext.fromMaterializedView(materializedView, projectScan, cascadesContext)); + MaterializationContext.fromMaterializedView(materializedView, + MaterializedViewUtils.generateMvScanPlan(materializedView, cascadesContext), + cascadesContext)); } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/LogicalCompatibilityContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/LogicalCompatibilityContext.java index 3cd3000a670..7f92735da64 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/LogicalCompatibilityContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/LogicalCompatibilityContext.java @@ -56,7 +56,7 @@ public class LogicalCompatibilityContext { this.queryToViewNodeMapping = queryToViewNodeMapping; this.queryToViewEdgeExpressionMapping = queryToViewEdgeExpressionMapping; this.queryToViewNodeIDMapping = HashBiMap.create(); - this.planNodeId = queryStructInfo.getOriginalPlan().getGroupExpression() + this.planNodeId = queryStructInfo.getTopPlan().getGroupExpression() .map(GroupExpression::getId).orElseGet(() -> new ObjectId(-1)); queryToViewNodeMapping.forEach((k, v) -> queryToViewNodeIDMapping.put(k.getIndex(), v.getIndex())); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java index db9f58ae070..1af63dab21f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java @@ -26,6 +26,7 @@ import org.apache.doris.mtmv.MTMVCache; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.memo.GroupId; import org.apache.doris.nereids.rules.exploration.mv.mapping.ExpressionMapping; +import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.plans.ObjectId; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.commands.ExplainCommand.ExplainLevel; @@ -37,6 +38,7 @@ import com.google.common.collect.ImmutableSet; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import java.util.BitSet; import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; @@ -53,23 +55,27 @@ public class MaterializationContext { private static final Logger LOG = LogManager.getLogger(MaterializationContext.class); private final MTMV mtmv; - // Should use stmt id generator in query context - private final Plan mvScanPlan; private final List<Table> baseTables; private final List<Table> baseViews; // Group ids that are rewritten by this mv to reduce rewrite times - private final Set<GroupId> matchedGroups = new HashSet<>(); - // generate form mv scan plan + private final Set<GroupId> matchedFailGroups = new HashSet<>(); + private final Set<GroupId> matchedSuccessGroups = new HashSet<>(); + // if rewrite by mv fail, record the reason, if success the failReason should be empty. + // The key is the query belonged group expression objectId, the value is the fail reason + private final Map<ObjectId, Pair<String, String>> failReason = new LinkedHashMap<>(); + // Should regenerate when materialization is already rewritten successfully because one query may hit repeatedly + // make sure output is different in multi using + private Plan mvScanPlan; + // generated expressions form mv scan plan private ExpressionMapping mvExprToMvScanExprMapping; + private List<? extends Expression> mvPlanOutputShuttledExpressions; private boolean available = true; // the mv plan from cache at present, record it to make sure query rewrite by mv is right when cache change. private Plan mvPlan; // mark rewrite success or not private boolean success = false; - // if rewrite by mv fail, record the reason, if success the failReason should be empty. - // The key is the query belonged group expression objectId, the value is the fail reason - private final Map<ObjectId, Pair<String, String>> failReason = new LinkedHashMap<>(); private boolean enableRecordFailureDetail = false; + private StructInfo structInfo; /** * MaterializationContext, this contains necessary info for query rewriting by mv @@ -93,22 +99,50 @@ public class MaterializationContext { this.available = false; return; } + this.mvPlanOutputShuttledExpressions = ExpressionUtils.shuttleExpressionWithLineage( + mtmvCache.getOriginalPlan().getOutput(), + mtmvCache.getOriginalPlan(), + new BitSet()); // mv output expression shuttle, this will be used to expression rewrite - this.mvExprToMvScanExprMapping = ExpressionMapping.generate( - ExpressionUtils.shuttleExpressionWithLineage( - mtmvCache.getOriginalPlan().getOutput(), - mtmvCache.getOriginalPlan()), - mvScanPlan.getExpressions()); + this.mvExprToMvScanExprMapping = ExpressionMapping.generate(this.mvPlanOutputShuttledExpressions, + this.mvScanPlan.getExpressions()); // copy the plan from cache, which the plan in cache may change this.mvPlan = mtmvCache.getLogicalPlan(); + List<StructInfo> viewStructInfos = MaterializedViewUtils.extractStructInfo( + mtmvCache.getLogicalPlan(), cascadesContext, new BitSet()); + if (viewStructInfos.size() > 1) { + // view struct info should only have one, log error and use the first struct info + LOG.warn(String.format("view strut info is more than one, mv name is %s, mv plan is %s", + mtmv.getName(), mvPlan.treeString())); + } + this.structInfo = viewStructInfos.get(0); } public boolean alreadyRewrite(GroupId groupId) { - return this.matchedGroups.contains(groupId); + return this.matchedFailGroups.contains(groupId) || this.matchedSuccessGroups.contains(groupId); } - public void addMatchedGroup(GroupId groupId) { - matchedGroups.add(groupId); + public void addMatchedGroup(GroupId groupId, boolean rewriteSuccess) { + if (rewriteSuccess) { + this.matchedSuccessGroups.add(groupId); + } else { + this.matchedFailGroups.add(groupId); + } + } + + /** + * Try to generate scan plan for materialized view + * if MaterializationContext is already rewritten by materialized view, then should generate in real time + * when query rewrite, because one plan may hit the materialized view repeatedly and the mv scan output + * should be different + */ + public void tryReGenerateMvScanPlan(CascadesContext cascadesContext) { + if (!this.matchedSuccessGroups.isEmpty()) { + this.mvScanPlan = MaterializedViewUtils.generateMvScanPlan(this.mtmv, cascadesContext); + // mv output expression shuttle, this will be used to expression rewrite + this.mvExprToMvScanExprMapping = ExpressionMapping.generate(this.mvPlanOutputShuttledExpressions, + this.mvScanPlan.getExpressions()); + } } public MTMV getMTMV() { @@ -152,13 +186,18 @@ public class MaterializationContext { this.failReason.clear(); } + public StructInfo getStructInfo() { + return structInfo; + } + /** * recordFailReason */ public void recordFailReason(StructInfo structInfo, String summary, Supplier<String> failureReasonSupplier) { // record it's rewritten if (structInfo.getTopPlan().getGroupExpression().isPresent()) { - this.addMatchedGroup(structInfo.getTopPlan().getGroupExpression().get().getOwnerGroup().getGroupId()); + this.addMatchedGroup(structInfo.getTopPlan().getGroupExpression().get().getOwnerGroup().getGroupId(), + false); } // once success, do not record the fail reason if (this.success) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewAggregateRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewAggregateRule.java index 8e0f8d6f717..20a7965f08b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewAggregateRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewAggregateRule.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import com.google.common.collect.ImmutableList; @@ -36,7 +37,7 @@ public class MaterializedViewAggregateRule extends AbstractMaterializedViewAggre @Override public List<Rule> buildRules() { return ImmutableList.of( - logicalAggregate(any()).thenApplyMultiNoThrow(ctx -> { + logicalAggregate(any().when(LogicalPlan.class::isInstance)).thenApplyMultiNoThrow(ctx -> { LogicalAggregate<Plan> root = ctx.root; return rewrite(root, ctx.cascadesContext); }).toRule(RuleType.MATERIALIZED_VIEW_ONLY_AGGREGATE)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewFilterAggregateRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewFilterAggregateRule.java index 40c8b6d681b..10b7aa5e84d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewFilterAggregateRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewFilterAggregateRule.java @@ -22,6 +22,7 @@ import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import com.google.common.collect.ImmutableList; @@ -37,9 +38,10 @@ public class MaterializedViewFilterAggregateRule extends AbstractMaterializedVie @Override public List<Rule> buildRules() { return ImmutableList.of( - logicalFilter(logicalAggregate(any())).thenApplyMultiNoThrow(ctx -> { - LogicalFilter<LogicalAggregate<Plan>> root = ctx.root; - return rewrite(root, ctx.cascadesContext); - }).toRule(RuleType.MATERIALIZED_VIEW_FILTER_AGGREGATE)); + logicalFilter(logicalAggregate(any().when(LogicalPlan.class::isInstance))).thenApplyMultiNoThrow( + ctx -> { + LogicalFilter<LogicalAggregate<Plan>> root = ctx.root; + return rewrite(root, ctx.cascadesContext); + }).toRule(RuleType.MATERIALIZED_VIEW_FILTER_AGGREGATE)); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewFilterJoinRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewFilterJoinRule.java index d1ffa522bb0..e7b12e6bb1f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewFilterJoinRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewFilterJoinRule.java @@ -22,6 +22,7 @@ import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import com.google.common.collect.ImmutableList; @@ -36,8 +37,8 @@ public class MaterializedViewFilterJoinRule extends AbstractMaterializedViewJoin @Override public List<Rule> buildRules() { - return ImmutableList.of( - logicalFilter(logicalJoin(any(), any())).thenApplyMultiNoThrow(ctx -> { + return ImmutableList.of(logicalFilter(logicalJoin(any().when(LogicalPlan.class::isInstance), + any().when(LogicalPlan.class::isInstance))).thenApplyMultiNoThrow(ctx -> { LogicalFilter<LogicalJoin<Plan, Plan>> root = ctx.root; return rewrite(root, ctx.cascadesContext); }).toRule(RuleType.MATERIALIZED_VIEW_FILTER_JOIN)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewFilterProjectAggregateRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewFilterProjectAggregateRule.java index 40655e97394..bd7437a7d90 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewFilterProjectAggregateRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewFilterProjectAggregateRule.java @@ -22,6 +22,7 @@ import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import com.google.common.collect.ImmutableList; @@ -39,9 +40,10 @@ public class MaterializedViewFilterProjectAggregateRule extends AbstractMaterial @Override public List<Rule> buildRules() { return ImmutableList.of( - logicalFilter(logicalProject(logicalAggregate(any()))).thenApplyMultiNoThrow(ctx -> { - LogicalFilter<LogicalProject<LogicalAggregate<Plan>>> root = ctx.root; - return rewrite(root, ctx.cascadesContext); - }).toRule(RuleType.MATERIALIZED_VIEW_FILTER_AGGREGATE)); + logicalFilter(logicalProject(logicalAggregate(any().when(LogicalPlan.class::isInstance)))) + .thenApplyMultiNoThrow(ctx -> { + LogicalFilter<LogicalProject<LogicalAggregate<Plan>>> root = ctx.root; + return rewrite(root, ctx.cascadesContext); + }).toRule(RuleType.MATERIALIZED_VIEW_FILTER_AGGREGATE)); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewFilterProjectJoinRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewFilterProjectJoinRule.java index 2c9cca199de..c18347dce08 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewFilterProjectJoinRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewFilterProjectJoinRule.java @@ -22,6 +22,7 @@ import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import com.google.common.collect.ImmutableList; @@ -38,9 +39,10 @@ public class MaterializedViewFilterProjectJoinRule extends AbstractMaterializedV @Override public List<Rule> buildRules() { return ImmutableList.of( - logicalFilter(logicalProject(logicalJoin(any(), any()))).thenApplyMultiNoThrow(ctx -> { - LogicalFilter<LogicalProject<LogicalJoin<Plan, Plan>>> root = ctx.root; - return rewrite(root, ctx.cascadesContext); - }).toRule(RuleType.MATERIALIZED_VIEW_FILTER_PROJECT_JOIN)); + logicalFilter(logicalProject(logicalJoin(any().when(LogicalPlan.class::isInstance), + any().when(LogicalPlan.class::isInstance)))).thenApplyMultiNoThrow(ctx -> { + LogicalFilter<LogicalProject<LogicalJoin<Plan, Plan>>> root = ctx.root; + return rewrite(root, ctx.cascadesContext); + }).toRule(RuleType.MATERIALIZED_VIEW_FILTER_PROJECT_JOIN)); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewOnlyJoinRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewOnlyJoinRule.java index a9f8333f4df..2735ca87fe9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewOnlyJoinRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewOnlyJoinRule.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import com.google.common.collect.ImmutableList; @@ -35,8 +36,8 @@ public class MaterializedViewOnlyJoinRule extends AbstractMaterializedViewJoinRu @Override public List<Rule> buildRules() { - return ImmutableList.of( - logicalJoin(any(), any()).thenApplyMultiNoThrow(ctx -> { + return ImmutableList.of(logicalJoin(any().when(LogicalPlan.class::isInstance), + any().when(LogicalPlan.class::isInstance)).thenApplyMultiNoThrow(ctx -> { LogicalJoin<Plan, Plan> root = ctx.root; return rewrite(root, ctx.cascadesContext); }).toRule(RuleType.MATERIALIZED_VIEW_ONLY_JOIN)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectAggregateRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectAggregateRule.java index 106de2a7f9c..d857d4eab0b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectAggregateRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectAggregateRule.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import com.google.common.collect.ImmutableList; @@ -35,9 +36,10 @@ public class MaterializedViewProjectAggregateRule extends AbstractMaterializedVi @Override public List<Rule> buildRules() { return ImmutableList.of( - logicalProject(logicalAggregate(any())).thenApplyMultiNoThrow(ctx -> { - LogicalProject<LogicalAggregate<Plan>> root = ctx.root; - return rewrite(root, ctx.cascadesContext); - }).toRule(RuleType.MATERIALIZED_VIEW_PROJECT_AGGREGATE)); + logicalProject(logicalAggregate(any().when(LogicalPlan.class::isInstance))).thenApplyMultiNoThrow( + ctx -> { + LogicalProject<LogicalAggregate<Plan>> root = ctx.root; + return rewrite(root, ctx.cascadesContext); + }).toRule(RuleType.MATERIALIZED_VIEW_PROJECT_AGGREGATE)); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectFilterAggregateRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectFilterAggregateRule.java index b841862d5b2..906ca31b4c2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectFilterAggregateRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectFilterAggregateRule.java @@ -22,6 +22,7 @@ import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import com.google.common.collect.ImmutableList; @@ -38,8 +39,8 @@ public class MaterializedViewProjectFilterAggregateRule extends AbstractMaterial @Override public List<Rule> buildRules() { - return ImmutableList.of( - logicalProject(logicalFilter(logicalAggregate(any()))).thenApplyMultiNoThrow(ctx -> { + return ImmutableList.of(logicalProject(logicalFilter(logicalAggregate( + any().when(LogicalPlan.class::isInstance)))).thenApplyMultiNoThrow(ctx -> { LogicalProject<LogicalFilter<LogicalAggregate<Plan>>> root = ctx.root; return rewrite(root, ctx.cascadesContext); }).toRule(RuleType.MATERIALIZED_VIEW_FILTER_AGGREGATE)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectFilterJoinRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectFilterJoinRule.java index e97a5378b83..d82f838ea6b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectFilterJoinRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectFilterJoinRule.java @@ -22,6 +22,7 @@ import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import com.google.common.collect.ImmutableList; @@ -38,9 +39,10 @@ public class MaterializedViewProjectFilterJoinRule extends AbstractMaterializedV @Override public List<Rule> buildRules() { return ImmutableList.of( - logicalProject(logicalFilter(logicalJoin(any(), any()))).thenApplyMultiNoThrow(ctx -> { - LogicalProject<LogicalFilter<LogicalJoin<Plan, Plan>>> root = ctx.root; - return rewrite(root, ctx.cascadesContext); - }).toRule(RuleType.MATERIALIZED_VIEW_PROJECT_FILTER_JOIN)); + logicalProject(logicalFilter(logicalJoin(any().when(LogicalPlan.class::isInstance), + any().when(LogicalPlan.class::isInstance)))).thenApplyMultiNoThrow(ctx -> { + LogicalProject<LogicalFilter<LogicalJoin<Plan, Plan>>> root = ctx.root; + return rewrite(root, ctx.cascadesContext); + }).toRule(RuleType.MATERIALIZED_VIEW_PROJECT_FILTER_JOIN)); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectJoinRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectJoinRule.java index c9e309c0096..bbeda87191a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectJoinRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectJoinRule.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import com.google.common.collect.ImmutableList; @@ -37,9 +38,10 @@ public class MaterializedViewProjectJoinRule extends AbstractMaterializedViewJoi @Override public List<Rule> buildRules() { return ImmutableList.of( - logicalProject(logicalJoin(any(), any())).thenApplyMultiNoThrow(ctx -> { - LogicalProject<LogicalJoin<Plan, Plan>> root = ctx.root; - return rewrite(root, ctx.cascadesContext); - }).toRule(RuleType.MATERIALIZED_VIEW_PROJECT_JOIN)); + logicalProject(logicalJoin(any().when(LogicalPlan.class::isInstance), + any().when(LogicalPlan.class::isInstance))).thenApplyMultiNoThrow(ctx -> { + LogicalProject<LogicalJoin<Plan, Plan>> root = ctx.root; + return rewrite(root, ctx.cascadesContext); + }).toRule(RuleType.MATERIALIZED_VIEW_PROJECT_JOIN)); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java index ce12e059b77..ac4b9ccad38 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java @@ -18,12 +18,15 @@ package org.apache.doris.nereids.rules.exploration.mv; import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.MTMV; import org.apache.doris.catalog.PartitionType; import org.apache.doris.catalog.TableIf; import org.apache.doris.common.Pair; import org.apache.doris.mtmv.BaseTableInfo; import org.apache.doris.mtmv.MTMVRelatedTableIf; import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.memo.Group; +import org.apache.doris.nereids.memo.StructInfoMap; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; @@ -31,6 +34,7 @@ import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.WindowExpression; import org.apache.doris.nereids.trees.plans.JoinType; import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.PreAggStatus; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; import org.apache.doris.nereids.trees.plans.logical.LogicalCatalogRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan; @@ -44,9 +48,12 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalWindow; import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanVisitor; import com.google.common.collect.HashMultimap; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; import com.google.common.collect.Multimap; import java.util.ArrayList; +import java.util.BitSet; import java.util.HashSet; import java.util.List; import java.util.Optional; @@ -132,18 +139,46 @@ public class MaterializedViewUtils { /** * Extract struct info from plan, support to get struct info from logical plan or plan in group. */ - public static List<StructInfo> extractStructInfo(Plan plan, CascadesContext cascadesContext) { - if (plan.getGroupExpression().isPresent() && !plan.getGroupExpression().get().getOwnerGroup().getStructInfos() - .isEmpty()) { - return plan.getGroupExpression().get().getOwnerGroup().getStructInfos(); - } else { - // build struct info and add them to current group - List<StructInfo> structInfos = StructInfo.of(plan); - if (plan.getGroupExpression().isPresent()) { - plan.getGroupExpression().get().getOwnerGroup().addStructInfo(structInfos); + public static List<StructInfo> extractStructInfo(Plan plan, CascadesContext cascadesContext, + BitSet materializedViewTableSet) { + // If plan belong to some group, construct it with group struct info + if (plan.getGroupExpression().isPresent()) { + Group ownerGroup = plan.getGroupExpression().get().getOwnerGroup(); + StructInfoMap structInfoMap = ownerGroup.getstructInfoMap(); + structInfoMap.refresh(ownerGroup); + Set<BitSet> queryTableSets = structInfoMap.getTableMaps(); + if (!queryTableSets.isEmpty()) { + return queryTableSets.stream() + // Just construct the struct info which mv table set contains all the query table set + .filter(queryTableSet -> materializedViewTableSet.isEmpty() + || StructInfo.containsAll(materializedViewTableSet, queryTableSet)) + .map(tableMap -> structInfoMap.getStructInfo(tableMap, tableMap, ownerGroup, plan)) + .collect(Collectors.toList()); } - return structInfos; } + // if plan doesn't belong to any group, construct it directly + return ImmutableList.of(StructInfo.of(plan)); + } + + /** + * Generate scan plan for materialized view + * if MaterializationContext is already rewritten by materialized view, then should generate in real time + * when query rewrite, because one plan may hit the materialized view repeatedly and the mv scan output + * should be different + */ + public static Plan generateMvScanPlan(MTMV materializedView, CascadesContext cascadesContext) { + LogicalOlapScan mvScan = new LogicalOlapScan( + cascadesContext.getStatementContext().getNextRelationId(), + materializedView, + ImmutableList.of(materializedView.getQualifiedDbName()), + // this must be empty, or it will be used to sample + Lists.newArrayList(), + Lists.newArrayList(), + Optional.empty()); + mvScan = mvScan.withMaterializedIndexSelected(PreAggStatus.on(), materializedView.getBaseIndexId()); + List<NamedExpression> mvProjects = mvScan.getOutput().stream().map(NamedExpression.class::cast) + .collect(Collectors.toList()); + return new LogicalProject<Plan>(mvProjects, mvScan); } private static final class TableQueryOperatorChecker extends DefaultPlanVisitor<Boolean, Void> { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java index 0d280bb8340..604e7853d48 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java @@ -47,12 +47,12 @@ import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanVisitor; import org.apache.doris.nereids.trees.plans.visitor.ExpressionLineageReplacer; import org.apache.doris.nereids.util.ExpressionUtils; -import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; import com.google.common.collect.Sets; import java.util.ArrayList; +import java.util.BitSet; import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; @@ -83,6 +83,7 @@ public class StructInfo { // bottom plan which top plan only contain join or scan. this is needed by hyper graph private final Plan bottomPlan; private final List<CatalogRelation> relations; + private final BitSet tableBitSet = new BitSet(); // this is for LogicalCompatibilityContext later private final Map<RelationId, StructInfoNode> relationIdStructInfoNodeMap; // this recorde the predicates which can pull up, not shuttled @@ -112,6 +113,7 @@ public class StructInfo { this.topPlan = topPlan; this.bottomPlan = bottomPlan; this.relations = relations; + relations.forEach(relation -> this.tableBitSet.set((int) (relation.getTable().getId()))); this.relationIdStructInfoNodeMap = relationIdStructInfoNodeMap; this.predicates = predicates; if (predicates == null) { @@ -120,7 +122,8 @@ public class StructInfo { topPlan.accept(PREDICATE_COLLECTOR, topPlanPredicates); this.predicates = Predicates.of(topPlanPredicates); } - Pair<SplitPredicate, EquivalenceClass> derivedPredicates = predicatesDerive(this.predicates, originalPlan); + Pair<SplitPredicate, EquivalenceClass> derivedPredicates = + predicatesDerive(this.predicates, topPlan, tableBitSet); this.splitPredicate = derivedPredicates.key(); this.equivalenceClass = derivedPredicates.value(); this.shuttledHashConjunctsToConjunctsMap = shuttledHashConjunctsToConjunctsMap; @@ -140,8 +143,24 @@ public class StructInfo { Plan topPlan, Map<Expression, Expression> shuttledHashConjunctsToConjunctsMap, Map<ExprId, Expression> namedExprIdAndExprMapping, - ImmutableList.Builder<CatalogRelation> relationBuilder, + List<CatalogRelation> relations, Map<RelationId, StructInfoNode> relationIdStructInfoNodeMap) { + + // Collect relations from hyper graph which in the bottom plan firstly + BitSet hyperTableBitSet = new BitSet(); + hyperGraph.getNodes().forEach(node -> { + // plan relation collector and set to map + Plan nodePlan = node.getPlan(); + List<CatalogRelation> nodeRelations = new ArrayList<>(); + nodePlan.accept(RELATION_COLLECTOR, nodeRelations); + relations.addAll(nodeRelations); + nodeRelations.forEach(relation -> hyperTableBitSet.set((int) relation.getTable().getId())); + // every node should only have one relation, this is for LogicalCompatibilityContext + if (!nodeRelations.isEmpty()) { + relationIdStructInfoNodeMap.put(nodeRelations.get(0).getRelationId(), (StructInfoNode) node); + } + }); + // Collect expression from join condition in hyper graph for (JoinEdge edge : hyperGraph.getJoinEdges()) { List<Expression> hashJoinConjuncts = edge.getHashJoinConjuncts(); @@ -151,9 +170,8 @@ public class StructInfo { hashJoinConjuncts.forEach(conjunctExpr -> { ExpressionLineageReplacer.ExpressionReplaceContext replaceContext = new ExpressionLineageReplacer.ExpressionReplaceContext( - Lists.newArrayList(conjunctExpr), - ImmutableSet.of(), - ImmutableSet.of()); + Lists.newArrayList(conjunctExpr), ImmutableSet.of(), + ImmutableSet.of(), hyperTableBitSet); topPlan.accept(ExpressionLineageReplacer.INSTANCE, replaceContext); // Replace expressions by expression map List<Expression> replacedExpressions = replaceContext.getReplacedExpressions(); @@ -167,27 +185,17 @@ public class StructInfo { return false; } } - // Collect relations from hyper graph which in the bottom plan + // Record expressions in node hyperGraph.getNodes().forEach(node -> { // plan relation collector and set to map StructInfoNode structInfoNode = (StructInfoNode) node; - // plan relation collector and set to map - Plan nodePlan = node.getPlan(); - List<CatalogRelation> nodeRelations = new ArrayList<>(); - nodePlan.accept(RELATION_COLLECTOR, nodeRelations); - relationBuilder.addAll(nodeRelations); - // every node should only have one relation, this is for LogicalCompatibilityContext - if (!nodeRelations.isEmpty()) { - relationIdStructInfoNodeMap.put(nodeRelations.get(0).getRelationId(), (StructInfoNode) node); - } // record expressions in node if (structInfoNode.getExpressions() != null) { structInfoNode.getExpressions().forEach(expression -> { ExpressionLineageReplacer.ExpressionReplaceContext replaceContext = new ExpressionLineageReplacer.ExpressionReplaceContext( - Lists.newArrayList(expression), - ImmutableSet.of(), - ImmutableSet.of()); + Lists.newArrayList(expression), ImmutableSet.of(), + ImmutableSet.of(), hyperTableBitSet); structInfoNode.getPlan().accept(ExpressionLineageReplacer.INSTANCE, replaceContext); // Replace expressions by expression map List<Expression> replacedExpressions = replaceContext.getReplacedExpressions(); @@ -204,18 +212,19 @@ public class StructInfo { filterExpressions.forEach(predicate -> { // this is used for LogicalCompatibilityContext ExpressionUtils.extractConjunction(predicate).forEach(expr -> - shuttledHashConjunctsToConjunctsMap.put( - ExpressionUtils.shuttleExpressionWithLineage(predicate, topPlan), predicate)); + shuttledHashConjunctsToConjunctsMap.put(ExpressionUtils.shuttleExpressionWithLineage( + predicate, topPlan, hyperTableBitSet), predicate)); }); }); return true; } // derive some useful predicate by predicates - private Pair<SplitPredicate, EquivalenceClass> predicatesDerive(Predicates predicates, Plan originalPlan) { + private Pair<SplitPredicate, EquivalenceClass> predicatesDerive(Predicates predicates, Plan originalPlan, + BitSet tableBitSet) { // construct equivalenceClass according to equals predicates List<Expression> shuttledExpression = ExpressionUtils.shuttleExpressionWithLineage( - new ArrayList<>(predicates.getPulledUpPredicates()), originalPlan).stream() + new ArrayList<>(predicates.getPulledUpPredicates()), originalPlan, tableBitSet).stream() .map(Expression.class::cast) .collect(Collectors.toList()); SplitPredicate splitPredicate = Predicates.splitPredicates(ExpressionUtils.and(shuttledExpression)); @@ -238,20 +247,23 @@ public class StructInfo { * Build Struct info from plan. * Maybe return multi structInfo when original plan already be rewritten by mv */ - public static List<StructInfo> of(Plan originalPlan) { - // TODO only consider the inner join currently, Should support outer join + public static StructInfo of(Plan originalPlan) { + return of(originalPlan, originalPlan); + } + + /** + * Build Struct info from plan. + * Maybe return multi structInfo when original plan already be rewritten by mv + */ + public static StructInfo of(Plan derivedPlan, Plan originalPlan) { // Split plan by the boundary which contains multi child LinkedHashSet<Class<? extends Plan>> set = Sets.newLinkedHashSet(); set.add(LogicalJoin.class); PlanSplitContext planSplitContext = new PlanSplitContext(set); // if single table without join, the bottom is - originalPlan.accept(PLAN_SPLITTER, planSplitContext); - - List<HyperGraph> structInfos = HyperGraph.builderForMv(planSplitContext.getBottomPlan()).buildAll(); - return structInfos.stream() - .map(hyperGraph -> StructInfo.of(originalPlan, planSplitContext.getTopPlan(), - planSplitContext.getBottomPlan(), hyperGraph)) - .collect(Collectors.toList()); + derivedPlan.accept(PLAN_SPLITTER, planSplitContext); + return StructInfo.of(originalPlan, planSplitContext.getTopPlan(), planSplitContext.getBottomPlan(), + HyperGraph.builderForMv(planSplitContext.getBottomPlan()).build()); } /** @@ -271,16 +283,16 @@ public class StructInfo { topPlan = planSplitContext.getTopPlan(); } // collect struct info fromGraph - ImmutableList.Builder<CatalogRelation> relationBuilder = ImmutableList.builder(); + List<CatalogRelation> relationList = new ArrayList<>(); Map<RelationId, StructInfoNode> relationIdStructInfoNodeMap = new LinkedHashMap<>(); Map<Expression, Expression> shuttledHashConjunctsToConjunctsMap = new LinkedHashMap<>(); Map<ExprId, Expression> namedExprIdAndExprMapping = new LinkedHashMap<>(); boolean valid = collectStructInfoFromGraph(hyperGraph, topPlan, shuttledHashConjunctsToConjunctsMap, namedExprIdAndExprMapping, - relationBuilder, + relationList, relationIdStructInfoNodeMap); return new StructInfo(originalPlan, originalPlanId, hyperGraph, valid, topPlan, bottomPlan, - relationBuilder.build(), relationIdStructInfoNodeMap, null, shuttledHashConjunctsToConjunctsMap, + relationList, relationIdStructInfoNodeMap, null, shuttledHashConjunctsToConjunctsMap, namedExprIdAndExprMapping); } @@ -338,8 +350,8 @@ public class StructInfo { } public List<? extends Expression> getExpressions() { - return originalPlan instanceof LogicalProject - ? ((LogicalProject<Plan>) originalPlan).getProjects() : originalPlan.getOutput(); + return topPlan instanceof LogicalProject + ? ((LogicalProject<Plan>) topPlan).getProjects() : topPlan.getOutput(); } public ObjectId getOriginalPlanId() { @@ -350,6 +362,10 @@ public class StructInfo { return namedExprIdAndExprMapping; } + public BitSet getTableBitSet() { + return tableBitSet; + } + /** * Judge the source graph logical is whether the same as target * For inner join should judge only the join tables, @@ -361,6 +377,11 @@ public class StructInfo { .isLogicCompatible(queryStructInfo.hyperGraph, viewStructInfo.hyperGraph, compatibilityContext); } + @Override + public String toString() { + return "StructInfo{ originalPlanId = " + originalPlanId + ", relations = " + relations + '}'; + } + private static class RelationCollector extends DefaultPlanVisitor<Void, List<CatalogRelation>> { @Override public Void visit(Plan plan, List<CatalogRelation> collectedRelations) { @@ -409,6 +430,12 @@ public class StructInfo { } } + public static boolean containsAll(BitSet source, BitSet target) { + BitSet intersection = (BitSet) source.clone(); + intersection.and(target); + return intersection.equals(target); + } + /** * Plan split context, this hold bottom and top plan, and boundary plan setting */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/ExpressionLineageReplacer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/ExpressionLineageReplacer.java index b7cc0dff80b..2060718ec13 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/ExpressionLineageReplacer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/ExpressionLineageReplacer.java @@ -32,9 +32,11 @@ import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.visitor.ExpressionLineageReplacer.ExpressionReplaceContext; import java.util.ArrayList; +import java.util.BitSet; import java.util.Collection; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -43,12 +45,12 @@ import java.util.stream.Collectors; * Get from rewrite plan and can also get from plan struct info, if from plan struct info it depends on * the nodes from graph. */ -public class ExpressionLineageReplacer extends DefaultPlanVisitor<Expression, ExpressionReplaceContext> { +public class ExpressionLineageReplacer extends DefaultPlanVisitor<Void, ExpressionReplaceContext> { public static final ExpressionLineageReplacer INSTANCE = new ExpressionLineageReplacer(); @Override - public Expression visit(Plan plan, ExpressionReplaceContext context) { + public Void visit(Plan plan, ExpressionReplaceContext context) { List<? extends Expression> expressions = plan.getExpressions(); Map<ExprId, Expression> targetExpressionMap = context.getExprIdExpressionMap(); // Filter the namedExpression used by target and collect the namedExpression @@ -60,19 +62,27 @@ public class ExpressionLineageReplacer extends DefaultPlanVisitor<Expression, Ex } @Override - public Expression visitGroupPlan(GroupPlan groupPlan, ExpressionReplaceContext context) { + public Void visitGroupPlan(GroupPlan groupPlan, ExpressionReplaceContext context) { Group group = groupPlan.getGroup(); if (group == null) { return visit(groupPlan, context); } - List<StructInfo> structInfos = group.getStructInfos(); + Collection<StructInfo> structInfos = group.getstructInfoMap().getStructInfos(); if (structInfos.isEmpty()) { return visit(groupPlan, context); } - // TODO only support group has one struct info, will support more struct info later - StructInfo structInfo = structInfos.get(0); - context.getExprIdExpressionMap().putAll(structInfo.getNamedExprIdAndExprMapping()); - return visit(groupPlan, context); + // Find first info which the context's bitmap contains all to make sure that + // the expression lineage is correct + Optional<StructInfo> structInfoOptional = structInfos.stream() + .filter(info -> (context.getTableBitSet().isEmpty() + || StructInfo.containsAll(context.getTableBitSet(), info.getTableBitSet())) + && !info.getNamedExprIdAndExprMapping().isEmpty()) + .findFirst(); + if (!structInfoOptional.isPresent()) { + return visit(groupPlan, context); + } + context.getExprIdExpressionMap().putAll(structInfoOptional.get().getNamedExprIdAndExprMapping()); + return null; } /** @@ -144,7 +154,8 @@ public class ExpressionLineageReplacer extends DefaultPlanVisitor<Expression, Ex private final List<Expression> targetExpressions; private final Set<TableType> targetTypes; private final Set<String> tableIdentifiers; - private Map<ExprId, Expression> exprIdExpressionMap; + private final Map<ExprId, Expression> exprIdExpressionMap; + private final BitSet tableBitSet; private List<Expression> replacedExpressions; /** @@ -152,10 +163,12 @@ public class ExpressionLineageReplacer extends DefaultPlanVisitor<Expression, Ex */ public ExpressionReplaceContext(List<Expression> targetExpressions, Set<TableType> targetTypes, - Set<String> tableIdentifiers) { + Set<String> tableIdentifiers, + BitSet tableBitSet) { this.targetExpressions = targetExpressions; this.targetTypes = targetTypes; this.tableIdentifiers = tableIdentifiers; + this.tableBitSet = tableBitSet; // collect the named expressions used in target expression and will be replaced later this.exprIdExpressionMap = targetExpressions.stream() .map(each -> each.collectToList(NamedExpression.class::isInstance)) @@ -181,6 +194,10 @@ public class ExpressionLineageReplacer extends DefaultPlanVisitor<Expression, Ex return exprIdExpressionMap; } + public BitSet getTableBitSet() { + return tableBitSet; + } + /** * getReplacedExpressions */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java index e5e79f7b725..51eef31de2f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java @@ -65,6 +65,7 @@ import com.google.common.collect.Maps; import com.google.common.collect.Sets; import java.util.Arrays; +import java.util.BitSet; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; @@ -225,14 +226,14 @@ public class ExpressionUtils { return result; } - public static Expression shuttleExpressionWithLineage(Expression expression, Plan plan) { + public static Expression shuttleExpressionWithLineage(Expression expression, Plan plan, BitSet tableBitSet) { return shuttleExpressionWithLineage(Lists.newArrayList(expression), - plan, ImmutableSet.of(), ImmutableSet.of()).get(0); + plan, ImmutableSet.of(), ImmutableSet.of(), tableBitSet).get(0); } public static List<? extends Expression> shuttleExpressionWithLineage(List<? extends Expression> expressions, - Plan plan) { - return shuttleExpressionWithLineage(expressions, plan, ImmutableSet.of(), ImmutableSet.of()); + Plan plan, BitSet tableBitSet) { + return shuttleExpressionWithLineage(expressions, plan, ImmutableSet.of(), ImmutableSet.of(), tableBitSet); } /** @@ -247,7 +248,8 @@ public class ExpressionUtils { public static List<? extends Expression> shuttleExpressionWithLineage(List<? extends Expression> expressions, Plan plan, Set<TableType> targetTypes, - Set<String> tableIdentifiers) { + Set<String> tableIdentifiers, + BitSet tableBitSet) { if (expressions.isEmpty()) { return ImmutableList.of(); } @@ -255,7 +257,8 @@ public class ExpressionUtils { new ExpressionLineageReplacer.ExpressionReplaceContext( expressions.stream().map(Expression.class::cast).collect(Collectors.toList()), targetTypes, - tableIdentifiers); + tableIdentifiers, + tableBitSet); plan.accept(ExpressionLineageReplacer.INSTANCE, replaceContext); // Replace expressions by expression map diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/CompareOuterJoinTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/CompareOuterJoinTest.java index afdce7ca08d..9ebb04e6ba3 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/CompareOuterJoinTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/CompareOuterJoinTest.java @@ -37,6 +37,7 @@ import com.google.common.collect.Sets; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import java.util.BitSet; import java.util.List; class CompareOuterJoinTest extends SqlTestBase { @@ -63,8 +64,8 @@ class CompareOuterJoinTest extends SqlTestBase { .rewrite() .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) .getAllPlan().get(0).child(0); - HyperGraph h1 = HyperGraph.builderForMv(p1).buildAll().get(0); - HyperGraph h2 = HyperGraph.builderForMv(p2).buildAll().get(0); + HyperGraph h1 = HyperGraph.builderForMv(p1).build(); + HyperGraph h2 = HyperGraph.builderForMv(p2).build(); Assertions.assertFalse( HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2)).isInvalid()); } @@ -82,8 +83,8 @@ class CompareOuterJoinTest extends SqlTestBase { .rewrite() .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) .getAllPlan().get(0); - HyperGraph h1 = HyperGraph.builderForMv(p1).buildAll().get(0); - HyperGraph h2 = HyperGraph.builderForMv(p2).buildAll().get(0); + HyperGraph h1 = HyperGraph.builderForMv(p1).build(); + HyperGraph h2 = HyperGraph.builderForMv(p2).build(); Assertions.assertFalse( HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2)).isInvalid()); } @@ -108,8 +109,8 @@ class CompareOuterJoinTest extends SqlTestBase { .rewrite() .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) .getAllPlan().get(0).child(0); - HyperGraph h1 = HyperGraph.builderForMv(p1).buildAll().get(0); - HyperGraph h2 = HyperGraph.builderForMv(p2).buildAll().get(0); + HyperGraph h1 = HyperGraph.builderForMv(p1).build(); + HyperGraph h2 = HyperGraph.builderForMv(p2).build(); ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2)); Assertions.assertEquals(1, res.getQueryExpressions().size()); Assertions.assertEquals("(id = 0)", res.getQueryExpressions().get(0).toSql()); @@ -135,8 +136,8 @@ class CompareOuterJoinTest extends SqlTestBase { .rewrite() .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) .getAllPlan().get(0).child(0); - HyperGraph h1 = HyperGraph.builderForMv(p1).buildAll().get(0); - HyperGraph h2 = HyperGraph.builderForMv(p2).buildAll().get(0); + HyperGraph h1 = HyperGraph.builderForMv(p1).build(); + HyperGraph h2 = HyperGraph.builderForMv(p2).build(); List<Expression> exprList = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2)).getQueryExpressions(); Assertions.assertEquals(0, exprList.size()); } @@ -162,8 +163,8 @@ class CompareOuterJoinTest extends SqlTestBase { .rewrite() .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) .getAllPlan().get(0).child(0); - HyperGraph h1 = HyperGraph.builderForMv(p1).buildAll().get(0); - HyperGraph h2 = HyperGraph.builderForMv(p2).buildAll().get(0); + HyperGraph h1 = HyperGraph.builderForMv(p1).build(); + HyperGraph h2 = HyperGraph.builderForMv(p2).build(); ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2)); Assertions.assertEquals(1, res.getQueryExpressions().size()); Assertions.assertEquals("(id = 0)", res.getQueryExpressions().get(0).toSql()); @@ -190,17 +191,17 @@ class CompareOuterJoinTest extends SqlTestBase { .rewrite() .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) .getAllPlan().get(0).child(0); - HyperGraph h1 = HyperGraph.builderForMv(p1).buildAll().get(0); - HyperGraph h2 = HyperGraph.builderForMv(p2).buildAll().get(0); + HyperGraph h1 = HyperGraph.builderForMv(p1).build(); + HyperGraph h2 = HyperGraph.builderForMv(p2).build(); ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2)); Assertions.assertTrue(res.isInvalid()); } LogicalCompatibilityContext constructContext(Plan p1, Plan p2) { StructInfo st1 = MaterializedViewUtils.extractStructInfo(p1, - null).get(0); + null, new BitSet()).get(0); StructInfo st2 = MaterializedViewUtils.extractStructInfo(p2, - null).get(0); + null, new BitSet()).get(0); RelationMapping rm = RelationMapping.generate(st1.getRelations(), st2.getRelations()).get(0); SlotMapping sm = SlotMapping.generate(rm); return LogicalCompatibilityContext.from(rm, sm, st1, st2); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/InferJoinTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/InferJoinTest.java index 05f56c4de20..d61224c1433 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/InferJoinTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/InferJoinTest.java @@ -35,6 +35,7 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; +import java.util.BitSet; import java.util.stream.Collectors; class InferJoinTest extends SqlTestBase { @@ -58,8 +59,8 @@ class InferJoinTest extends SqlTestBase { .rewrite() .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) .getAllPlan().get(0).child(0); - HyperGraph h1 = HyperGraph.builderForMv(p1).buildAll().get(0); - HyperGraph h2 = HyperGraph.builderForMv(p2).buildAll().get(0); + HyperGraph h1 = HyperGraph.builderForMv(p1).build(); + HyperGraph h2 = HyperGraph.builderForMv(p2).build(); ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2)); Assertions.assertFalse(res.isInvalid()); Assertions.assertEquals(1, res.getViewNoNullableSlot().size()); @@ -87,8 +88,8 @@ class InferJoinTest extends SqlTestBase { .rewrite() .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) .getAllPlan().get(0).child(0); - HyperGraph h1 = HyperGraph.builderForMv(p1).buildAll().get(0); - HyperGraph h2 = HyperGraph.builderForMv(p2).buildAll().get(0); + HyperGraph h1 = HyperGraph.builderForMv(p1).build(); + HyperGraph h2 = HyperGraph.builderForMv(p2).build(); ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2)); Assertions.assertFalse(res.isInvalid()); Assertions.assertEquals(1, res.getViewNoNullableSlot().size()); @@ -124,8 +125,8 @@ class InferJoinTest extends SqlTestBase { .rewrite() .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) .getAllPlan().get(0).child(0); - HyperGraph h1 = HyperGraph.builderForMv(p1).buildAll().get(0); - HyperGraph h2 = HyperGraph.builderForMv(p2).buildAll().get(0); + HyperGraph h1 = HyperGraph.builderForMv(p1).build(); + HyperGraph h2 = HyperGraph.builderForMv(p2).build(); ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2)); Assertions.assertFalse(res.isInvalid()); Assertions.assertEquals(1, res.getViewNoNullableSlot().size()); @@ -155,17 +156,17 @@ class InferJoinTest extends SqlTestBase { .rewrite() .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) .getAllPlan().get(0).child(0); - HyperGraph h1 = HyperGraph.builderForMv(p1).buildAll().get(0); - HyperGraph h2 = HyperGraph.builderForMv(p2).buildAll().get(0); + HyperGraph h1 = HyperGraph.builderForMv(p1).build(); + HyperGraph h2 = HyperGraph.builderForMv(p2).build(); ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2)); Assertions.assertTrue(res.isInvalid()); } LogicalCompatibilityContext constructContext(Plan p1, Plan p2) { StructInfo st1 = MaterializedViewUtils.extractStructInfo(p1, - null).get(0); + null, new BitSet()).get(0); StructInfo st2 = MaterializedViewUtils.extractStructInfo(p2, - null).get(0); + null, new BitSet()).get(0); RelationMapping rm = RelationMapping.generate(st1.getRelations(), st2.getRelations()).get(0); SlotMapping sm = SlotMapping.generate(rm); return LogicalCompatibilityContext.from(rm, sm, st1, st2); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/InferPredicateTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/InferPredicateTest.java index 8bb1ede8048..fe705fda72c 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/InferPredicateTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/InferPredicateTest.java @@ -33,6 +33,8 @@ import org.apache.doris.nereids.util.PlanChecker; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import java.util.BitSet; + class InferPredicateTest extends SqlTestBase { @Test void testPullUpQueryFilter() { @@ -53,8 +55,8 @@ class InferPredicateTest extends SqlTestBase { .rewrite() .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) .getAllPlan().get(0).child(0); - HyperGraph h1 = HyperGraph.builderForMv(p1).buildAll().get(0); - HyperGraph h2 = HyperGraph.builderForMv(p2).buildAll().get(0); + HyperGraph h1 = HyperGraph.builderForMv(p1).build(); + HyperGraph h2 = HyperGraph.builderForMv(p2).build(); ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2)); Assertions.assertFalse(res.isInvalid()); Assertions.assertEquals("(id = 1)", res.getQueryExpressions().get(0).toSql()); @@ -62,9 +64,9 @@ class InferPredicateTest extends SqlTestBase { LogicalCompatibilityContext constructContext(Plan p1, Plan p2) { StructInfo st1 = MaterializedViewUtils.extractStructInfo(p1, - null).get(0); + null, new BitSet()).get(0); StructInfo st2 = MaterializedViewUtils.extractStructInfo(p2, - null).get(0); + null, new BitSet()).get(0); RelationMapping rm = RelationMapping.generate(st1.getRelations(), st2.getRelations()).get(0); SlotMapping sm = SlotMapping.generate(rm); return LogicalCompatibilityContext.from(rm, sm, st1, st2); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/PullupExpressionTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/PullupExpressionTest.java index 6e65dba4f03..b564f166ed5 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/PullupExpressionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/PullupExpressionTest.java @@ -33,6 +33,8 @@ import org.apache.doris.nereids.util.PlanChecker; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import java.util.BitSet; + class PullupExpressionTest extends SqlTestBase { @Test void testPullUpQueryFilter() { @@ -53,8 +55,8 @@ class PullupExpressionTest extends SqlTestBase { .rewrite() .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) .getAllPlan().get(0).child(0); - HyperGraph h1 = HyperGraph.builderForMv(p1).buildAll().get(0); - HyperGraph h2 = HyperGraph.builderForMv(p2).buildAll().get(0); + HyperGraph h1 = HyperGraph.builderForMv(p1).build(); + HyperGraph h2 = HyperGraph.builderForMv(p2).build(); ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2)); Assertions.assertEquals(1, res.getQueryExpressions().size()); Assertions.assertEquals("(id = 1)", res.getQueryExpressions().get(0).toSql()); @@ -79,8 +81,8 @@ class PullupExpressionTest extends SqlTestBase { .rewrite() .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) .getAllPlan().get(0).child(0); - HyperGraph h1 = HyperGraph.builderForMv(p1).buildAll().get(0); - HyperGraph h2 = HyperGraph.builderForMv(p2).buildAll().get(0); + HyperGraph h1 = HyperGraph.builderForMv(p1).build(); + HyperGraph h2 = HyperGraph.builderForMv(p2).build(); ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2)); Assertions.assertEquals(1, res.getQueryExpressions().size()); Assertions.assertEquals("(score = score)", res.getQueryExpressions().get(0).toSql()); @@ -105,8 +107,8 @@ class PullupExpressionTest extends SqlTestBase { .rewrite() .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) .getAllPlan().get(0).child(0); - HyperGraph h1 = HyperGraph.builderForMv(p1).buildAll().get(0); - HyperGraph h2 = HyperGraph.builderForMv(p2).buildAll().get(0); + HyperGraph h1 = HyperGraph.builderForMv(p1).build(); + HyperGraph h2 = HyperGraph.builderForMv(p2).build(); ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2)); Assertions.assertEquals(2, res.getViewExpressions().size()); Assertions.assertEquals("(id = 1)", res.getViewExpressions().get(0).toSql()); @@ -132,8 +134,8 @@ class PullupExpressionTest extends SqlTestBase { .rewrite() .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) .getAllPlan().get(0).child(0); - HyperGraph h1 = HyperGraph.builderForMv(p1).buildAll().get(0); - HyperGraph h2 = HyperGraph.builderForMv(p2).buildAll().get(0); + HyperGraph h1 = HyperGraph.builderForMv(p1).build(); + HyperGraph h2 = HyperGraph.builderForMv(p2).build(); ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2)); Assertions.assertEquals(1, res.getViewExpressions().size()); Assertions.assertEquals("(score = score)", res.getViewExpressions().get(0).toSql()); @@ -141,9 +143,9 @@ class PullupExpressionTest extends SqlTestBase { LogicalCompatibilityContext constructContext(Plan p1, Plan p2) { StructInfo st1 = MaterializedViewUtils.extractStructInfo(p1, - null).get(0); + null, new BitSet()).get(0); StructInfo st2 = MaterializedViewUtils.extractStructInfo(p2, - null).get(0); + null, new BitSet()).get(0); RelationMapping rm = RelationMapping.generate(st1.getRelations(), st2.getRelations()).get(0); SlotMapping sm = SlotMapping.generate(rm); return LogicalCompatibilityContext.from(rm, sm, st1, st2); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java index c2d9d072ac3..13bdf35252e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java @@ -172,7 +172,7 @@ class StructInfoMapTest extends SqlTestBase { BitSet mvMap = structInfoMap.getTableMaps().stream() .filter(b -> b.cardinality() == 2) .collect(Collectors.toList()).get(0); - StructInfo structInfo = structInfoMap.getStructInfo(mvMap, mvMap, root); + StructInfo structInfo = structInfoMap.getStructInfo(mvMap, mvMap, root, null); System.out.println(structInfo.getOriginalPlan().treeString()); BitSet bitSet = new BitSet(); structInfo.getRelations().forEach(r -> bitSet.set((int) r.getTable().getId())); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/BuildStructInfoTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/BuildStructInfoTest.java index 20324ff8739..a3fc234f983 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/BuildStructInfoTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/BuildStructInfoTest.java @@ -25,8 +25,6 @@ import org.apache.doris.nereids.util.PlanChecker; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import java.util.List; - class BuildStructInfoTest extends SqlTestBase { @Test void testSimpleSQL() { @@ -58,8 +56,8 @@ class BuildStructInfoTest extends SqlTestBase { .deriveStats() .matches(logicalJoin() .when(j -> { - List<HyperGraph> hyperGraph = HyperGraph.builderForMv(j).buildAll(); - Assertions.assertTrue(hyperGraph.get(0).getNodes().stream() + HyperGraph hyperGraph = HyperGraph.builderForMv(j).build(); + Assertions.assertTrue(hyperGraph.getNodes().stream() .allMatch(n -> n.getPlan() .collectToList(GroupPlan.class::isInstance).isEmpty())); return true; @@ -77,7 +75,7 @@ class BuildStructInfoTest extends SqlTestBase { .rewrite() .matches(logicalJoin() .when(j -> { - HyperGraph structInfo = HyperGraph.builderForMv(j).buildAll().get(0); + HyperGraph structInfo = HyperGraph.builderForMv(j).build(); Assertions.assertTrue(structInfo.getJoinEdge(0).getJoinType().isLeftOuterJoin()); Assertions.assertEquals(0, structInfo.getFilterEdge(0).getLeftRejectEdge().size()); Assertions.assertEquals(1, structInfo.getFilterEdge(0).getRightRejectEdge().size()); @@ -91,7 +89,7 @@ class BuildStructInfoTest extends SqlTestBase { .rewrite() .matches(logicalJoin() .when(j -> { - HyperGraph structInfo = HyperGraph.builderForMv(j).buildAll().get(0); + HyperGraph structInfo = HyperGraph.builderForMv(j).build(); Assertions.assertTrue(structInfo.getJoinEdge(0).getJoinType().isLeftOuterJoin()); return true; })); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/EliminateJoinTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/EliminateJoinTest.java index cc8b7147423..08faf589390 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/EliminateJoinTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/EliminateJoinTest.java @@ -30,6 +30,8 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; +import java.util.BitSet; + class EliminateJoinTest extends SqlTestBase { @Test void testLOJWithGroupBy() { @@ -62,9 +64,9 @@ class EliminateJoinTest extends SqlTestBase { .rewrite() .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) .getAllPlan().get(0).child(0); - HyperGraph h1 = HyperGraph.builderForMv(p1).buildAll().get(0); - HyperGraph h2 = HyperGraph.builderForMv(p2).buildAll().get(0); - HyperGraph h3 = HyperGraph.builderForMv(p3).buildAll().get(0); + HyperGraph h1 = HyperGraph.builderForMv(p1).build(); + HyperGraph h2 = HyperGraph.builderForMv(p2).build(); + HyperGraph h3 = HyperGraph.builderForMv(p3).build(); ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2)); Assertions.assertTrue(!res.isInvalid()); Assertions.assertTrue(!HyperGraphComparator.isLogicCompatible(h1, h3, constructContext(p1, p2)).isInvalid()); @@ -93,8 +95,8 @@ class EliminateJoinTest extends SqlTestBase { .rewrite() .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) .getAllPlan().get(0).child(0); - HyperGraph h1 = HyperGraph.builderForMv(p1).buildAll().get(0); - HyperGraph h2 = HyperGraph.builderForMv(p2).buildAll().get(0); + HyperGraph h1 = HyperGraph.builderForMv(p1).build(); + HyperGraph h2 = HyperGraph.builderForMv(p2).build(); ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2)); Assertions.assertTrue(!res.isInvalid()); Assertions.assertTrue(res.getViewExpressions().isEmpty()); @@ -134,9 +136,9 @@ class EliminateJoinTest extends SqlTestBase { .rewrite() .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) .getAllPlan().get(0).child(0); - HyperGraph h1 = HyperGraph.builderForMv(p1).buildAll().get(0); - HyperGraph h2 = HyperGraph.builderForMv(p2).buildAll().get(0); - HyperGraph h3 = HyperGraph.builderForMv(p3).buildAll().get(0); + HyperGraph h1 = HyperGraph.builderForMv(p1).build(); + HyperGraph h2 = HyperGraph.builderForMv(p2).build(); + HyperGraph h3 = HyperGraph.builderForMv(p3).build(); ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2)); Assertions.assertTrue(!res.isInvalid()); Assertions.assertTrue(res.getViewExpressions().isEmpty()); @@ -169,8 +171,8 @@ class EliminateJoinTest extends SqlTestBase { .rewrite() .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) .getAllPlan().get(0).child(0); - HyperGraph h1 = HyperGraph.builderForMv(p1).buildAll().get(0); - HyperGraph h2 = HyperGraph.builderForMv(p2).buildAll().get(0); + HyperGraph h1 = HyperGraph.builderForMv(p1).build(); + HyperGraph h2 = HyperGraph.builderForMv(p2).build(); ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2)); Assertions.assertTrue(!res.isInvalid()); Assertions.assertTrue(res.getViewExpressions().isEmpty()); @@ -202,8 +204,8 @@ class EliminateJoinTest extends SqlTestBase { .rewrite() .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) .getAllPlan().get(0).child(0); - HyperGraph h1 = HyperGraph.builderForMv(p1).buildAll().get(0); - HyperGraph h2 = HyperGraph.builderForMv(p2).buildAll().get(0); + HyperGraph h1 = HyperGraph.builderForMv(p1).build(); + HyperGraph h2 = HyperGraph.builderForMv(p2).build(); ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2)); Assertions.assertTrue(!res.isInvalid()); Assertions.assertTrue(res.getViewExpressions().isEmpty()); @@ -213,9 +215,9 @@ class EliminateJoinTest extends SqlTestBase { LogicalCompatibilityContext constructContext(Plan p1, Plan p2) { StructInfo st1 = MaterializedViewUtils.extractStructInfo(p1, - null).get(0); + null, new BitSet()).get(0); StructInfo st2 = MaterializedViewUtils.extractStructInfo(p2, - null).get(0); + null, new BitSet()).get(0); RelationMapping rm = RelationMapping.generate(st1.getRelations(), st2.getRelations()).get(0); SlotMapping sm = SlotMapping.generate(rm); return LogicalCompatibilityContext.from(rm, sm, st1, st2); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphAggTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphAggTest.java index 38ebc99c479..be4ce1cd054 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphAggTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphAggTest.java @@ -31,6 +31,7 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; +import java.util.BitSet; import java.util.Objects; class HyperGraphAggTest extends SqlTestBase { @@ -49,7 +50,7 @@ class HyperGraphAggTest extends SqlTestBase { .rewrite() .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) .getAllPlan().get(0).child(0); - HyperGraph h1 = HyperGraph.builderForMv(p2).buildAll().get(0); + HyperGraph h1 = HyperGraph.builderForMv(p2).build(); Assertions.assertEquals("id", Objects.requireNonNull(((StructInfoNode) h1.getNode(1)).getExpressions()).get(0).toSql()); } @@ -79,8 +80,8 @@ class HyperGraphAggTest extends SqlTestBase { .rewrite() .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) .getAllPlan().get(0).child(0); - HyperGraph h1 = HyperGraph.builderForMv(p1).buildAll().get(0); - HyperGraph h2 = HyperGraph.builderForMv(p2).buildAll().get(0); + HyperGraph h1 = HyperGraph.builderForMv(p1).build(); + HyperGraph h2 = HyperGraph.builderForMv(p2).build(); ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2)); Assertions.assertTrue(!res.isInvalid()); Assertions.assertEquals(2, res.getViewNoNullableSlot().size()); @@ -88,9 +89,9 @@ class HyperGraphAggTest extends SqlTestBase { LogicalCompatibilityContext constructContext(Plan p1, Plan p2) { StructInfo st1 = MaterializedViewUtils.extractStructInfo(p1, - null).get(0); + null, new BitSet()).get(0); StructInfo st2 = MaterializedViewUtils.extractStructInfo(p2, - null).get(0); + null, new BitSet()).get(0); RelationMapping rm = RelationMapping.generate(st1.getRelations(), st2.getRelations()).get(0); SlotMapping sm = SlotMapping.generate(rm); return LogicalCompatibilityContext.from(rm, sm, st1, st2); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparatorTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparatorTest.java index 1fd2ac86ab6..066f9dd0881 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparatorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparatorTest.java @@ -30,6 +30,8 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; +import java.util.BitSet; + class HyperGraphComparatorTest extends SqlTestBase { @Test void testInnerJoinAndLOJ() { @@ -55,8 +57,8 @@ class HyperGraphComparatorTest extends SqlTestBase { .rewrite() .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) .getAllPlan().get(0).child(0); - HyperGraph h1 = HyperGraph.builderForMv(p1).buildAll().get(0); - HyperGraph h2 = HyperGraph.builderForMv(p2).buildAll().get(0); + HyperGraph h1 = HyperGraph.builderForMv(p1).build(); + HyperGraph h2 = HyperGraph.builderForMv(p2).build(); ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2)); Assertions.assertTrue(!res.isInvalid()); Assertions.assertEquals(2, res.getViewNoNullableSlot().size()); @@ -86,8 +88,8 @@ class HyperGraphComparatorTest extends SqlTestBase { .rewrite() .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) .getAllPlan().get(0).child(0); - HyperGraph h1 = HyperGraph.builderForMv(p1).buildAll().get(0); - HyperGraph h2 = HyperGraph.builderForMv(p2).buildAll().get(0); + HyperGraph h1 = HyperGraph.builderForMv(p1).build(); + HyperGraph h2 = HyperGraph.builderForMv(p2).build(); ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2)); Assertions.assertTrue(!res.isInvalid()); Assertions.assertEquals(2, res.getViewNoNullableSlot().size()); @@ -118,8 +120,8 @@ class HyperGraphComparatorTest extends SqlTestBase { .rewrite() .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) .getAllPlan().get(0).child(0); - HyperGraph h1 = HyperGraph.builderForMv(p1).buildAll().get(0); - HyperGraph h2 = HyperGraph.builderForMv(p2).buildAll().get(0); + HyperGraph h1 = HyperGraph.builderForMv(p1).build(); + HyperGraph h2 = HyperGraph.builderForMv(p2).build(); ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2)); Assertions.assertTrue(!res.isInvalid()); Assertions.assertEquals(2, res.getViewNoNullableSlot().size()); @@ -153,8 +155,8 @@ class HyperGraphComparatorTest extends SqlTestBase { .rewrite() .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER) .getAllPlan().get(0).child(0); - HyperGraph h1 = HyperGraph.builderForMv(p1).buildAll().get(0); - HyperGraph h2 = HyperGraph.builderForMv(p2).buildAll().get(0); + HyperGraph h1 = HyperGraph.builderForMv(p1).build(); + HyperGraph h2 = HyperGraph.builderForMv(p2).build(); ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2)); Assertions.assertTrue(!res.isInvalid()); Assertions.assertEquals(2, res.getViewNoNullableSlot().size()); @@ -162,9 +164,9 @@ class HyperGraphComparatorTest extends SqlTestBase { LogicalCompatibilityContext constructContext(Plan p1, Plan p2) { StructInfo st1 = MaterializedViewUtils.extractStructInfo(p1, - null).get(0); + null, new BitSet()).get(0); StructInfo st2 = MaterializedViewUtils.extractStructInfo(p2, - null).get(0); + null, new BitSet()).get(0); RelationMapping rm = RelationMapping.generate(st1.getRelations(), st2.getRelations()).get(0); SlotMapping sm = SlotMapping.generate(rm); return LogicalCompatibilityContext.from(rm, sm, st1, st2); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/ExpressionUtilsTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/ExpressionUtilsTest.java index f68d37a24db..83a1faace1e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/ExpressionUtilsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/ExpressionUtilsTest.java @@ -26,6 +26,7 @@ import com.google.common.collect.Sets; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import java.util.BitSet; import java.util.List; /** @@ -173,11 +174,8 @@ public class ExpressionUtilsTest extends TestWithFeService { Plan rewrittenPlan = nereidsPlanner.getRewrittenPlan(); List<? extends Expression> originalExpressions = rewrittenPlan.getExpressions(); List<? extends Expression> shuttledExpressions - = ExpressionUtils.shuttleExpressionWithLineage( - originalExpressions, - rewrittenPlan, - Sets.newHashSet(), - Sets.newHashSet()); + = ExpressionUtils.shuttleExpressionWithLineage(originalExpressions, rewrittenPlan, + Sets.newHashSet(), Sets.newHashSet(), new BitSet()); assertExpect(originalExpressions, shuttledExpressions, "(cast(abs((cast(O_TOTALPRICE as DECIMALV3(16, 2)) + 10.00)) as " + "DOUBLE) + abs(sqrt(cast(PS_SUPPLYCOST as DOUBLE))))", diff --git a/regression-test/data/nereids_rules_p0/mv/nested/nested_materialized_view.out b/regression-test/data/nereids_rules_p0/mv/nested/nested_materialized_view.out new file mode 100644 index 00000000000..09c11bcee6a --- /dev/null +++ b/regression-test/data/nereids_rules_p0/mv/nested/nested_materialized_view.out @@ -0,0 +1,20 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !query1_0_before -- +4 +4 +4 +4 +6 +6 +6 +6 + +-- !query1_0_after -- +4 +4 +4 +4 +6 +6 +6 +6 \ No newline at end of file diff --git a/regression-test/suites/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.groovy b/regression-test/suites/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.groovy index 01672af1be0..d8037301c04 100644 --- a/regression-test/suites/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.groovy @@ -176,31 +176,37 @@ suite("aggregate_with_roll_up") { // multi table // filter inside + left + use roll up dimension - def mv13_0 = "select l_shipdate, o_orderdate, l_partkey, l_suppkey, " + - "sum(o_totalprice) as sum_total, " + - "max(o_totalprice) as max_total, " + - "min(o_totalprice) as min_total, " + - "count(*) as count_all, " + - "bitmap_union(to_bitmap(case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end)) as bitmap_union_basic " + - "from lineitem " + - "left join orders on lineitem.l_orderkey = orders.o_orderkey and l_shipdate = o_orderdate " + - "group by " + - "l_shipdate, " + - "o_orderdate, " + - "l_partkey, " + - "l_suppkey" - def query13_0 = "select t1.l_partkey, t1.l_suppkey, o_orderdate, " + - "sum(o_totalprice), " + - "max(o_totalprice), " + - "min(o_totalprice), " + - "count(*), " + - "count(distinct case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end) " + - "from (select * from lineitem where l_shipdate = '2023-12-11') t1 " + - "left join orders on t1.l_orderkey = orders.o_orderkey and t1.l_shipdate = o_orderdate " + - "group by " + - "o_orderdate, " + - "l_partkey, " + - "l_suppkey" + def mv13_0 = + """ + select l_shipdate, o_orderdate, l_partkey, l_suppkey, + sum(o_totalprice) as sum_total, + max(o_totalprice) as max_total, + min(o_totalprice) as min_total, + count(*) as count_all, + bitmap_union(to_bitmap(case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end)) as bitmap_union_basic + from lineitem + left join orders on lineitem.l_orderkey = orders.o_orderkey and l_shipdate = o_orderdate + group by + l_shipdate, + o_orderdate, + l_partkey, + l_suppkey + """ + def query13_0 = + """ + select t1.l_partkey, t1.l_suppkey, o_orderdate, + sum(o_totalprice), + max(o_totalprice), + min(o_totalprice), + count(*), + count(distinct case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end) + from (select * from lineitem where l_shipdate = '2023-12-11') t1 + left join orders on t1.l_orderkey = orders.o_orderkey and t1.l_shipdate = o_orderdate + group by + o_orderdate, + l_partkey, + l_suppkey + """ order_qt_query13_0_before "${query13_0}" check_mv_rewrite_success(db, mv13_0, query13_0, "mv13_0") order_qt_query13_0_after "${query13_0}" diff --git a/regression-test/suites/nereids_rules_p0/mv/join/inner/inner_join.groovy b/regression-test/suites/nereids_rules_p0/mv/join/inner/inner_join.groovy index 118fa959c9a..9f0b66fc66a 100644 --- a/regression-test/suites/nereids_rules_p0/mv/join/inner/inner_join.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/join/inner/inner_join.groovy @@ -355,13 +355,19 @@ suite("inner_join") { // filter outside + left + right - def mv4_0 = "select l_linenumber, o_custkey, o_orderkey, o_orderstatus " + - "from lineitem " + - "inner join orders on lineitem.l_orderkey = orders.o_orderkey " - def query4_0 = "select lineitem.l_linenumber " + - "from lineitem " + - "inner join orders on lineitem.l_orderkey = orders.o_orderkey " + - "where o_orderstatus = 'o' AND l_linenumber in (1, 2, 3, 4, 5) " + def mv4_0 = + """ + select l_linenumber, o_custkey, o_orderkey, o_orderstatus + from lineitem + inner join orders on lineitem.l_orderkey = orders.o_orderkey + """ + def query4_0 = + """ + select lineitem.l_linenumber + from lineitem + inner join orders on lineitem.l_orderkey = orders.o_orderkey + where o_orderstatus = 'o' AND l_linenumber in (1, 2, 3, 4, 5) + """ order_qt_query4_0_before "${query4_0}" check_mv_rewrite_success(db, mv4_0, query4_0, "mv4_0") order_qt_query4_0_after "${query4_0}" diff --git a/regression-test/suites/nereids_rules_p0/mv/nested/nested_materialized_view.groovy b/regression-test/suites/nereids_rules_p0/mv/nested/nested_materialized_view.groovy new file mode 100644 index 00000000000..6ad175e85ce --- /dev/null +++ b/regression-test/suites/nereids_rules_p0/mv/nested/nested_materialized_view.groovy @@ -0,0 +1,182 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("nested_materialized_view") { + String db = context.config.getDbNameByFile(context.file) + sql "use ${db}" + sql "SET enable_nereids_planner=true" + sql "set runtime_filter_mode=OFF" + sql "SET enable_fallback_to_original_planner=false" + sql "SET enable_materialized_view_rewrite=true" + sql "SET enable_nereids_timeout = false" + + def create_mtmv = { db_name, mv_name, mv_sql -> + sql """DROP MATERIALIZED VIEW IF EXISTS ${mv_name}""" + sql""" + CREATE MATERIALIZED VIEW ${mv_name} + BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ('replication_num' = '1') + AS ${mv_sql} + """ + + def job_name = getJobName(db_name, mv_name); + waitingMTMVTaskFinished(job_name) + } + + sql """ + drop table if exists orders + """ + + sql """ + CREATE TABLE IF NOT EXISTS orders ( + o_orderkey INTEGER NOT NULL, + o_custkey INTEGER NOT NULL, + o_orderstatus CHAR(1) NOT NULL, + o_totalprice DECIMALV3(15,2) NOT NULL, + o_orderdate DATE NOT NULL, + o_orderpriority CHAR(15) NOT NULL, + o_clerk CHAR(15) NOT NULL, + o_shippriority INTEGER NOT NULL, + o_comment VARCHAR(79) NOT NULL + ) + DUPLICATE KEY(o_orderkey, o_custkey) + PARTITION BY RANGE(o_orderdate) (PARTITION `day_2` VALUES LESS THAN ('2023-12-30')) + DISTRIBUTED BY HASH(o_orderkey) BUCKETS 3 + PROPERTIES ( + "replication_num" = "1" + ) + """ + + sql """ + drop table if exists lineitem + """ + + sql""" + CREATE TABLE IF NOT EXISTS lineitem ( + l_orderkey INTEGER NOT NULL, + l_partkey INTEGER NOT NULL, + l_suppkey INTEGER NOT NULL, + l_linenumber INTEGER NOT NULL, + l_quantity DECIMALV3(15,2) NOT NULL, + l_extendedprice DECIMALV3(15,2) NOT NULL, + l_discount DECIMALV3(15,2) NOT NULL, + l_tax DECIMALV3(15,2) NOT NULL, + l_returnflag CHAR(1) NOT NULL, + l_linestatus CHAR(1) NOT NULL, + l_shipdate DATE NOT NULL, + l_commitdate DATE NOT NULL, + l_receiptdate DATE NOT NULL, + l_shipinstruct CHAR(25) NOT NULL, + l_shipmode CHAR(10) NOT NULL, + l_comment VARCHAR(44) NOT NULL + ) + DUPLICATE KEY(l_orderkey, l_partkey, l_suppkey, l_linenumber) + PARTITION BY RANGE(l_shipdate) (PARTITION `day_1` VALUES LESS THAN ('2023-12-30')) + DISTRIBUTED BY HASH(l_orderkey) BUCKETS 3 + PROPERTIES ( + "replication_num" = "1" + ) + """ + + sql """ + drop table if exists partsupp + """ + + sql """ + CREATE TABLE IF NOT EXISTS partsupp ( + ps_partkey INTEGER NOT NULL, + ps_suppkey INTEGER NOT NULL, + ps_availqty INTEGER NOT NULL, + ps_supplycost DECIMALV3(15,2) NOT NULL, + ps_comment VARCHAR(199) NOT NULL + ) + DUPLICATE KEY(ps_partkey, ps_suppkey) + DISTRIBUTED BY HASH(ps_partkey) BUCKETS 3 + PROPERTIES ( + "replication_num" = "1" + ) + """ + + sql """ insert into lineitem values + (1, 2, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-08', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy'), + (2, 4, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-09', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy'), + (3, 2, 4, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-10', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy'), + (4, 3, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-11', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy'), + (5, 2, 3, 6, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-12-12', '2023-12-12', '2023-12-13', 'c', 'd', 'xxxxxxxxx'); + """ + + sql """ + insert into orders values + (1, 1, 'o', 9.5, '2023-12-08', 'a', 'b', 1, 'yy'), + (1, 1, 'o', 10.5, '2023-12-08', 'a', 'b', 1, 'yy'), + (2, 1, 'o', 11.5, '2023-12-09', 'a', 'b', 1, 'yy'), + (3, 1, 'o', 12.5, '2023-12-10', 'a', 'b', 1, 'yy'), + (3, 1, 'o', 33.5, '2023-12-10', 'a', 'b', 1, 'yy'), + (4, 2, 'o', 43.2, '2023-12-11', 'c','d',2, 'mm'), + (5, 2, 'o', 56.2, '2023-12-12', 'c','d',2, 'mi'), + (5, 2, 'o', 1.2, '2023-12-12', 'c','d',2, 'mi'); + """ + + sql """ + insert into partsupp values + (2, 3, 9, 10.01, 'supply1'), + (2, 3, 10, 11.01, 'supply2'); + """ + + // simple nested materialized view + def mv1_0_inner_mv = """ + select + l_linenumber, + o_custkey, + o_orderkey, + o_orderstatus, + l_partkey, + l_suppkey, + l_orderkey + from lineitem + inner join orders on lineitem.l_orderkey = orders.o_orderkey; + """ + + def mv1_0 = + """ + select + l_linenumber, + o_custkey, + o_orderkey, + o_orderstatus, + l_partkey, + l_suppkey, + l_orderkey, + ps_availqty + from mv1_0_inner_mv + inner join partsupp on l_partkey = ps_partkey AND l_suppkey = ps_suppkey; + """ + def query1_0 = """ + select lineitem.l_linenumber + from lineitem + inner join orders on l_orderkey = o_orderkey + inner join partsupp on l_partkey = ps_partkey AND l_suppkey = ps_suppkey + where o_orderstatus = 'o' + """ + order_qt_query1_0_before "${query1_0}" + create_mtmv(db, "mv1_0_inner_mv", mv1_0_inner_mv) + check_mv_rewrite_success(db, mv1_0, query1_0, "mv1_0") + order_qt_query1_0_after "${query1_0}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_0_inner_mv""" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_0""" +} --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
