This is an automated email from the ASF dual-hosted git repository. yiguolei pushed a commit to branch branch-2.1 in repository https://gitbox.apache.org/repos/asf/doris.git
commit 2da57526a3098ae009288ae478407beccee31bc0 Author: 谢健 <[email protected]> AuthorDate: Tue Mar 12 15:52:36 2024 +0800 [feat](Nereids): use table map to construct struct info (#32058) --- .../apache/doris/nereids/memo/StructInfoMap.java | 46 ++++++++++++++++----- .../doris/nereids/memo/StructInfoMapTest.java | 48 ++++++++++++++++++++++ .../apache/doris/utframe/TestWithFeService.java | 25 +++++++++++ 3 files changed, 108 insertions(+), 11 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java index f7e09fd44d9..c90cc35c85d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java @@ -17,6 +17,7 @@ package org.apache.doris.nereids.memo; +import org.apache.doris.common.Pair; import org.apache.doris.nereids.rules.exploration.mv.StructInfo; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalCatalogRelation; @@ -37,11 +38,12 @@ import javax.annotation.Nullable; * Representation for group in cascades optimizer. */ public class StructInfoMap { - private final Map<BitSet, GroupExpression> groupExpressionMap = new HashMap<>(); + private final Map<BitSet, Pair<GroupExpression, List<BitSet>>> groupExpressionMap = new HashMap<>(); private final Map<BitSet, StructInfo> infoMap = new HashMap<>(); /** * get struct info according to table map + * * @param mvTableMap the original table map * @param foldTableMap the fold table map * @param group the group that the mv matched @@ -54,7 +56,10 @@ public class StructInfoMap { refresh(group); } if (groupExpressionMap.containsKey(mvTableMap)) { - StructInfo structInfo = constructStructInfo(groupExpressionMap.get(mvTableMap)); + Pair<GroupExpression, List<BitSet>> groupExpressionBitSetPair = getGroupExpressionWithChildren( + mvTableMap); + StructInfo structInfo = constructStructInfo(groupExpressionBitSetPair.first, + groupExpressionBitSetPair.second, mvTableMap); infoMap.put(mvTableMap, structInfo); } } @@ -66,20 +71,39 @@ public class StructInfoMap { return groupExpressionMap.keySet(); } - private StructInfo constructStructInfo(GroupExpression groupExpression) { - throw new RuntimeException("has not been implemented for" + groupExpression); + public Pair<GroupExpression, List<BitSet>> getGroupExpressionWithChildren(BitSet tableMap) { + return groupExpressionMap.get(tableMap); + } + + private StructInfo constructStructInfo(GroupExpression groupExpression, List<BitSet> children, BitSet tableMap) { + Plan plan = constructPlan(groupExpression, children, tableMap); + return StructInfo.of(plan).get(0); + } + + private Plan constructPlan(GroupExpression groupExpression, List<BitSet> children, BitSet tableMap) { + List<Plan> childrenPlan = new ArrayList<>(); + for (int i = 0; i < children.size(); i++) { + StructInfoMap structInfoMap = groupExpression.child(i).getstructInfoMap(); + BitSet childMap = children.get(i); + Pair<GroupExpression, List<BitSet>> groupExpressionBitSetPair + = structInfoMap.getGroupExpressionWithChildren(childMap); + childrenPlan.add( + constructPlan(groupExpressionBitSetPair.first, groupExpressionBitSetPair.second, childMap)); + } + return groupExpression.getPlan().withChildren(childrenPlan); } /** * refresh group expression map + * * @param group the root group */ public void refresh(Group group) { - List<Set<BitSet>> childrenTableMap = new ArrayList<>(); Set<Group> refreshedGroup = new HashSet<>(); for (GroupExpression groupExpression : group.getLogicalExpressions()) { + List<Set<BitSet>> childrenTableMap = new ArrayList<>(); if (groupExpression.children().isEmpty()) { - groupExpressionMap.put(constructLeaf(groupExpression), groupExpression); + groupExpressionMap.put(constructLeaf(groupExpression), Pair.of(groupExpression, new ArrayList<>())); continue; } for (Group child : groupExpression.children()) { @@ -90,9 +114,9 @@ public class StructInfoMap { refreshedGroup.add(child); childrenTableMap.add(child.getstructInfoMap().getTableMaps()); } - Set<BitSet> bitSets = cartesianProduct(childrenTableMap); - for (BitSet bitSet : bitSets) { - groupExpressionMap.put(bitSet, groupExpression); + Set<Pair<BitSet, List<BitSet>>> bitSetWithChildren = cartesianProduct(childrenTableMap); + for (Pair<BitSet, List<BitSet>> bitSetWithChild : bitSetWithChildren) { + groupExpressionMap.put(bitSetWithChild.first, Pair.of(groupExpression, bitSetWithChild.second)); } } } @@ -108,7 +132,7 @@ public class StructInfoMap { return tableMap; } - private Set<BitSet> cartesianProduct(List<Set<BitSet>> childrenTableMap) { + private Set<Pair<BitSet, List<BitSet>>> cartesianProduct(List<Set<BitSet>> childrenTableMap) { return Sets.cartesianProduct(childrenTableMap) .stream() .map(bitSetList -> { @@ -116,7 +140,7 @@ public class StructInfoMap { for (BitSet b : bitSetList) { bitSet.or(b); } - return bitSet; + return Pair.of(bitSet, bitSetList); }) .collect(Collectors.toSet()); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java index 0699f04b92f..08db05b5065 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java @@ -20,6 +20,7 @@ package org.apache.doris.nereids.memo; import org.apache.doris.catalog.MTMV; import org.apache.doris.mtmv.MTMVRelationManager; import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.rules.exploration.mv.StructInfo; import org.apache.doris.nereids.sqltest.SqlTestBase; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.qe.ConnectContext; @@ -31,6 +32,7 @@ import org.junit.jupiter.api.Test; import java.util.BitSet; import java.util.Set; +import java.util.stream.Collectors; class StructInfoMapTest extends SqlTestBase { @Test @@ -77,5 +79,51 @@ class StructInfoMapTest extends SqlTestBase { root.getstructInfoMap().refresh(root); tableMaps = root.getstructInfoMap().getTableMaps(); Assertions.assertEquals(2, tableMaps.size()); + dropMvByNereids("drop materialized view mv1"); + } + + @Test + void testTableChild() throws Exception { + CascadesContext c1 = createCascadesContext( + "select T1.id from T1 inner join T2 " + + "on T1.id = T2.id " + + "inner join T3 on T1.id = T3.id", + connectContext + ); + new MockUp<MTMVRelationManager>() { + @Mock + public boolean isMVPartitionValid(MTMV mtmv, ConnectContext ctx) { + return true; + } + }; + connectContext.getSessionVariable().enableMaterializedViewRewrite = true; + createMvByNereids("create materialized view mv1 BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL\n" + + " DISTRIBUTED BY RANDOM BUCKETS 1\n" + + " PROPERTIES ('replication_num' = '1') \n" + + " as select T1.id from T1 inner join T2 " + + "on T1.id = T2.id;"); + c1 = createCascadesContext( + "select T1.id from T1 inner join T2 " + + "on T1.id = T2.id " + + "inner join T3 on T1.id = T3.id", + connectContext + ); + PlanChecker.from(c1) + .analyze() + .rewrite() + .optimize(); + Group root = c1.getMemo().getRoot(); + root.getstructInfoMap().refresh(root); + StructInfoMap structInfoMap = root.getstructInfoMap(); + Assertions.assertEquals(2, structInfoMap.getTableMaps().size()); + BitSet mvMap = structInfoMap.getTableMaps().stream() + .filter(b -> b.cardinality() == 2) + .collect(Collectors.toList()).get(0); + StructInfo structInfo = structInfoMap.getStructInfo(mvMap, mvMap, root); + System.out.println(structInfo.getOriginalPlan().treeString()); + BitSet bitSet = new BitSet(); + structInfo.getRelations().forEach(r -> bitSet.set((int) r.getTable().getId())); + Assertions.assertEquals(bitSet, mvMap); + dropMvByNereids("drop materialized view mv1"); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java b/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java index 7de8a9a830b..c43ab447885 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java +++ b/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java @@ -69,6 +69,7 @@ import org.apache.doris.nereids.trees.plans.commands.AddConstraintCommand; import org.apache.doris.nereids.trees.plans.commands.CreateMTMVCommand; import org.apache.doris.nereids.trees.plans.commands.CreateTableCommand; import org.apache.doris.nereids.trees.plans.commands.DropConstraintCommand; +import org.apache.doris.nereids.trees.plans.commands.DropMTMVCommand; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.util.MemoTestUtils; import org.apache.doris.persist.CreateTableInfo; @@ -823,6 +824,30 @@ public abstract class TestWithFeService { } + protected void dropMvByNereids(String sql) throws Exception { + new MockUp<EditLog>() { + @Mock + public void logCreateTable(CreateTableInfo info) { + System.out.println("skip log create table..."); + } + + @Mock + public void logCreateJob(AbstractJob job) { + System.out.println("skip log create job..."); + } + }; + NereidsParser nereidsParser = new NereidsParser(); + LogicalPlan parsed = nereidsParser.parseSingle(sql); + StmtExecutor stmtExecutor = new StmtExecutor(connectContext, sql); + if (parsed instanceof DropMTMVCommand) { + ((DropMTMVCommand) parsed).run(connectContext, stmtExecutor); + } + checkAlterJob(); + // waiting table state to normal + Thread.sleep(1000); + + } + private void updateReplicaPathHash() { com.google.common.collect.Table<Long, Long, Replica> replicaMetaTable = Env.getCurrentInvertedIndex() .getReplicaMetaTable(); --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
