This is an automated email from the ASF dual-hosted git repository. yiguolei pushed a commit to branch branch-2.1 in repository https://gitbox.apache.org/repos/asf/doris.git
commit 958f1a2d2dd7391188cba8bf75529ebbd494f170 Author: zhangdong <[email protected]> AuthorDate: Fri May 10 10:27:45 2024 +0800 [enhance](mtmv) add baseTablesOneLevel for MTMVRelation and MTMVRelationManager (#34593) baseTables in MTMVRelation stores all baseTables in the nested materialized view, now adding baseTablesOneLevel to only store the baseTables at the current level. --- .../java/org/apache/doris/mtmv/MTMVPlanUtil.java | 6 +- .../java/org/apache/doris/mtmv/MTMVRelation.java | 12 ++- .../org/apache/doris/mtmv/MTMVRelationManager.java | 34 ++++++- .../apache/doris/mtmv/MTMVRelationManagerTest.java | 105 +++++++++++++++++++++ .../java/org/apache/doris/mtmv/MTMVTaskTest.java | 2 +- .../test/java/org/apache/doris/mtmv/MTMVTest.java | 4 +- 6 files changed, 154 insertions(+), 9 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java index 16eab00dca9..e74ca1f8aff 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java @@ -117,14 +117,14 @@ public class MTMVPlanUtil { } public static MTMVRelation generateMTMVRelation(Plan plan) { - return new MTMVRelation(getBaseTables(plan), getBaseViews(plan)); + return new MTMVRelation(getBaseTables(plan, true), getBaseTables(plan, false), getBaseViews(plan)); } - private static Set<BaseTableInfo> getBaseTables(Plan plan) { + private static Set<BaseTableInfo> getBaseTables(Plan plan, boolean expand) { TableCollectorContext collectorContext = new TableCollector.TableCollectorContext( com.google.common.collect.Sets - .newHashSet(TableType.values()), true); + .newHashSet(TableType.values()), expand); plan.accept(TableCollector.INSTANCE, collectorContext); Set<TableIf> collectedTables = collectorContext.getCollectedTables(); return transferTableIfToInfo(collectedTables); diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelation.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelation.java index b9f5350086a..aec89caa508 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelation.java @@ -30,9 +30,13 @@ public class MTMVRelation { private Set<BaseTableInfo> baseTables; @SerializedName("bv") private Set<BaseTableInfo> baseViews; + @SerializedName("btol") + private Set<BaseTableInfo> baseTablesOneLevel; - public MTMVRelation(Set<BaseTableInfo> baseTables, Set<BaseTableInfo> baseViews) { + public MTMVRelation(Set<BaseTableInfo> baseTables, Set<BaseTableInfo> baseTablesOneLevel, + Set<BaseTableInfo> baseViews) { this.baseTables = baseTables; + this.baseTablesOneLevel = baseTablesOneLevel; this.baseViews = baseViews; } @@ -40,6 +44,11 @@ public class MTMVRelation { return baseTables; } + public Set<BaseTableInfo> getBaseTablesOneLevel() { + // For compatibility, previously created MTMV may not have baseTablesOneLevel + return baseTablesOneLevel == null ? baseTables : baseTablesOneLevel; + } + public Set<BaseTableInfo> getBaseViews() { return baseViews; } @@ -48,6 +57,7 @@ public class MTMVRelation { public String toInfoString() { return "MTMVRelation{" + "baseTables=" + baseTables + + ", baseTablesOneLevel=" + baseTablesOneLevel + ", baseViews=" + baseViews + '}'; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelationManager.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelationManager.java index 693bb4b19de..a41687bb6d4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelationManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelationManager.java @@ -53,12 +53,22 @@ import java.util.Set; */ public class MTMVRelationManager implements MTMVHookService { private static final Logger LOG = LogManager.getLogger(MTMVRelationManager.class); + // when + // create mv1 as select * from table1; + // create mv2 as select * from mv1; + // `tableMTMVs` will have 3 pair: table1 ==> mv1,mv1==>mv2, table1 ==> mv2 + // `tableMTMVsOneLevel` will have 2 pair: table1 ==> mv1,mv1==>mv2 private Map<BaseTableInfo, Set<BaseTableInfo>> tableMTMVs = Maps.newConcurrentMap(); + private Map<BaseTableInfo, Set<BaseTableInfo>> tableMTMVsOneLevel = Maps.newConcurrentMap(); - private Set<BaseTableInfo> getMtmvsByBaseTable(BaseTableInfo table) { + public Set<BaseTableInfo> getMtmvsByBaseTable(BaseTableInfo table) { return tableMTMVs.getOrDefault(table, ImmutableSet.of()); } + public Set<BaseTableInfo> getMtmvsByBaseTableOneLevel(BaseTableInfo table) { + return tableMTMVsOneLevel.getOrDefault(table, ImmutableSet.of()); + } + /** * if At least one partition is available, return this mtmv * @@ -104,7 +114,14 @@ public class MTMVRelationManager implements MTMVHookService { return tableMTMVs.get(baseTableInfo); } - private void refreshMTMVCache(MTMVRelation relation, BaseTableInfo mtmvInfo) { + private Set<BaseTableInfo> getOrCreateMTMVsOneLevel(BaseTableInfo baseTableInfo) { + if (!tableMTMVsOneLevel.containsKey(baseTableInfo)) { + tableMTMVsOneLevel.put(baseTableInfo, Sets.newConcurrentHashSet()); + } + return tableMTMVsOneLevel.get(baseTableInfo); + } + + public void refreshMTMVCache(MTMVRelation relation, BaseTableInfo mtmvInfo) { LOG.info("refreshMTMVCache,relation: {}, mtmvInfo: {}", relation, mtmvInfo); removeMTMV(mtmvInfo); addMTMV(relation, mtmvInfo); @@ -116,6 +133,7 @@ public class MTMVRelationManager implements MTMVHookService { } addMTMVTables(relation.getBaseTables(), mtmvInfo); addMTMVTables(relation.getBaseViews(), mtmvInfo); + addMTMVTablesOneLevel(relation.getBaseTablesOneLevel(), mtmvInfo); } private void addMTMVTables(Set<BaseTableInfo> baseTables, BaseTableInfo mtmvInfo) { @@ -127,10 +145,22 @@ public class MTMVRelationManager implements MTMVHookService { } } + private void addMTMVTablesOneLevel(Set<BaseTableInfo> baseTables, BaseTableInfo mtmvInfo) { + if (CollectionUtils.isEmpty(baseTables)) { + return; + } + for (BaseTableInfo baseTableInfo : baseTables) { + getOrCreateMTMVsOneLevel(baseTableInfo).add(mtmvInfo); + } + } + private void removeMTMV(BaseTableInfo mtmvInfo) { for (Set<BaseTableInfo> sets : tableMTMVs.values()) { sets.remove(mtmvInfo); } + for (Set<BaseTableInfo> sets : tableMTMVsOneLevel.values()) { + sets.remove(mtmvInfo); + } } @Override diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRelationManagerTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRelationManagerTest.java new file mode 100644 index 00000000000..697643337c2 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRelationManagerTest.java @@ -0,0 +1,105 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.mtmv; + +import com.google.common.collect.Sets; +import org.apache.commons.collections.CollectionUtils; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Set; + +public class MTMVRelationManagerTest { + @Test + public void testGetMtmvsByBaseTableOneLevel() { + // mock mv2==>mv1,t3; mv1==>t4 + MTMVRelationManager manager = new MTMVRelationManager(); + BaseTableInfo mv1 = new BaseTableInfo(0L, 1L); + BaseTableInfo mv2 = new BaseTableInfo(0L, 2L); + BaseTableInfo t3 = new BaseTableInfo(0L, 3L); + BaseTableInfo t4 = new BaseTableInfo(0L, 4L); + MTMVRelation mv2Relation = new MTMVRelation(Sets.newHashSet(mv1, t3, t4), Sets.newHashSet(mv1, t3), + Sets.newHashSet()); + MTMVRelation mv1Relation = new MTMVRelation(Sets.newHashSet(t4), Sets.newHashSet(t4), + Sets.newHashSet()); + manager.refreshMTMVCache(mv2Relation, mv2); + manager.refreshMTMVCache(mv1Relation, mv1); + // should return mv2 + Set<BaseTableInfo> mv1OneLevel = manager.getMtmvsByBaseTableOneLevel(mv1); + Assert.assertTrue(CollectionUtils.isEqualCollection(Sets.newHashSet(mv2), mv1OneLevel)); + // should return mv2 + Set<BaseTableInfo> t3OneLevel = manager.getMtmvsByBaseTableOneLevel(t3); + Assert.assertTrue(CollectionUtils.isEqualCollection(Sets.newHashSet(mv2), t3OneLevel)); + // should return mv1 + Set<BaseTableInfo> t4OneLevel = manager.getMtmvsByBaseTableOneLevel(t4); + Assert.assertTrue(CollectionUtils.isEqualCollection(Sets.newHashSet(mv1), t4OneLevel)); + + // update mv2 only use t3,remove mv1 + mv2Relation = new MTMVRelation(Sets.newHashSet(t3), Sets.newHashSet(t3), + Sets.newHashSet()); + manager.refreshMTMVCache(mv2Relation, mv2); + // should return empty + mv1OneLevel = manager.getMtmvsByBaseTableOneLevel(mv1); + Assert.assertTrue(CollectionUtils.isEqualCollection(Sets.newHashSet(), mv1OneLevel)); + // should return mv2 + t3OneLevel = manager.getMtmvsByBaseTableOneLevel(t3); + Assert.assertTrue(CollectionUtils.isEqualCollection(Sets.newHashSet(mv2), t3OneLevel)); + // should return mv1 + t4OneLevel = manager.getMtmvsByBaseTableOneLevel(t4); + Assert.assertTrue(CollectionUtils.isEqualCollection(Sets.newHashSet(mv1), t4OneLevel)); + } + + @Test + public void testGetMtmvsByBaseTable() { + // mock mv2==>mv1,t3; mv1==>t4 + MTMVRelationManager manager = new MTMVRelationManager(); + BaseTableInfo mv1 = new BaseTableInfo(0L, 1L); + BaseTableInfo mv2 = new BaseTableInfo(0L, 2L); + BaseTableInfo t3 = new BaseTableInfo(0L, 3L); + BaseTableInfo t4 = new BaseTableInfo(0L, 4L); + MTMVRelation mv2Relation = new MTMVRelation(Sets.newHashSet(mv1, t3, t4), Sets.newHashSet(mv1, t3), + Sets.newHashSet()); + MTMVRelation mv1Relation = new MTMVRelation(Sets.newHashSet(t4), Sets.newHashSet(t4), + Sets.newHashSet()); + manager.refreshMTMVCache(mv2Relation, mv2); + manager.refreshMTMVCache(mv1Relation, mv1); + // should return mv2 + Set<BaseTableInfo> mv1All = manager.getMtmvsByBaseTable(mv1); + Assert.assertTrue(CollectionUtils.isEqualCollection(Sets.newHashSet(mv2), mv1All)); + // should return mv2 + Set<BaseTableInfo> t3All = manager.getMtmvsByBaseTable(t3); + Assert.assertTrue(CollectionUtils.isEqualCollection(Sets.newHashSet(mv2), t3All)); + // should return mv1 + Set<BaseTableInfo> t4All = manager.getMtmvsByBaseTable(t4); + Assert.assertTrue(CollectionUtils.isEqualCollection(Sets.newHashSet(mv1, mv2), t4All)); + + // update mv2 only use t3,remove mv1 + mv2Relation = new MTMVRelation(Sets.newHashSet(t3), Sets.newHashSet(t3), + Sets.newHashSet()); + manager.refreshMTMVCache(mv2Relation, mv2); + // should return empty + mv1All = manager.getMtmvsByBaseTableOneLevel(mv1); + Assert.assertTrue(CollectionUtils.isEqualCollection(Sets.newHashSet(), mv1All)); + // should return mv2 + t3All = manager.getMtmvsByBaseTableOneLevel(t3); + Assert.assertTrue(CollectionUtils.isEqualCollection(Sets.newHashSet(mv2), t3All)); + // should return mv1 + t4All = manager.getMtmvsByBaseTableOneLevel(t4); + Assert.assertTrue(CollectionUtils.isEqualCollection(Sets.newHashSet(mv1), t4All)); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVTaskTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVTaskTest.java index 55ddc85768c..3bc60339500 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVTaskTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVTaskTest.java @@ -47,7 +47,7 @@ public class MTMVTaskTest { private long ptwoId = 2L; private String ptwoName = "p2"; private List<Long> allPartitionIds = Lists.newArrayList(poneId, ptwoId); - private MTMVRelation relation = new MTMVRelation(Sets.newHashSet(), Sets.newHashSet()); + private MTMVRelation relation = new MTMVRelation(Sets.newHashSet(), Sets.newHashSet(), Sets.newHashSet()); @Mocked private MTMV mtmv; diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVTest.java index 695203b9cec..4a3bf12f889 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVTest.java @@ -43,7 +43,7 @@ public class MTMVTest { + "needRefreshPartitions=null, completedPartitions=null, refreshMode=null} " + "AbstractTask(jobId=null, taskId=1, status=null, createTimeMs=null, startTimeMs=null, " + "finishTimeMs=null, taskType=null, errMsg=null)]}, mvProperties={}, " - + "relation=MTMVRelation{baseTables=[], baseViews=[]}, " + + "relation=MTMVRelation{baseTables=[], baseTablesOneLevel=[], baseViews=[]}, " + "mvPartitionInfo=MTMVPartitionInfo{partitionType=null, relatedTable=null, " + "relatedCol='null', partitionCol='null'}, " + "refreshSnapshot=MTMVRefreshSnapshot{partitionSnapshots={}}, id=1, name='null', " @@ -58,7 +58,7 @@ public class MTMVTest { mtmv.setEnvInfo(new EnvInfo(1L, 2L)); mtmv.setJobInfo(buildMTMVJobInfo(mtmv)); mtmv.setMvProperties(new HashMap<>()); - mtmv.setRelation(new MTMVRelation(Sets.newHashSet(), Sets.newHashSet())); + mtmv.setRelation(new MTMVRelation(Sets.newHashSet(), Sets.newHashSet(), Sets.newHashSet())); mtmv.setMvPartitionInfo(new MTMVPartitionInfo()); mtmv.setRefreshSnapshot(new MTMVRefreshSnapshot()); Assert.assertEquals(expect, mtmv.toInfoString()); --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
