This is an automated email from the ASF dual-hosted git repository.
morrySnow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 7cea7c9c216 [opt](mtmv) Optimize mv rewrite performance when mv has
many partition by reduce partition mapping num (#59972)
7cea7c9c216 is described below
commit 7cea7c9c216554b77e3e4bcecb1386d1d6bc6f0e
Author: seawinde <[email protected]>
AuthorDate: Thu Aug 13 16:21:32 2026 +0800
[opt](mtmv) Optimize mv rewrite performance when mv has many partition by
reduce partition mapping num (#59972)
Summary
- Make MTMV partition descriptor generation honor queryUsedPartitions,
so only partitions referenced by the current query are carried through
the pipeline instead of all partitions.
- Applied the filter in
MTMVRelatedPartitionDescOnePartitionColGenerator, preserving ordering of
existing generators.
- Added unit tests for RANGE and LIST sources to assert the filter
reduces descriptors to the specified partitions.
Example
If a query only touches t1 partition p20210201,
generateRelatedPartitionDescs now emits a single descriptor for
p20210201 instead of all t1 partitions;
---
.../main/java/org/apache/doris/catalog/MTMV.java | 39 ++-
.../apache/doris/job/extensions/mtmv/MTMVTask.java | 2 +-
.../apache/doris/mtmv/MTMVPartitionExpander.java | 124 ++++++++++
.../org/apache/doris/mtmv/MTMVPartitionUtil.java | 46 +++-
.../org/apache/doris/mtmv/MTMVRefreshContext.java | 8 +-
.../MTMVRelatedPartitionDescGeneratorService.java | 4 +-
.../MTMVRelatedPartitionDescInitGenerator.java | 4 +-
...latedPartitionDescOnePartitionColGenerator.java | 8 +-
.../MTMVRelatedPartitionDescRollUpGenerator.java | 3 +-
...MTMVRelatedPartitionDescSyncLimitGenerator.java | 4 +-
.../MTMVRelatedPartitionDescTransferGenerator.java | 3 +-
.../org/apache/doris/mtmv/MTMVRewriteUtil.java | 3 +-
.../mv/AsyncMaterializationContext.java | 69 +++++-
.../rules/exploration/mv/PartitionCompensator.java | 2 +-
.../apache/doris/mtmv/MTMVExpandPartitionTest.java | 269 +++++++++++++++++++++
.../apache/doris/mtmv/MTMVPartitionUtilTest.java | 78 +++++-
.../MTMVRelatedPartitionDescGeneratorTest.java | 55 ++++-
.../mv/AsyncMaterializationContextTest.java | 136 +++++++++++
.../exploration/mv/PartitionCompensatorTest.java | 4 +-
19 files changed, 818 insertions(+), 43 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java
index c760d63563e..d0fa55678d7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java
@@ -31,6 +31,7 @@ import org.apache.doris.mtmv.EnvInfo;
import org.apache.doris.mtmv.MTMVCache;
import org.apache.doris.mtmv.MTMVJobInfo;
import org.apache.doris.mtmv.MTMVJobManager;
+import org.apache.doris.mtmv.MTMVPartitionExpander;
import org.apache.doris.mtmv.MTMVPartitionInfo;
import org.apache.doris.mtmv.MTMVPartitionInfo.MTMVPartitionType;
import org.apache.doris.mtmv.MTMVPartitionUtil;
@@ -56,6 +57,7 @@ import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.io.IOException;
+import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Optional;
@@ -522,6 +524,27 @@ public class MTMV extends OlapTable {
return result;
}
+ /**
+ * Normalize query-used base table partitions to the effective filter
consumed by
+ * partition-mapping generation.
+ */
+ public Map<List<String>, Set<String>>
getEffectiveQueryUsedBaseTablePartitionMap(
+ Map<List<String>, Set<String>> queryUsedBaseTablePartitionMap)
throws AnalysisException {
+ return
getEffectiveQueryUsedBaseTablePartitionMap(queryUsedBaseTablePartitionMap,
null);
+ }
+
+ private Map<List<String>, Set<String>>
getEffectiveQueryUsedBaseTablePartitionMap(
+ Map<List<String>, Set<String>> queryUsedBaseTablePartitionMap,
+ Map<String, PartitionItem> mvPartitionItems) throws
AnalysisException {
+ if (queryUsedBaseTablePartitionMap.isEmpty()
+ || mvPartitionInfo.getPartitionType() !=
MTMVPartitionType.EXPR) {
+ return queryUsedBaseTablePartitionMap;
+ }
+ return
MTMVPartitionExpander.expandToMvPartitionGranularity(queryUsedBaseTablePartitionMap,
+ mvPartitionItems != null ? mvPartitionItems :
getAndCopyPartitionItems(),
+ mvPartitionInfo.getPctTables());
+ }
+
/**
* Calculate the partition and associated partition mapping relationship
of the MTMV
* It is the result of real-time comparison calculation, so there may be
some costs,
@@ -530,15 +553,25 @@ public class MTMV extends OlapTable {
* @return mvPartitionName ==> pctTable ==> pctPartitionName
* @throws AnalysisException
*/
- public Map<String, Map<MTMVRelatedTableIf, Set<String>>>
calculatePartitionMappings() throws AnalysisException {
+ public Map<String, Map<MTMVRelatedTableIf, Set<String>>>
calculatePartitionMappings(
+ Map<List<String>, Set<String>> queryUsedBaseTablePartitionMap)
throws AnalysisException {
if (mvPartitionInfo.getPartitionType() ==
MTMVPartitionType.SELF_MANAGE) {
return Maps.newHashMap();
}
long start = System.currentTimeMillis();
+ // For EXPR-type partitions with RANGE base tables, expand the
query-used partition
+ // filter to MV partition granularity. This ensures complete partition
mappings per
+ // MV partition (needed for isSyncWithPartitions correctness) while
skipping
+ // irrelevant MV partitions entirely (the performance optimization).
+ // For nested MVs where pctTable is not in the filter, the expanded
map is empty,
+ // so the pipeline runs without filtering (full computation) — correct
behavior.
+ Map<String, PartitionItem> mvPartitionItems =
getAndCopyPartitionItems();
+ Map<List<String>, Set<String>> effectiveFilter
+ =
getEffectiveQueryUsedBaseTablePartitionMap(queryUsedBaseTablePartitionMap,
mvPartitionItems);
Map<String, Map<MTMVRelatedTableIf, Set<String>>> res =
Maps.newHashMap();
Map<PartitionKeyDesc, Map<MTMVRelatedTableIf, Set<String>>>
pctPartitionDescs = MTMVPartitionUtil
- .generateRelatedPartitionDescs(mvPartitionInfo, mvProperties,
getPartitionColumns());
- Map<String, PartitionItem> mvPartitionItems =
getAndCopyPartitionItems();
+ .generateRelatedPartitionDescs(mvPartitionInfo, mvProperties,
getPartitionColumns(),
+ effectiveFilter);
for (Entry<String, PartitionItem> entry : mvPartitionItems.entrySet())
{
res.put(entry.getKey(),
pctPartitionDescs.getOrDefault(entry.getValue().toPartitionKeyDesc(),
Maps.newHashMap()));
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java
b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java
index 9766a175a56..95e79495c45 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java
@@ -240,7 +240,7 @@ public class MTMVTask extends AbstractTask {
}
MetaLockUtils.readLockTables(tableIfs);
try {
- context = MTMVRefreshContext.buildContext(mtmv);
+ context = MTMVRefreshContext.buildContext(mtmv,
Maps.newHashMap());
this.needRefreshPartitions =
calculateNeedRefreshPartitions(context);
} finally {
MetaLockUtils.readUnlockTables(tableIfs);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionExpander.java
b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionExpander.java
new file mode 100644
index 00000000000..7a20b9fd0a0
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionExpander.java
@@ -0,0 +1,124 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.mtmv;
+
+import org.apache.doris.catalog.PartitionItem;
+import org.apache.doris.catalog.PartitionKey;
+import org.apache.doris.catalog.PartitionType;
+import org.apache.doris.catalog.RangePartitionItem;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.datasource.mvcc.MvccSnapshot;
+import org.apache.doris.datasource.mvcc.MvccUtil;
+
+import com.google.common.collect.Maps;
+import com.google.common.collect.Range;
+import com.google.common.collect.Sets;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.NavigableMap;
+import java.util.Optional;
+import java.util.Set;
+import java.util.TreeMap;
+
+/**
+ * Utility to expand query-used partition filters to MV partition granularity
+ * using Range.encloses(), avoiding expensive dateTrunc / strToDate /
dateIncrement
+ * per-partition operations in the rollup pipeline.
+ * Separated from MTMV to keep a lightweight dependency tree for testability —
+ * loading this class does not trigger MTMV → OlapTable → CloudReplica class
loading.
+ */
+public class MTMVPartitionExpander {
+
+ /**
+ * Expand queryUsedPartitions to MV partition granularity for RANGE base
tables.
+ * For example, if MV is monthly partitioned (date_trunc(month)) and base
table is daily:
+ * - Query uses p_20250115 (Jan 15)
+ * - Find MV partition p_202501 that encloses [20250115, 20250116)
+ * - Expand to ALL daily partitions within p_202501's range [20250101,
20250201)
+ * - Result: {p_20250101, p_20250102, ..., p_20250131}
+ */
+ public static Map<List<String>, Set<String>>
expandToMvPartitionGranularity(
+ Map<List<String>, Set<String>> queryUsedBaseTablePartitionMap,
+ Map<String, PartitionItem> mvPartitionItems,
+ Set<MTMVRelatedTableIf> pctTables) throws AnalysisException {
+ NavigableMap<PartitionKey, Range<PartitionKey>> mvRanges = new
TreeMap<>();
+ for (PartitionItem item : mvPartitionItems.values()) {
+ Range<PartitionKey> range = ((RangePartitionItem) item).getItems();
+ mvRanges.put(range.lowerEndpoint(), range);
+ }
+
+ Map<List<String>, Set<String>> expanded = Maps.newHashMap();
+ for (MTMVRelatedTableIf pctTable : pctTables) {
+ List<String> qualifiers = pctTable.getFullQualifiers();
+ Set<String> queryUsedPartitions =
queryUsedBaseTablePartitionMap.get(qualifiers);
+ if (queryUsedPartitions == null) {
+ continue;
+ }
+
+ Optional<MvccSnapshot> snapshot =
MvccUtil.getSnapshotFromContext(pctTable);
+ if (pctTable.getPartitionType(snapshot) != PartitionType.RANGE) {
+ expanded.put(qualifiers, queryUsedPartitions);
+ continue;
+ }
+
+ Map<String, PartitionItem> basePartitionItems =
pctTable.getAndCopyPartitionItems(snapshot);
+
+ NavigableMap<PartitionKey, Range<PartitionKey>> relevantMvRanges =
new TreeMap<>();
+ for (String queriedBasePartition : queryUsedPartitions) {
+ PartitionItem baseItem =
basePartitionItems.get(queriedBasePartition);
+ if (baseItem == null) {
+ continue;
+ }
+ Range<PartitionKey> baseRange = ((RangePartitionItem)
baseItem).getItems();
+ Range<PartitionKey> mvRange = findEnclosingRange(mvRanges,
baseRange);
+ if (mvRange != null) {
+ relevantMvRanges.put(mvRange.lowerEndpoint(), mvRange);
+ }
+ }
+
+ if (relevantMvRanges.isEmpty()) {
+ expanded.put(qualifiers, Sets.newHashSet());
+ continue;
+ }
+
+ Set<String> expandedPartitions = Sets.newHashSet();
+ for (Entry<String, PartitionItem> baseEntry :
basePartitionItems.entrySet()) {
+ Range<PartitionKey> baseRange = ((RangePartitionItem)
baseEntry.getValue()).getItems();
+ if (findEnclosingRange(relevantMvRanges, baseRange) != null) {
+ expandedPartitions.add(baseEntry.getKey());
+ }
+ }
+
+ expanded.put(qualifiers, expandedPartitions);
+ }
+
+ return expanded;
+ }
+
+ private static Range<PartitionKey> findEnclosingRange(
+ NavigableMap<PartitionKey, Range<PartitionKey>> ranges,
Range<PartitionKey> baseRange) {
+ // RANGE partitions do not overlap, so only the range with the nearest
lower endpoint can enclose baseRange.
+ Entry<PartitionKey, Range<PartitionKey>> candidate =
ranges.floorEntry(baseRange.lowerEndpoint());
+ return candidate != null && candidate.getValue().encloses(baseRange) ?
candidate.getValue() : null;
+ }
+
+ private MTMVPartitionExpander() {
+ }
+}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java
b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java
index d2bcfc4ca69..700ab47fa44 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java
@@ -52,6 +52,7 @@ import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -136,7 +137,7 @@ public class MTMVPartitionUtil {
public static Pair<List<String>, List<PartitionKeyDesc>>
alignMvPartition(MTMV mtmv) throws AnalysisException {
Map<String, PartitionKeyDesc> mtmvPartitionDescs =
mtmv.generateMvPartitionDescs();
Set<PartitionKeyDesc> relatedPartitionDescs =
generateRelatedPartitionDescs(mtmv.getMvPartitionInfo(),
- mtmv.getMvProperties(), mtmv.getPartitionColumns()).keySet();
+ mtmv.getMvProperties(), mtmv.getPartitionColumns(),
Maps.newHashMap()).keySet();
List<String> partitionsToDrop = new ArrayList<>();
List<PartitionKeyDesc> partitionsToAdd = new ArrayList<>();
// drop partition of mtmv
@@ -171,7 +172,7 @@ public class MTMVPartitionUtil {
List<AllPartitionDesc> res = Lists.newArrayList();
HashMap<String, String> partitionProperties = Maps.newHashMap();
Set<PartitionKeyDesc> relatedPartitionDescs =
generateRelatedPartitionDescs(mvPartitionInfo, mvProperties,
- partitionColumns)
+ partitionColumns, Maps.newHashMap())
.keySet();
for (PartitionKeyDesc partitionKeyDesc : relatedPartitionDescs) {
SinglePartitionDesc singlePartitionDesc = new
SinglePartitionDesc(true,
@@ -184,13 +185,24 @@ public class MTMVPartitionUtil {
return res;
}
+ /**
+ * generateRelatedPartitionDescs
+ *
+ * @param mvPartitionInfo materialized view mvPartitionInfo
+ * @param mvProperties materialized view mvProperties when created
+ * @param partitionColumns materialized view partition columns
+ * @param queryUsedPartitions partitions current query used
+ * @return map of mv related table partition descs
+ * @throws AnalysisException
+ */
public static Map<PartitionKeyDesc, Map<MTMVRelatedTableIf, Set<String>>>
generateRelatedPartitionDescs(
MTMVPartitionInfo mvPartitionInfo,
- Map<String, String> mvProperties, List<Column> partitionColumns)
throws AnalysisException {
+ Map<String, String> mvProperties, List<Column> partitionColumns,
+ Map<List<String>, Set<String>> queryUsedPartitions) throws
AnalysisException {
long start = System.currentTimeMillis();
RelatedPartitionDescResult result = new RelatedPartitionDescResult();
for (MTMVRelatedPartitionDescGeneratorService service :
partitionDescGenerators) {
- service.apply(mvPartitionInfo, mvProperties, result,
partitionColumns);
+ service.apply(mvPartitionInfo, mvProperties, result,
partitionColumns, queryUsedPartitions);
}
if (LOG.isDebugEnabled()) {
LOG.debug("generateRelatedPartitionDescs use [{}] mills,
mvPartitionInfo is [{}]",
@@ -211,8 +223,8 @@ public class MTMVPartitionUtil {
return false;
}
try {
- return isMTMVSync(MTMVRefreshContext.buildContext(mtmv),
mtmvRelation.getBaseTablesOneLevelAndFromView(),
- Sets.newHashSet());
+ return isMTMVSync(MTMVRefreshContext.buildContext(mtmv,
Maps.newHashMap()),
+ mtmvRelation.getBaseTablesOneLevelAndFromView(),
Sets.newHashSet());
} catch (AnalysisException e) {
LOG.warn("isMTMVSync failed: ", e);
return false;
@@ -253,7 +265,7 @@ public class MTMVPartitionUtil {
throws AnalysisException {
List<Long> partitionIds = mtmv.getPartitionIds();
Map<Long, List<String>> res = Maps.newHashMap();
- MTMVRefreshContext context = MTMVRefreshContext.buildContext(mtmv);
+ MTMVRefreshContext context = MTMVRefreshContext.buildContext(mtmv,
Maps.newHashMap());
for (Long partitionId : partitionIds) {
String partitionName =
mtmv.getPartitionOrAnalysisException(partitionId).getName();
res.put(partitionId, getPartitionUnSyncTables(context,
partitionName));
@@ -632,22 +644,34 @@ public class MTMVPartitionUtil {
throw new AnalysisException("can not getPartitionColumnType by:" +
col);
}
- public static MTMVBaseVersions getBaseVersions(MTMV mtmv) throws
AnalysisException {
- return new MTMVBaseVersions(getTableVersions(mtmv),
getPartitionVersions(mtmv));
+ public static MTMVBaseVersions getBaseVersions(MTMV mtmv,
+ Map<String, Map<MTMVRelatedTableIf, Set<String>>>
partitionMappings) throws AnalysisException {
+ return new MTMVBaseVersions(getTableVersions(mtmv),
getPartitionVersions(mtmv, partitionMappings));
}
- private static Map<MTMVRelatedTableIf, Map<String, Long>>
getPartitionVersions(MTMV mtmv) throws AnalysisException {
+ private static Map<MTMVRelatedTableIf, Map<String, Long>>
getPartitionVersions(MTMV mtmv,
+ Map<String, Map<MTMVRelatedTableIf, Set<String>>>
partitionMappings) throws AnalysisException {
Map<MTMVRelatedTableIf, Map<String, Long>> res = Maps.newHashMap();
if
(mtmv.getMvPartitionInfo().getPartitionType().equals(MTMVPartitionType.SELF_MANAGE))
{
return res;
}
+ Map<MTMVRelatedTableIf, Set<String>> mappedPartitionNames =
Maps.newHashMap();
+ for (Map<MTMVRelatedTableIf, Set<String>> mapping :
partitionMappings.values()) {
+ for (Entry<MTMVRelatedTableIf, Set<String>> entry :
mapping.entrySet()) {
+ mappedPartitionNames.computeIfAbsent(entry.getKey(), key ->
Sets.newHashSet())
+ .addAll(entry.getValue());
+ }
+ }
Set<MTMVRelatedTableIf> pctTables =
mtmv.getMvPartitionInfo().getPctTables();
for (MTMVRelatedTableIf pctTable : pctTables) {
if (!(pctTable instanceof OlapTable)) {
continue;
}
Map<String, Long> onePctResult = Maps.newHashMap();
- List<Partition> partitions = Lists.newArrayList(((OlapTable)
pctTable).getPartitions());
+ List<Partition> partitions = Lists.newArrayList();
+ for (String partitionName :
mappedPartitionNames.getOrDefault(pctTable, Collections.emptySet())) {
+ partitions.add(((OlapTable)
pctTable).getPartitionOrAnalysisException(partitionName));
+ }
List<Long> versions = null;
try {
versions = Partition.getVisibleVersions(partitions);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRefreshContext.java
b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRefreshContext.java
index 603f89bbec5..683d2f353c4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRefreshContext.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRefreshContext.java
@@ -22,6 +22,7 @@ import org.apache.doris.common.AnalysisException;
import com.google.common.collect.Maps;
+import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -58,10 +59,11 @@ public class MTMVRefreshContext {
return baseTableSnapshotCache;
}
- public static MTMVRefreshContext buildContext(MTMV mtmv) throws
AnalysisException {
+ public static MTMVRefreshContext buildContext(MTMV mtmv, Map<List<String>,
Set<String>> queryUsedPartitions)
+ throws AnalysisException {
MTMVRefreshContext context = new MTMVRefreshContext(mtmv);
- context.partitionMappings = mtmv.calculatePartitionMappings();
- context.baseVersions = MTMVPartitionUtil.getBaseVersions(mtmv);
+ context.partitionMappings =
mtmv.calculatePartitionMappings(queryUsedPartitions);
+ context.baseVersions = MTMVPartitionUtil.getBaseVersions(mtmv,
context.partitionMappings);
return context;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescGeneratorService.java
b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescGeneratorService.java
index 5760995ac64..519cb54c20d 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescGeneratorService.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescGeneratorService.java
@@ -22,6 +22,7 @@ import org.apache.doris.common.AnalysisException;
import java.util.List;
import java.util.Map;
+import java.util.Set;
/**
* Interface for a series of processes to generate PartitionDesc
@@ -37,5 +38,6 @@ public interface MTMVRelatedPartitionDescGeneratorService {
* @throws AnalysisException
*/
void apply(MTMVPartitionInfo mvPartitionInfo, Map<String, String>
mvProperties,
- RelatedPartitionDescResult lastResult, List<Column>
partitionColumns) throws AnalysisException;
+ RelatedPartitionDescResult lastResult, List<Column>
partitionColumns,
+ Map<List<String>, Set<String>> queryUsedPartitionMap) throws
AnalysisException;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescInitGenerator.java
b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescInitGenerator.java
index 82d45f55288..5d5b161ed5f 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescInitGenerator.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescInitGenerator.java
@@ -35,8 +35,10 @@ public class MTMVRelatedPartitionDescInitGenerator
implements MTMVRelatedPartiti
@Override
public void apply(MTMVPartitionInfo mvPartitionInfo, Map<String, String>
mvProperties,
- RelatedPartitionDescResult lastResult, List<Column>
partitionColumns) throws AnalysisException {
+ RelatedPartitionDescResult lastResult, List<Column>
partitionColumns,
+ Map<List<String>, Set<String>> queryUsedPartitionMap)
throws AnalysisException {
Set<MTMVRelatedTableIf> relatedTables = mvPartitionInfo.getPctTables();
+ // the key is related table, the value is partition items of the
related table
Map<MTMVRelatedTableIf, Map<String, PartitionItem>> items =
Maps.newHashMap();
for (MTMVRelatedTableIf relatedTable : relatedTables) {
items.put(relatedTable,
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescOnePartitionColGenerator.java
b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescOnePartitionColGenerator.java
index 22148d0090f..21b3a21f346 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescOnePartitionColGenerator.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescOnePartitionColGenerator.java
@@ -49,16 +49,22 @@ public class
MTMVRelatedPartitionDescOnePartitionColGenerator implements MTMVRel
@Override
public void apply(MTMVPartitionInfo mvPartitionInfo, Map<String, String>
mvProperties,
- RelatedPartitionDescResult lastResult, List<Column>
partitionColumns) throws AnalysisException {
+ RelatedPartitionDescResult lastResult, List<Column>
partitionColumns,
+ Map<List<String>, Set<String>> queryUsedPartitionMap)
throws AnalysisException {
if (mvPartitionInfo.getPartitionType() ==
MTMVPartitionType.SELF_MANAGE) {
return;
}
Map<MTMVRelatedTableIf, Map<PartitionKeyDesc, Set<String>>> res =
Maps.newHashMap();
+ // the key is mv related table, the value is map<partition name,
partition item>
Map<MTMVRelatedTableIf, Map<String, PartitionItem>>
relatedPartitionItems = lastResult.getItems();
for (Entry<MTMVRelatedTableIf, Map<String, PartitionItem>> entry :
relatedPartitionItems.entrySet()) {
int relatedColPos = mvPartitionInfo.getPctColPos(entry.getKey());
Map<PartitionKeyDesc, Set<String>> onePctRes = Maps.newHashMap();
+ Set<String> queryUsedPartitions =
queryUsedPartitionMap.get(entry.getKey().getFullQualifiers());
for (Entry<String, PartitionItem> onePctEntry :
entry.getValue().entrySet()) {
+ if (queryUsedPartitions != null &&
!queryUsedPartitions.contains(onePctEntry.getKey())) {
+ continue;
+ }
PartitionKeyDesc partitionKeyDesc =
onePctEntry.getValue().toPartitionKeyDesc(relatedColPos);
if (onePctRes.containsKey(partitionKeyDesc)) {
onePctRes.get(partitionKeyDesc).add(onePctEntry.getKey());
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescRollUpGenerator.java
b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescRollUpGenerator.java
index e20910fb571..cb95fd67840 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescRollUpGenerator.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescRollUpGenerator.java
@@ -42,7 +42,8 @@ public class MTMVRelatedPartitionDescRollUpGenerator
implements MTMVRelatedParti
@Override
public void apply(MTMVPartitionInfo mvPartitionInfo, Map<String, String>
mvProperties,
- RelatedPartitionDescResult lastResult, List<Column>
partitionColumns) throws AnalysisException {
+ RelatedPartitionDescResult lastResult, List<Column>
partitionColumns,
+ Map<List<String>, Set<String>> queryUsedPartitionMap)
throws AnalysisException {
if (mvPartitionInfo.getPartitionType() != MTMVPartitionType.EXPR) {
return;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescSyncLimitGenerator.java
b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescSyncLimitGenerator.java
index b66cf822516..3b48165ec41 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescSyncLimitGenerator.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescSyncLimitGenerator.java
@@ -37,6 +37,7 @@ import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Optional;
+import java.util.Set;
/**
* Only focus on partial partitions of related tables
@@ -45,7 +46,8 @@ public class MTMVRelatedPartitionDescSyncLimitGenerator
implements MTMVRelatedPa
@Override
public void apply(MTMVPartitionInfo mvPartitionInfo, Map<String, String>
mvProperties,
- RelatedPartitionDescResult lastResult, List<Column>
partitionColumns) throws AnalysisException {
+ RelatedPartitionDescResult lastResult, List<Column>
partitionColumns,
+ Map<List<String>, Set<String>> queryUsedPartitionMap)
throws AnalysisException {
Map<MTMVRelatedTableIf, Map<String, PartitionItem>> partitionItems =
lastResult.getItems();
MTMVPartitionSyncConfig config =
generateMTMVPartitionSyncConfigByProperties(mvProperties);
if (config.getSyncLimit() <= 0) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescTransferGenerator.java
b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescTransferGenerator.java
index 659889e11ca..06a88f03397 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescTransferGenerator.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescTransferGenerator.java
@@ -44,7 +44,8 @@ public class MTMVRelatedPartitionDescTransferGenerator
implements MTMVRelatedPar
@Override
public void apply(MTMVPartitionInfo mvPartitionInfo, Map<String, String>
mvProperties,
- RelatedPartitionDescResult lastResult, List<Column>
partitionColumns) throws AnalysisException {
+ RelatedPartitionDescResult lastResult, List<Column>
partitionColumns,
+ Map<List<String>, Set<String>> queryUsedPartitionMap)
throws AnalysisException {
Map<MTMVRelatedTableIf, Map<PartitionKeyDesc, Set<String>>> descs =
lastResult.getDescs();
Map<PartitionKeyDesc, Map<MTMVRelatedTableIf, Set<String>>> res =
Maps.newHashMap();
for (Entry<MTMVRelatedTableIf, Map<PartitionKeyDesc, Set<String>>>
entry : descs.entrySet()) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java
b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java
index 6dc0bd24da7..40414e9d641 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java
@@ -75,7 +75,8 @@ public class MTMVRewriteUtil {
}
if (refreshContext == null) {
try {
- refreshContext = MTMVRefreshContext.buildContext(mtmv);
+ refreshContext = MTMVRefreshContext.buildContext(mtmv,
+ queryUsedPartitions != null ? queryUsedPartitions
: Maps.newHashMap());
} catch (AnalysisException e) {
LOG.warn("buildContext failed", e);
// After failure, one should quickly return to avoid
repeated failures
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AsyncMaterializationContext.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AsyncMaterializationContext.java
index 20d28036fcb..e936c04c803 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AsyncMaterializationContext.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AsyncMaterializationContext.java
@@ -40,6 +40,7 @@ import com.google.common.collect.Multimap;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
+import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
@@ -56,6 +57,7 @@ public class AsyncMaterializationContext extends
MaterializationContext {
private static final Logger LOG =
LogManager.getLogger(AsyncMaterializationContext.class);
private final MTMV mtmv;
private Map<MTMVRelatedTableIf, Map<String, Set<String>>>
partitionMultiFlatMap;
+ private final Map<List<String>, Set<String>>
coveredQueryUsedBaseTablePartitionMap = new HashMap<>();
/**
* MaterializationContext, this contains necessary info for query
rewriting by mv
@@ -184,12 +186,19 @@ public class AsyncMaterializationContext extends
MaterializationContext {
/**
* Calculate partition mappings and cache
*/
- public Map<MTMVRelatedTableIf, Map<String, Set<String>>>
calculatePartitionMappings() throws AnalysisException {
- if (partitionMultiFlatMap != null) {
+ public Map<MTMVRelatedTableIf, Map<String, Set<String>>>
calculatePartitionMappings(
+ Map<List<String>, Set<String>> queryUsedBaseTablePartitionMap)
throws AnalysisException {
+ Map<List<String>, Set<String>> effectiveQueryUsedBaseTablePartitionMap
+ =
mtmv.getEffectiveQueryUsedBaseTablePartitionMap(queryUsedBaseTablePartitionMap);
+ Set<MTMVRelatedTableIf> pctTables =
mtmv.getMvPartitionInfo().getPctTables();
+ if
(isPartitionMappingsCovered(effectiveQueryUsedBaseTablePartitionMap,
pctTables)) {
return partitionMultiFlatMap;
}
- partitionMultiFlatMap = new HashMap<>();
- Map<String, Map<MTMVRelatedTableIf, Set<String>>> partitionMultiMap =
this.mtmv.calculatePartitionMappings();
+ if (partitionMultiFlatMap == null) {
+ partitionMultiFlatMap = new HashMap<>();
+ }
+ Map<String, Map<MTMVRelatedTableIf, Set<String>>> partitionMultiMap
+ =
this.mtmv.calculatePartitionMappings(queryUsedBaseTablePartitionMap);
for (Map.Entry<String, Map<MTMVRelatedTableIf, Set<String>>> entry :
partitionMultiMap.entrySet()) {
String partitionKey = entry.getKey();
Map<MTMVRelatedTableIf, Set<String>> tableMap = entry.getValue();
@@ -202,6 +211,58 @@ public class AsyncMaterializationContext extends
MaterializationContext {
.addAll(set);
}
}
+ mergeCoveredPartitions(effectiveQueryUsedBaseTablePartitionMap,
pctTables);
return partitionMultiFlatMap;
}
+
+ private boolean isPartitionMappingsCovered(Map<List<String>, Set<String>>
effectiveQueryUsedBaseTablePartitionMap,
+ Set<MTMVRelatedTableIf> pctTables) {
+ if (partitionMultiFlatMap == null) {
+ return false;
+ }
+ for (MTMVRelatedTableIf pctTable : pctTables) {
+ List<String> tableQualifiers = pctTable.getFullQualifiers();
+ Set<String> queryUsedPartitions =
effectiveQueryUsedBaseTablePartitionMap.containsKey(tableQualifiers)
+ ?
effectiveQueryUsedBaseTablePartitionMap.get(tableQualifiers)
+ : null;
+ if (!isTableCovered(tableQualifiers, queryUsedPartitions)) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ private boolean isTableCovered(List<String> tableQualifiers, Set<String>
queryUsedPartitions) {
+ if
(!coveredQueryUsedBaseTablePartitionMap.containsKey(tableQualifiers)) {
+ return false;
+ }
+ Set<String> coveredPartitions =
coveredQueryUsedBaseTablePartitionMap.get(tableQualifiers);
+ if (coveredPartitions == null) {
+ return true;
+ }
+ return queryUsedPartitions != null &&
coveredPartitions.containsAll(queryUsedPartitions);
+ }
+
+ private void mergeCoveredPartitions(Map<List<String>, Set<String>>
effectiveQueryUsedBaseTablePartitionMap,
+ Set<MTMVRelatedTableIf> pctTables) {
+ for (MTMVRelatedTableIf pctTable : pctTables) {
+ List<String> tableQualifiers = pctTable.getFullQualifiers();
+ Set<String> queryUsedPartitions =
effectiveQueryUsedBaseTablePartitionMap.containsKey(tableQualifiers)
+ ?
effectiveQueryUsedBaseTablePartitionMap.get(tableQualifiers)
+ : null;
+ List<String> copiedQualifiers = new ArrayList<>(tableQualifiers);
+ if (queryUsedPartitions == null) {
+ coveredQueryUsedBaseTablePartitionMap.put(copiedQualifiers,
null);
+ continue;
+ }
+ if
(!coveredQueryUsedBaseTablePartitionMap.containsKey(copiedQualifiers)) {
+ coveredQueryUsedBaseTablePartitionMap.put(copiedQualifiers,
new HashSet<>(queryUsedPartitions));
+ continue;
+ }
+ Set<String> coveredPartitions =
coveredQueryUsedBaseTablePartitionMap.get(copiedQualifiers);
+ if (coveredPartitions != null) {
+ coveredPartitions.addAll(queryUsedPartitions);
+ }
+ }
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java
index 7b07130aa59..a8613cf3dc6 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java
@@ -102,7 +102,7 @@ public class PartitionCompensator {
rewrittenPlanUsePartitionNameSet.add(olapScan.getTable().getPartition(id).getName()));
}
Map<MTMVRelatedTableIf, Map<String, Set<String>>>
mtmvRelatedTableIfMapMap
- = materializationContext.calculatePartitionMappings();
+ =
materializationContext.calculatePartitionMappings(queryUsedBaseTablePartitionMap);
boolean allCompensateIsNull = true;
Map<BaseTableInfo, Set<String>> mvPartitionNeedRemoveNameMap = new
HashMap<>();
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVExpandPartitionTest.java
b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVExpandPartitionTest.java
new file mode 100644
index 00000000000..145de3f4e57
--- /dev/null
+++
b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVExpandPartitionTest.java
@@ -0,0 +1,269 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.mtmv;
+
+import org.apache.doris.analysis.PartitionValue;
+import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.PartitionItem;
+import org.apache.doris.catalog.PartitionKey;
+import org.apache.doris.catalog.PartitionType;
+import org.apache.doris.catalog.PrimitiveType;
+import org.apache.doris.catalog.RangePartitionItem;
+import org.apache.doris.catalog.ScalarType;
+import org.apache.doris.common.AnalysisException;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Range;
+import com.google.common.collect.Sets;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.lang.reflect.InvocationHandler;
+import java.lang.reflect.Proxy;
+import java.time.LocalDate;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Tests for {@link MTMVPartitionExpander#expandToMvPartitionGranularity}.
+ * Scenario: base table has daily RANGE partitions, MV has monthly RANGE
partitions.
+ * The expansion should map a queried daily partition to ALL daily partitions
within
+ * the same MV (monthly) partition range, ensuring complete partition mappings
for
+ * isSyncWithPartitions correctness while reducing the number of partitions
processed
+ * by the rollup pipeline.
+ * Uses Java dynamic proxy to create lightweight MTMVRelatedTableIf mocks
+ * without triggering class-loading of the full TableIf hierarchy.
+ */
+public class MTMVExpandPartitionTest {
+
+ private static final Column DATE_COL = new Column("c1",
ScalarType.createType(PrimitiveType.DATE),
+ true, null, "", "");
+
+ private static final List<String> RANGE_TABLE_QUALIFIERS =
Lists.newArrayList("internal", "db", "daily_base");
+ private static final List<String> LIST_TABLE_QUALIFIERS =
Lists.newArrayList("internal", "db", "list_base");
+
+ private MTMVRelatedTableIf rangeTable;
+ private MTMVRelatedTableIf listTable;
+
+ private Map<String, PartitionItem> dailyBasePartitions;
+ private Map<String, PartitionItem> monthlyMvPartitions;
+
+ @Before
+ public void setUp() throws Exception {
+ dailyBasePartitions = Maps.newHashMap();
+ dailyBasePartitions.put("p20210101", buildRange("2021-01-01",
"2021-01-02"));
+ dailyBasePartitions.put("p20210102", buildRange("2021-01-02",
"2021-01-03"));
+ dailyBasePartitions.put("p20210103", buildRange("2021-01-03",
"2021-01-04"));
+ dailyBasePartitions.put("p20210201", buildRange("2021-02-01",
"2021-02-02"));
+ dailyBasePartitions.put("p20210202", buildRange("2021-02-02",
"2021-02-03"));
+
+ monthlyMvPartitions = Maps.newHashMap();
+ monthlyMvPartitions.put("mv_202101", buildRange("2021-01-01",
"2021-02-01"));
+ monthlyMvPartitions.put("mv_202102", buildRange("2021-02-01",
"2021-03-01"));
+
+ rangeTable = createMockTable(RANGE_TABLE_QUALIFIERS,
PartitionType.RANGE, dailyBasePartitions);
+ listTable = createMockTable(LIST_TABLE_QUALIFIERS, PartitionType.LIST,
Maps.newHashMap());
+ }
+
+ @Test
+ public void testExpandSinglePartitionToMonth() throws Exception {
+ Map<List<String>, Set<String>> queryUsed = Maps.newHashMap();
+ queryUsed.put(RANGE_TABLE_QUALIFIERS, Sets.newHashSet("p20210102"));
+
+ Map<List<String>, Set<String>> result =
MTMVPartitionExpander.expandToMvPartitionGranularity(
+ queryUsed, monthlyMvPartitions, Sets.newHashSet(rangeTable));
+
+ Set<String> expanded = result.get(RANGE_TABLE_QUALIFIERS);
+ Assert.assertNotNull(expanded);
+ Assert.assertEquals(Sets.newHashSet("p20210101", "p20210102",
"p20210103"), expanded);
+ }
+
+ @Test
+ public void testExpandMultipleMonths() throws Exception {
+ Map<List<String>, Set<String>> queryUsed = Maps.newHashMap();
+ queryUsed.put(RANGE_TABLE_QUALIFIERS, Sets.newHashSet("p20210101",
"p20210202"));
+
+ Map<List<String>, Set<String>> result =
MTMVPartitionExpander.expandToMvPartitionGranularity(
+ queryUsed, monthlyMvPartitions, Sets.newHashSet(rangeTable));
+
+ Set<String> expanded = result.get(RANGE_TABLE_QUALIFIERS);
+ Assert.assertNotNull(expanded);
+ Assert.assertEquals(
+ Sets.newHashSet("p20210101", "p20210102", "p20210103",
"p20210201", "p20210202"),
+ expanded);
+ }
+
+ @Test
+ public void testMultiplePartitionsSelectSameMvRange() throws Exception {
+ Map<List<String>, Set<String>> queryUsed = Maps.newHashMap();
+ queryUsed.put(RANGE_TABLE_QUALIFIERS, Sets.newHashSet("p20210101",
"p20210102"));
+
+ Map<List<String>, Set<String>> result =
MTMVPartitionExpander.expandToMvPartitionGranularity(
+ queryUsed, monthlyMvPartitions, Sets.newHashSet(rangeTable));
+
+ Assert.assertEquals(Sets.newHashSet("p20210101", "p20210102",
"p20210103"),
+ result.get(RANGE_TABLE_QUALIFIERS));
+ }
+
+ @Test
+ public void testRangeGapDoesNotMatch() throws Exception {
+ Map<String, PartitionItem> mvPartitionsWithGap = Maps.newHashMap();
+ mvPartitionsWithGap.put("mv_202101", buildRange("2021-01-01",
"2021-02-01"));
+ mvPartitionsWithGap.put("mv_202103", buildRange("2021-03-01",
"2021-04-01"));
+ Map<List<String>, Set<String>> queryUsed = Maps.newHashMap();
+ queryUsed.put(RANGE_TABLE_QUALIFIERS, Sets.newHashSet("p20210201"));
+
+ Map<List<String>, Set<String>> result =
MTMVPartitionExpander.expandToMvPartitionGranularity(
+ queryUsed, mvPartitionsWithGap, Sets.newHashSet(rangeTable));
+
+ Assert.assertTrue(result.get(RANGE_TABLE_QUALIFIERS).isEmpty());
+ }
+
+ @Test
+ public void testManyPartitionsSparseAndBroadFilters() throws Exception {
+ Map<String, PartitionItem> basePartitions = Maps.newHashMap();
+ Map<String, PartitionItem> mvPartitions = Maps.newHashMap();
+ LocalDate start = LocalDate.of(2021, 1, 1);
+ for (int i = 0; i < 360; i++) {
+ basePartitions.put("p" + i,
buildRange(start.plusDays(i).toString(),
+ start.plusDays(i + 1).toString()));
+ }
+ for (int i = 0; i < 12; i++) {
+ mvPartitions.put("mv" + i, buildRange(start.plusDays(i *
30L).toString(),
+ start.plusDays((i + 1) * 30L).toString()));
+ }
+ MTMVRelatedTableIf manyPartitionTable = createMockTable(
+ RANGE_TABLE_QUALIFIERS, PartitionType.RANGE, basePartitions);
+ Map<List<String>, Set<String>> sparseFilter = Maps.newHashMap();
+ sparseFilter.put(RANGE_TABLE_QUALIFIERS, Sets.newHashSet("p45"));
+
+ Map<List<String>, Set<String>> sparseResult =
MTMVPartitionExpander.expandToMvPartitionGranularity(
+ sparseFilter, mvPartitions,
Sets.newHashSet(manyPartitionTable));
+
+ Assert.assertEquals(30,
sparseResult.get(RANGE_TABLE_QUALIFIERS).size());
+
Assert.assertTrue(sparseResult.get(RANGE_TABLE_QUALIFIERS).contains("p30"));
+
Assert.assertTrue(sparseResult.get(RANGE_TABLE_QUALIFIERS).contains("p59"));
+
+ Map<List<String>, Set<String>> broadFilter = Maps.newHashMap();
+ broadFilter.put(RANGE_TABLE_QUALIFIERS, basePartitions.keySet());
+ Map<List<String>, Set<String>> broadResult =
MTMVPartitionExpander.expandToMvPartitionGranularity(
+ broadFilter, mvPartitions,
Sets.newHashSet(manyPartitionTable));
+
+ Assert.assertEquals(basePartitions.keySet(),
broadResult.get(RANGE_TABLE_QUALIFIERS));
+ }
+
+ @Test
+ public void testListPartitionPassthrough() throws Exception {
+ Map<List<String>, Set<String>> queryUsed = Maps.newHashMap();
+ queryUsed.put(LIST_TABLE_QUALIFIERS, Sets.newHashSet("p1"));
+
+ Map<List<String>, Set<String>> result =
MTMVPartitionExpander.expandToMvPartitionGranularity(
+ queryUsed, monthlyMvPartitions, Sets.newHashSet(listTable));
+
+ Set<String> expanded = result.get(LIST_TABLE_QUALIFIERS);
+ Assert.assertNotNull(expanded);
+ Assert.assertEquals(Sets.newHashSet("p1"), expanded);
+ }
+
+ @Test
+ public void testNonExistentPartition() throws Exception {
+ Map<List<String>, Set<String>> queryUsed = Maps.newHashMap();
+ queryUsed.put(RANGE_TABLE_QUALIFIERS,
Sets.newHashSet("p_nonexistent"));
+
+ Map<List<String>, Set<String>> result =
MTMVPartitionExpander.expandToMvPartitionGranularity(
+ queryUsed, monthlyMvPartitions, Sets.newHashSet(rangeTable));
+
+ Set<String> expanded = result.get(RANGE_TABLE_QUALIFIERS);
+ Assert.assertNotNull(expanded);
+ Assert.assertTrue(expanded.isEmpty());
+ }
+
+ @Test
+ public void testBasePartitionOutsideMvRange() throws Exception {
+ Map<String, PartitionItem> janOnlyMv = Maps.newHashMap();
+ janOnlyMv.put("mv_202101", buildRange("2021-01-01", "2021-02-01"));
+
+ Map<List<String>, Set<String>> queryUsed = Maps.newHashMap();
+ queryUsed.put(RANGE_TABLE_QUALIFIERS, Sets.newHashSet("p20210101"));
+
+ Map<List<String>, Set<String>> result =
MTMVPartitionExpander.expandToMvPartitionGranularity(
+ queryUsed, janOnlyMv, Sets.newHashSet(rangeTable));
+
+ Set<String> expanded = result.get(RANGE_TABLE_QUALIFIERS);
+ Assert.assertNotNull(expanded);
+ Assert.assertEquals(Sets.newHashSet("p20210101", "p20210102",
"p20210103"), expanded);
+ }
+
+ @Test
+ public void testPctTableNotInFilter() throws Exception {
+ Map<List<String>, Set<String>> queryUsed = Maps.newHashMap();
+ queryUsed.put(Lists.newArrayList("internal", "db", "other_table"),
+ Sets.newHashSet("p20210101"));
+
+ Map<List<String>, Set<String>> result =
MTMVPartitionExpander.expandToMvPartitionGranularity(
+ queryUsed, monthlyMvPartitions, Sets.newHashSet(rangeTable));
+
+ Assert.assertNull(result.get(RANGE_TABLE_QUALIFIERS));
+ }
+
+ @Test
+ public void testEmptyFilter() throws Exception {
+ Map<List<String>, Set<String>> result =
MTMVPartitionExpander.expandToMvPartitionGranularity(
+ Maps.newHashMap(), monthlyMvPartitions,
Sets.newHashSet(rangeTable));
+
+ Assert.assertTrue(result.isEmpty());
+ }
+
+ // --- helpers ---
+
+ private static MTMVRelatedTableIf createMockTable(List<String> qualifiers,
+ PartitionType partitionType, Map<String, PartitionItem>
partitionItems) {
+ InvocationHandler handler = (proxy, method, args) -> {
+ switch (method.getName()) {
+ case "getFullQualifiers":
+ return qualifiers;
+ case "getPartitionType":
+ return partitionType;
+ case "getAndCopyPartitionItems":
+ return partitionItems;
+ case "hashCode":
+ return System.identityHashCode(proxy);
+ case "equals":
+ return proxy == args[0];
+ default:
+ throw new UnsupportedOperationException(
+ "MTMVExpandPartitionTest mock does not support: "
+ method.getName());
+ }
+ };
+ return (MTMVRelatedTableIf) Proxy.newProxyInstance(
+ MTMVRelatedTableIf.class.getClassLoader(),
+ new Class<?>[] {MTMVRelatedTableIf.class},
+ handler);
+ }
+
+ private static RangePartitionItem buildRange(String lower, String upper)
throws AnalysisException {
+ PartitionKey lowerKey = PartitionKey.createPartitionKey(
+ Lists.newArrayList(new PartitionValue(lower)),
Lists.newArrayList(DATE_COL));
+ PartitionKey upperKey = PartitionKey.createPartitionKey(
+ Lists.newArrayList(new PartitionValue(upper)),
Lists.newArrayList(DATE_COL));
+ return new RangePartitionItem(Range.closedOpen(lowerKey, upperKey));
+ }
+}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVPartitionUtilTest.java
b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVPartitionUtilTest.java
index 41ca743e76a..80f91de7912 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVPartitionUtilTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVPartitionUtilTest.java
@@ -68,7 +68,8 @@ public class MTMVPartitionUtilTest {
mtmvUtilStatic = Mockito.mockStatic(MTMVUtil.class);
refreshContextStatic = Mockito.mockStatic(MTMVRefreshContext.class);
- refreshContextStatic.when(() ->
MTMVRefreshContext.buildContext(Mockito.any(MTMV.class))).thenReturn(context);
+ refreshContextStatic.when(() ->
MTMVRefreshContext.buildContext(Mockito.any(MTMV.class), Mockito.anyMap()))
+ .thenReturn(context);
Mockito.when(mtmv.getRelation()).thenReturn(relation);
@@ -252,6 +253,34 @@ public class MTMVPartitionUtilTest {
Assert.assertFalse(MTMVPartitionUtil.isTableNamelike(new
TableNameInfo("ctl1"), tableNameToCheck));
}
+ @Test
+ public void testGetBaseVersionsUsesMappedPartitions() throws
AnalysisException {
+ Map<String, Map<MTMVRelatedTableIf, Set<String>>> partitionMappings =
Maps.newHashMap();
+ partitionMappings.put("mv1", pctMapping("p1"));
+
+ assertFetchedPartitionNames(partitionMappings, Sets.newHashSet("p1",
"p2", "p3"),
+ Sets.newHashSet("p1"));
+ }
+
+ @Test
+ public void testGetBaseVersionsDeduplicatesMappedPartitions() throws
AnalysisException {
+ Map<String, Map<MTMVRelatedTableIf, Set<String>>> partitionMappings =
Maps.newHashMap();
+ partitionMappings.put("mv1", pctMapping("p1", "p2"));
+ partitionMappings.put("mv2", pctMapping("p2", "p3"));
+
+ assertFetchedPartitionNames(partitionMappings, Sets.newHashSet("p1",
"p2", "p3", "p4"),
+ Sets.newHashSet("p1", "p2", "p3"));
+ }
+
+ @Test
+ public void testGetBaseVersionsUsesAllFullyMappedPartitions() throws
AnalysisException {
+ Map<String, Map<MTMVRelatedTableIf, Set<String>>> partitionMappings =
Maps.newHashMap();
+ partitionMappings.put("mv1", pctMapping("p1", "p2", "p3"));
+
+ assertFetchedPartitionNames(partitionMappings, Sets.newHashSet("p1",
"p2", "p3"),
+ Sets.newHashSet("p1", "p2", "p3"));
+ }
+
@Test
public void testGetTableSnapshotFromContext() throws AnalysisException {
Map<BaseTableInfo, MTMVSnapshotIf> cache = Maps.newHashMap();
@@ -261,4 +290,51 @@ public class MTMVPartitionUtilTest {
Assert.assertEquals(1, cache.size());
Assert.assertEquals(baseSnapshotIf, cache.values().iterator().next());
}
+
+ private Map<MTMVRelatedTableIf, Set<String>> pctMapping(String...
partitionNames) {
+ Map<MTMVRelatedTableIf, Set<String>> mapping = Maps.newHashMap();
+ mapping.put(baseOlapTable, Sets.newHashSet(partitionNames));
+ return mapping;
+ }
+
+ private void assertFetchedPartitionNames(
+ Map<String, Map<MTMVRelatedTableIf, Set<String>>>
partitionMappings,
+ Set<String> allPartitionNames, Set<String> expectedPartitionNames)
throws AnalysisException {
+ Mockito.when(mtmv.getRelation()).thenReturn(null);
+
Mockito.when(mtmvPartitionInfo.getPartitionType()).thenReturn(MTMVPartitionType.FOLLOW_BASE_TABLE);
+
Mockito.when(mtmvPartitionInfo.getPctTables()).thenReturn(Sets.newHashSet(baseOlapTable));
+
+ Map<String, Partition> partitions = Maps.newHashMap();
+ long visibleVersion = 1;
+ for (String partitionName : allPartitionNames) {
+ Partition partition = Mockito.mock(Partition.class);
+ Mockito.when(partition.getName()).thenReturn(partitionName);
+
Mockito.when(partition.getVisibleVersion()).thenReturn(visibleVersion++);
+
Mockito.when(baseOlapTable.getPartitionOrAnalysisException(partitionName)).thenReturn(partition);
+ partitions.put(partitionName, partition);
+ }
+
Mockito.when(baseOlapTable.getPartitions()).thenReturn(partitions.values());
+
+ List<Set<String>> versionRequests = Lists.newArrayList();
+ try (MockedStatic<Partition> partitionStatic =
Mockito.mockStatic(Partition.class, Mockito.CALLS_REAL_METHODS)) {
+ partitionStatic.when(() ->
Partition.getVisibleVersions(Mockito.anyList())).thenAnswer(invocation -> {
+ List<? extends Partition> requestedPartitions =
invocation.getArgument(0);
+ Set<String> requestedPartitionNames = Sets.newHashSet();
+ List<Long> visibleVersions = Lists.newArrayList();
+ for (Partition partition : requestedPartitions) {
+ requestedPartitionNames.add(partition.getName());
+ visibleVersions.add(partition.getVisibleVersion());
+ }
+ versionRequests.add(requestedPartitionNames);
+ return visibleVersions;
+ });
+
+ Assert.assertEquals(expectedPartitionNames,
+ MTMVPartitionUtil.getBaseVersions(mtmv, partitionMappings)
+ .getPartitionVersions(baseOlapTable).keySet());
+ }
+ Assert.assertEquals(1, versionRequests.size());
+ Assert.assertEquals(expectedPartitionNames, versionRequests.get(0));
+ Mockito.verify(baseOlapTable, Mockito.never()).getPartitions();
+ }
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescGeneratorTest.java
b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescGeneratorTest.java
index d60f4be7f8f..ab158432866 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescGeneratorTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescGeneratorTest.java
@@ -32,6 +32,7 @@ import org.apache.doris.utframe.TestWithFeService;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
@@ -106,7 +107,7 @@ public class MTMVRelatedPartitionDescGeneratorTest extends
TestWithFeService {
Column c1Column = new Column("c1", PrimitiveType.DATE);
Map<PartitionKeyDesc, Map<MTMVRelatedTableIf, Set<String>>>
partitionKeyDescMap
=
MTMVPartitionUtil.generateRelatedPartitionDescs(mtmvPartitionInfo,
Maps.newHashMap(),
- Lists.newArrayList(c1Column));
+ Lists.newArrayList(c1Column), Maps.newHashMap());
// 3 partition
Assertions.assertEquals(3, partitionKeyDescMap.size());
OlapTable t1 = (OlapTable)
Env.getCurrentEnv().getInternalCatalog().getDbOrAnalysisException("test")
@@ -119,6 +120,40 @@ public class MTMVRelatedPartitionDescGeneratorTest extends
TestWithFeService {
}
}
+ @Test
+ public void testQueryUsedPartitionsRange() throws Exception {
+ MTMVPartitionInfo mtmvPartitionInfo =
getMTMVPartitionInfo(Lists.newArrayList("t1"));
+ Column c1Column = new Column("c1", PrimitiveType.DATE);
+ Map<List<String>, Set<String>> queryUsed = Maps.newHashMap();
+ queryUsed.put(Lists.newArrayList("internal", "test", "t1"),
Sets.newHashSet("p20210201"));
+
+ Map<PartitionKeyDesc, Map<MTMVRelatedTableIf, Set<String>>>
partitionKeyDescMap
+ =
MTMVPartitionUtil.generateRelatedPartitionDescs(mtmvPartitionInfo,
Maps.newHashMap(),
+ Lists.newArrayList(c1Column), queryUsed);
+ Assertions.assertEquals(1, partitionKeyDescMap.size());
+
+ Set<String> collected = Sets.newHashSet();
+ partitionKeyDescMap.values().forEach(m ->
m.values().forEach(collected::addAll));
+ Assertions.assertEquals(Sets.newHashSet("p20210201"), collected);
+ }
+
+ @Test
+ public void testQueryUsedPartitionsList() throws Exception {
+ MTMVPartitionInfo mtmvPartitionInfo =
getMTMVPartitionInfo(Lists.newArrayList("t2"));
+ Column c1Column = new Column("c1", PrimitiveType.INT);
+ Map<List<String>, Set<String>> queryUsed = Maps.newHashMap();
+ queryUsed.put(Lists.newArrayList("internal", "test", "t2"),
Sets.newHashSet("p1_bj"));
+
+ Map<PartitionKeyDesc, Map<MTMVRelatedTableIf, Set<String>>>
partitionKeyDescMap
+ =
MTMVPartitionUtil.generateRelatedPartitionDescs(mtmvPartitionInfo,
Maps.newHashMap(),
+ Lists.newArrayList(c1Column), queryUsed);
+ Assertions.assertEquals(1, partitionKeyDescMap.size());
+
+ Set<String> collected = Sets.newHashSet();
+ partitionKeyDescMap.values().forEach(m ->
m.values().forEach(collected::addAll));
+ Assertions.assertEquals(Sets.newHashSet("p1_bj"), collected);
+ }
+
@Test
public void testLimit() throws Exception {
MTMVPartitionInfo mtmvPartitionInfo =
getMTMVPartitionInfo(Lists.newArrayList("t1"));
@@ -127,7 +162,7 @@ public class MTMVRelatedPartitionDescGeneratorTest extends
TestWithFeService {
mvProperty.put(PropertyAnalyzer.PROPERTIES_PARTITION_SYNC_LIMIT, "1");
Map<PartitionKeyDesc, Map<MTMVRelatedTableIf, Set<String>>>
partitionKeyDescMap
=
MTMVPartitionUtil.generateRelatedPartitionDescs(mtmvPartitionInfo, mvProperty,
- Lists.newArrayList(c1Column));
+ Lists.newArrayList(c1Column), Maps.newHashMap());
// 3 partition
Assertions.assertEquals(1, partitionKeyDescMap.size());
OlapTable t1 = (OlapTable)
Env.getCurrentEnv().getInternalCatalog().getDbOrAnalysisException("test")
@@ -146,7 +181,7 @@ public class MTMVRelatedPartitionDescGeneratorTest extends
TestWithFeService {
Column c1Column = new Column("c1", PrimitiveType.INT);
Map<PartitionKeyDesc, Map<MTMVRelatedTableIf, Set<String>>>
partitionKeyDescMap
=
MTMVPartitionUtil.generateRelatedPartitionDescs(mtmvPartitionInfo,
Maps.newHashMap(),
- Lists.newArrayList(c1Column));
+ Lists.newArrayList(c1Column), Maps.newHashMap());
// 3 partition
Assertions.assertEquals(2, partitionKeyDescMap.size());
OlapTable t2 = (OlapTable)
Env.getCurrentEnv().getInternalCatalog().getDbOrAnalysisException("test")
@@ -172,7 +207,7 @@ public class MTMVRelatedPartitionDescGeneratorTest extends
TestWithFeService {
Column c1Column = new Column("c1", PrimitiveType.INT);
Map<PartitionKeyDesc, Map<MTMVRelatedTableIf, Set<String>>>
partitionKeyDescMap
=
MTMVPartitionUtil.generateRelatedPartitionDescs(mtmvPartitionInfo,
Maps.newHashMap(),
- Lists.newArrayList(c1Column));
+ Lists.newArrayList(c1Column), Maps.newHashMap());
// 2 partition
Assertions.assertEquals(2, partitionKeyDescMap.size());
OlapTable t1 = (OlapTable)
Env.getCurrentEnv().getInternalCatalog().getDbOrAnalysisException("test")
@@ -190,7 +225,7 @@ public class MTMVRelatedPartitionDescGeneratorTest extends
TestWithFeService {
Column c1Column = new Column("c1", PrimitiveType.DATE);
Assertions.assertThrows(AnalysisException.class,
() ->
MTMVPartitionUtil.generateRelatedPartitionDescs(mtmvPartitionInfo,
Maps.newHashMap(),
- Lists.newArrayList(c1Column)));
+ Lists.newArrayList(c1Column), Maps.newHashMap()));
}
@Test
@@ -199,7 +234,7 @@ public class MTMVRelatedPartitionDescGeneratorTest extends
TestWithFeService {
Column c1Column = new Column("c1", PrimitiveType.DATE);
Assertions.assertThrows(AnalysisException.class,
() ->
MTMVPartitionUtil.generateRelatedPartitionDescs(mtmvPartitionInfo,
Maps.newHashMap(),
- Lists.newArrayList(c1Column)));
+ Lists.newArrayList(c1Column), Maps.newHashMap()));
}
@Test
@@ -208,7 +243,7 @@ public class MTMVRelatedPartitionDescGeneratorTest extends
TestWithFeService {
Column c1Column = new Column("c1", PrimitiveType.DATE);
Map<PartitionKeyDesc, Map<MTMVRelatedTableIf, Set<String>>>
partitionKeyDescMap
=
MTMVPartitionUtil.generateRelatedPartitionDescs(mtmvPartitionInfo,
Maps.newHashMap(),
- Lists.newArrayList(c1Column));
+ Lists.newArrayList(c1Column), Maps.newHashMap());
// 4 partition
Assertions.assertEquals(4, partitionKeyDescMap.size());
boolean hasOne = false;
@@ -232,7 +267,7 @@ public class MTMVRelatedPartitionDescGeneratorTest extends
TestWithFeService {
Column c1Column = new Column("c1", PrimitiveType.INT);
Map<PartitionKeyDesc, Map<MTMVRelatedTableIf, Set<String>>>
partitionKeyDescMap
=
MTMVPartitionUtil.generateRelatedPartitionDescs(mtmvPartitionInfo,
Maps.newHashMap(),
- Lists.newArrayList(c1Column));
+ Lists.newArrayList(c1Column), Maps.newHashMap());
// 2 partition
Assertions.assertEquals(2, partitionKeyDescMap.size());
OlapTable t2 = (OlapTable)
Env.getCurrentEnv().getInternalCatalog().getDbOrAnalysisException("test")
@@ -252,7 +287,7 @@ public class MTMVRelatedPartitionDescGeneratorTest extends
TestWithFeService {
Column c1Column = new Column("c1", PrimitiveType.DATE);
Map<PartitionKeyDesc, Map<MTMVRelatedTableIf, Set<String>>>
partitionKeyDescMap
=
MTMVPartitionUtil.generateRelatedPartitionDescs(mtmvPartitionInfo,
Maps.newHashMap(),
- Lists.newArrayList(c1Column));
+ Lists.newArrayList(c1Column), Maps.newHashMap());
// 2 partition
Assertions.assertEquals(2, partitionKeyDescMap.size());
OlapTable t6 = (OlapTable)
Env.getCurrentEnv().getInternalCatalog().getDbOrAnalysisException("test")
@@ -272,7 +307,7 @@ public class MTMVRelatedPartitionDescGeneratorTest extends
TestWithFeService {
Column c1Column = new Column("c1", PrimitiveType.DATE);
Assertions.assertThrows(AnalysisException.class,
() ->
MTMVPartitionUtil.generateRelatedPartitionDescs(mtmvPartitionInfo,
Maps.newHashMap(),
- Lists.newArrayList(c1Column)));
+ Lists.newArrayList(c1Column), Maps.newHashMap()));
}
private MTMVPartitionInfo getMTMVPartitionInfo(List<String> pctTableNames)
throws AnalysisException {
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/AsyncMaterializationContextTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/AsyncMaterializationContextTest.java
new file mode 100644
index 00000000000..2490cead969
--- /dev/null
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/AsyncMaterializationContextTest.java
@@ -0,0 +1,136 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.exploration.mv;
+
+import org.apache.doris.catalog.MTMV;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.mtmv.MTMVPartitionInfo;
+import org.apache.doris.mtmv.MTMVRelatedTableIf;
+import org.apache.doris.nereids.CascadesContext;
+import org.apache.doris.nereids.StatementContext;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.plans.Plan;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Maps;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+import org.mockito.Mockito;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+public class AsyncMaterializationContextTest {
+
+ @Test
+ public void testCalculatePartitionMappingsMergesUncoveredFilters() throws
Exception {
+ List<String> qualifiers = ImmutableList.of("ctl", "db", "t1");
+ MTMVRelatedTableIf pctTable = mockPctTable(qualifiers);
+ MTMV mtmv = mockMtmv(pctTable);
+ AsyncMaterializationContext context = createContext(mtmv);
+
+ Map<List<String>, Set<String>> queryUsedPartitionsA =
Maps.newHashMap();
+ queryUsedPartitionsA.put(qualifiers, ImmutableSet.of("p1"));
+ Map<List<String>, Set<String>> queryUsedPartitionsB =
Maps.newHashMap();
+ queryUsedPartitionsB.put(qualifiers, ImmutableSet.of("p2"));
+
+
Mockito.when(mtmv.getEffectiveQueryUsedBaseTablePartitionMap(queryUsedPartitionsA))
+ .thenReturn(queryUsedPartitionsA);
+
Mockito.when(mtmv.getEffectiveQueryUsedBaseTablePartitionMap(queryUsedPartitionsB))
+ .thenReturn(queryUsedPartitionsB);
+ Mockito.when(mtmv.calculatePartitionMappings(queryUsedPartitionsA))
+ .thenReturn(buildPartitionMultiMap("mv_p1", pctTable, "p1"));
+ Mockito.when(mtmv.calculatePartitionMappings(queryUsedPartitionsB))
+ .thenReturn(buildPartitionMultiMap("mv_p2", pctTable, "p2"));
+
+ Map<MTMVRelatedTableIf, Map<String, Set<String>>> resultA
+ = context.calculatePartitionMappings(queryUsedPartitionsA);
+ Map<MTMVRelatedTableIf, Map<String, Set<String>>> resultB
+ = context.calculatePartitionMappings(queryUsedPartitionsB);
+
+ Assertions.assertEquals(ImmutableSet.of("p1"),
resultA.get(pctTable).get("mv_p1"));
+ Assertions.assertEquals(ImmutableSet.of("p1"),
resultB.get(pctTable).get("mv_p1"));
+ Assertions.assertEquals(ImmutableSet.of("p2"),
resultB.get(pctTable).get("mv_p2"));
+ Mockito.verify(mtmv).calculatePartitionMappings(queryUsedPartitionsA);
+ Mockito.verify(mtmv).calculatePartitionMappings(queryUsedPartitionsB);
+ }
+
+ @Test
+ public void testCalculatePartitionMappingsReusesCoverageAfterFullMapping()
throws Exception {
+ List<String> qualifiers = ImmutableList.of("ctl", "db", "t1");
+ MTMVRelatedTableIf pctTable = mockPctTable(qualifiers);
+ MTMV mtmv = mockMtmv(pctTable);
+ AsyncMaterializationContext context = createContext(mtmv);
+
+ Map<List<String>, Set<String>> noFilter = Maps.newHashMap();
+ Map<List<String>, Set<String>> queryUsedPartitions = Maps.newHashMap();
+ queryUsedPartitions.put(qualifiers, ImmutableSet.of("p1"));
+
+
Mockito.when(mtmv.getEffectiveQueryUsedBaseTablePartitionMap(noFilter)).thenReturn(noFilter);
+
Mockito.when(mtmv.getEffectiveQueryUsedBaseTablePartitionMap(queryUsedPartitions))
+ .thenReturn(queryUsedPartitions);
+ Mockito.when(mtmv.calculatePartitionMappings(noFilter))
+ .thenReturn(buildPartitionMultiMap("mv_p1", pctTable, "p1"));
+
+ Map<MTMVRelatedTableIf, Map<String, Set<String>>> resultA =
context.calculatePartitionMappings(noFilter);
+ Map<MTMVRelatedTableIf, Map<String, Set<String>>> resultB
+ = context.calculatePartitionMappings(queryUsedPartitions);
+
+ Assertions.assertSame(resultA, resultB);
+ Assertions.assertEquals(ImmutableSet.of("p1"),
resultB.get(pctTable).get("mv_p1"));
+ Mockito.verify(mtmv).calculatePartitionMappings(noFilter);
+ Mockito.verify(mtmv,
Mockito.never()).calculatePartitionMappings(queryUsedPartitions);
+ }
+
+ private static AsyncMaterializationContext createContext(MTMV mtmv) {
+ CascadesContext cascadesContext = Mockito.mock(CascadesContext.class);
+ StatementContext statementContext =
Mockito.mock(StatementContext.class);
+
Mockito.when(cascadesContext.getStatementContext()).thenReturn(statementContext);
+ StructInfo structInfo = Mockito.mock(StructInfo.class);
+
Mockito.doReturn(Collections.<Expression>emptyList()).when(structInfo).getPlanOutputShuttledExpressions();
+ Plan plan = Mockito.mock(Plan.class);
+ return new AsyncMaterializationContext(mtmv, plan, plan,
ImmutableList.of(), ImmutableList.of(),
+ cascadesContext, structInfo);
+ }
+
+ private static MTMV mockMtmv(MTMVRelatedTableIf pctTable) throws
AnalysisException {
+ MTMV mtmv = Mockito.mock(MTMV.class);
+ MTMVPartitionInfo mtmvPartitionInfo =
Mockito.mock(MTMVPartitionInfo.class);
+ Mockito.when(mtmv.getMvPartitionInfo()).thenReturn(mtmvPartitionInfo);
+
Mockito.when(mtmvPartitionInfo.getPctTables()).thenReturn(ImmutableSet.of(pctTable));
+ return mtmv;
+ }
+
+ private static MTMVRelatedTableIf mockPctTable(List<String> qualifiers) {
+ MTMVRelatedTableIf pctTable = Mockito.mock(MTMVRelatedTableIf.class);
+ Mockito.when(pctTable.getFullQualifiers()).thenReturn(qualifiers);
+ return pctTable;
+ }
+
+ private static Map<String, Map<MTMVRelatedTableIf, Set<String>>>
buildPartitionMultiMap(
+ String mvPartitionName, MTMVRelatedTableIf pctTable, String
basePartitionName) {
+ Map<String, Map<MTMVRelatedTableIf, Set<String>>> partitionMultiMap =
Maps.newHashMap();
+ Map<MTMVRelatedTableIf, Set<String>> relatedTableToPartitions =
Maps.newHashMap();
+ relatedTableToPartitions.put(pctTable,
ImmutableSet.of(basePartitionName));
+ partitionMultiMap.put(mvPartitionName, relatedTableToPartitions);
+ return partitionMultiMap;
+ }
+}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensatorTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensatorTest.java
index 2d6cb15864a..0ad6d8b49ed 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensatorTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensatorTest.java
@@ -449,7 +449,7 @@ public class PartitionCompensatorTest extends
TestWithFeService {
AsyncMaterializationContext matCtx =
Mockito.mock(AsyncMaterializationContext.class);
Mockito.when(matCtx.getMtmv()).thenReturn(mtmv);
-
Mockito.when(matCtx.calculatePartitionMappings()).thenReturn(partitionMappings);
+
Mockito.when(matCtx.calculatePartitionMappings(ArgumentMatchers.any())).thenReturn(partitionMappings);
// StatementContext: the MV's two valid partitions are available for
rewrite
Map<BaseTableInfo, Collection<Partition>> canRewriteMap = new
HashMap<>();
@@ -539,7 +539,7 @@ public class PartitionCompensatorTest extends
TestWithFeService {
AsyncMaterializationContext matCtx =
Mockito.mock(AsyncMaterializationContext.class);
Mockito.when(matCtx.getMtmv()).thenReturn(mtmv);
-
Mockito.when(matCtx.calculatePartitionMappings()).thenReturn(partitionMappings);
+
Mockito.when(matCtx.calculatePartitionMappings(ArgumentMatchers.any())).thenReturn(partitionMappings);
Map<BaseTableInfo, Collection<Partition>> canRewriteMap = new
HashMap<>();
canRewriteMap.put(new BaseTableInfo(mtmv),
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
