This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.0 by this push:
new 0cb509daa26 branch-4.0:[fix](mtmv) Fix nested mv rewritten fail when
bottom mv is partitioned (#57558) (#58277)
0cb509daa26 is described below
commit 0cb509daa260506e0820afc97dcad183a44c27de
Author: seawinde <[email protected]>
AuthorDate: Tue Nov 25 10:04:48 2025 +0800
branch-4.0:[fix](mtmv) Fix nested mv rewritten fail when bottom mv is
partitioned (#57558) (#58277)
pr: https://github.com/apache/doris/pull/57558
commitId:
https://github.com/apache/doris/commit/5e431415c9087ba55a14531109083a3cc2153587
### What problem does this PR solve?
Issue Number: close #xxx
Related PR: #xxx
Problem Summary:
### Release note
None
### Check List (For Author)
- Test <!-- At least one of them must be included. -->
- [ ] Regression test
- [ ] Unit Test
- [ ] Manual test (add detailed scripts or steps below)
- [ ] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
- [ ] Previous test can cover this change.
- [ ] No code files have been changed.
- [ ] Other reason <!-- Add your reason? -->
- Behavior changed:
- [ ] No.
- [ ] Yes. <!-- Explain the behavior change -->
- Does this need documentation?
- [ ] No.
- [ ] Yes. <!-- Add document PR link here. eg:
https://github.com/apache/doris-website/pull/1214 -->
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label <!-- Add branch pick label that this PR
should merge into -->
---
.../org/apache/doris/mtmv/MTMVRewriteUtil.java | 16 +++-
.../rules/exploration/mv/PartitionCompensator.java | 43 +++++------
.../rules/rewrite/QueryPartitionCollector.java | 2 +
.../mv/nested_mtmv/nested_mtmv.groovy | 85 ++++++++++++++++++++++
4 files changed, 119 insertions(+), 27 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java
b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java
index 47bf20052a9..6dc0bd24da7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java
@@ -38,6 +38,7 @@ import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
+import java.util.stream.Collectors;
public class MTMVRewriteUtil {
private static final Logger LOG =
LogManager.getLogger(MTMVRewriteUtil.class);
@@ -108,6 +109,12 @@ public class MTMVRewriteUtil {
return res;
}
+ /**
+ * Get mtmv partitions by related table partitions, if relatedPartitions
is null, return all mtmv partitions
+ * if mtmv is self-manage partition, return all mtmv partitions,
+ * if mtmv is nested mv, return all mtmv partitions,
+ * else return mtmv partitions by relatedPartitions
+ */
private static Set<String> getMtmvPartitionsByRelatedPartitions(MTMV mtmv,
MTMVRefreshContext refreshContext,
Map<List<String>, Set<String>> queryUsedPartitions) throws
AnalysisException {
if
(mtmv.getMvPartitionInfo().getPartitionType().equals(MTMVPartitionType.SELF_MANAGE))
{
@@ -118,8 +125,15 @@ public class MTMVRewriteUtil {
if (queryUsedPartitions == null) {
return mtmv.getPartitionNames();
}
- Set<String> res = Sets.newHashSet();
+ // if nested mv, should return directly
Set<MTMVRelatedTableIf> pctTables =
mtmv.getMvPartitionInfo().getPctTables();
+ Set<List<String>> pctTableQualifiers =
pctTables.stream().map(MTMVRelatedTableIf::getFullQualifiers).collect(
+ Collectors.toSet());
+ if (Sets.intersection(pctTableQualifiers,
queryUsedPartitions.keySet()).isEmpty()) {
+ return mtmv.getPartitionNames();
+ }
+ Set<String> res = Sets.newHashSet();
+
Map<Pair<MTMVRelatedTableIf, String>, String> relatedToMv = getPctToMv(
refreshContext.getPartitionMappings());
for (Entry<List<String>, Set<String>> entry :
queryUsedPartitions.entrySet()) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java
index 7cc649a68b0..22c3540b5cb 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java
@@ -21,7 +21,6 @@ import org.apache.doris.catalog.MTMV;
import org.apache.doris.catalog.Partition;
import org.apache.doris.catalog.PartitionInfo;
import org.apache.doris.catalog.PartitionType;
-import org.apache.doris.catalog.TableIf;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Pair;
import org.apache.doris.mtmv.BaseColInfo;
@@ -57,9 +56,10 @@ import java.util.Set;
public class PartitionCompensator {
public static final Logger LOG =
LogManager.getLogger(PartitionCompensator.class);
- // if partition pair is null which means can not get partitions from table
in QueryPartitionCollector,
- // we think this table scan query all partitions default
+ // if the partition pair is null which means could not get partitions from
table in QueryPartitionCollector,
+ // we think the table scans query all-partitions default
public static final Pair<RelationId, Set<String>> ALL_PARTITIONS =
Pair.of(null, null);
+ // It means all partitions are used when query
public static final Collection<Pair<RelationId, Set<String>>>
ALL_PARTITIONS_LIST =
ImmutableList.of(ALL_PARTITIONS);
@@ -252,33 +252,24 @@ public class PartitionCompensator {
// if value is not empty, means query some partitions
Map<List<String>, Set<String>> queryUsedRelatedTablePartitionsMap =
new HashMap<>();
tableLoop:
- for (Map.Entry<List<String>, TableIf> queryUsedTableEntry :
statementContext.getTables().entrySet()) {
+ for (List<String> queryUsedTable : tableUsedPartitionNameMap.keySet())
{
Set<String> usedPartitionSet = new HashSet<>();
Collection<Pair<RelationId, Set<String>>> tableUsedPartitions =
-
tableUsedPartitionNameMap.get(queryUsedTableEntry.getKey());
- if (!tableUsedPartitions.isEmpty()) {
- if (ALL_PARTITIONS_LIST.equals(tableUsedPartitions)) {
-
queryUsedRelatedTablePartitionsMap.put(queryUsedTableEntry.getKey(), null);
- continue;
- }
- for (Pair<RelationId, Set<String>> partitionPair :
tableUsedPartitions) {
- if (!customRelationIdSet.isEmpty()) {
- if (ALL_PARTITIONS.equals(partitionPair)) {
- continue;
- }
- if
(customRelationIdSet.get(partitionPair.key().asInt())) {
- usedPartitionSet.addAll(partitionPair.value());
- }
- } else {
- if (ALL_PARTITIONS.equals(partitionPair)) {
-
queryUsedRelatedTablePartitionsMap.put(queryUsedTableEntry.getKey(), null);
- continue tableLoop;
- }
- usedPartitionSet.addAll(partitionPair.value());
- }
+ tableUsedPartitionNameMap.get(queryUsedTable);
+ if (ALL_PARTITIONS_LIST.equals(tableUsedPartitions)) {
+ // It means all partitions are used when query
+ queryUsedRelatedTablePartitionsMap.put(queryUsedTable, null);
+ continue;
+ }
+ for (Pair<RelationId, Set<String>> tableUsedPartitionPair :
tableUsedPartitions) {
+ if (ALL_PARTITIONS.equals(tableUsedPartitionPair)) {
+ // It means all partitions are used when query
+ queryUsedRelatedTablePartitionsMap.put(queryUsedTable,
null);
+ continue tableLoop;
}
+ usedPartitionSet.addAll(tableUsedPartitionPair.value());
}
-
queryUsedRelatedTablePartitionsMap.put(queryUsedTableEntry.getKey(),
usedPartitionSet);
+ queryUsedRelatedTablePartitionsMap.put(queryUsedTable,
usedPartitionSet);
}
return queryUsedRelatedTablePartitionsMap;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/QueryPartitionCollector.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/QueryPartitionCollector.java
index 8488a07cbc7..9907f5270e7 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/QueryPartitionCollector.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/QueryPartitionCollector.java
@@ -55,9 +55,11 @@ public class QueryPartitionCollector extends
DefaultPlanVisitor<Void, CascadesCo
return null;
}
StatementContext statementContext = context.getStatementContext();
+ // Collect relationId to tableId mapping
Map<Integer, Integer> relationIdToTableId =
statementContext.getRelationIdToCommonTableIdMap();
relationIdToTableId.put(catalogRelation.getRelationId().asInt(),
statementContext.getTableId(catalogRelation.getTable()).asInt());
+ // Collect table used partition mapping
Multimap<List<String>, Pair<RelationId, Set<String>>>
tableUsedPartitionNameMap = statementContext
.getTableUsedPartitionNameMap();
Set<String> tablePartitions = new HashSet<>();
diff --git
a/regression-test/suites/nereids_rules_p0/mv/nested_mtmv/nested_mtmv.groovy
b/regression-test/suites/nereids_rules_p0/mv/nested_mtmv/nested_mtmv.groovy
index 1972c2d505b..30ea11ec956 100644
--- a/regression-test/suites/nereids_rules_p0/mv/nested_mtmv/nested_mtmv.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/nested_mtmv/nested_mtmv.groovy
@@ -753,4 +753,89 @@ suite("nested_mtmv") {
mv_rewrite_any_success(sql_5, [mv_3, mv_4, mv_5])
compare_res(sql_5 + " order by 1,2,3,4,5,6,7,8,9,10,11,12,13")
+ sql """
+ drop table if exists sales_partitioned
+ """
+
+ sql """
+CREATE TABLE sales_partitioned (
+product_id INT NOT NULL,
+city VARCHAR(50) NOT NULL,
+sale_date DATE NOT NULL,
+amount DECIMAL(18, 2) NOT NULL
+)
+DUPLICATE KEY(product_id, city, sale_date)
+PARTITION BY RANGE(sale_date) (
+PARTITION p20251001 VALUES [('2025-10-01'), ('2025-10-02')),
+PARTITION p20251002 VALUES [('2025-10-02'), ('2025-10-03')),
+PARTITION p20251003 VALUES [('2025-10-03'), ('2025-10-04')),
+PARTITION p_other VALUES [('2025-10-04'), ('2025-11-01'))
+)
+DISTRIBUTED BY HASH(product_id) BUCKETS 10
+ PROPERTIES (
+ "replication_num" = "1"
+);
+ """
+
+
+ sql """
+INSERT INTO sales_partitioned (product_id, city, sale_date, amount) VALUES
+(101, 'Beijing', '2025-10-01', 100.00), -- p20251001
+(101, 'Shanghai', '2025-10-01', 150.00), -- p20251001
+(102, 'Beijing', '2025-10-02', 200.00), -- p20251002
+(102, 'Shanghai', '2025-10-02', 250.00), -- p20251002
+(101, 'Beijing', '2025-10-03', 120.00), -- p20251003
+(102, 'Shanghai', '2025-10-03', 300.00); -- p20251003
+ """
+
+ create_async_partition_mv(db, "zz_mtmv1", """
+ SELECT
+ city,
+ sale_date,
+ SUM(amount) AS daily_city_amount
+ FROM
+ sales_partitioned
+ GROUP BY
+ city, sale_date;
+ """, "(sale_date)")
+ mv_rewrite_success("""
+ SELECT
+ city,
+ SUM(amount) AS total_city_amount
+ FROM
+ sales_partitioned
+ WHERE
+ sale_date >= '2025-10-01' AND sale_date <= '2025-10-03'
+ GROUP BY
+ city;
+ """, "zz_mtmv1", is_partition_statistics_ready(db, ["zz_mtmv1"]))
+
+ create_async_partition_mv(db, "zz_mtmv2", """
+ SELECT
+ city,
+ date_trunc(sale_date, 'MONTH') AS sale_date,
+ SUM(daily_city_amount) AS monthly_city_amount
+ FROM
+ zz_mtmv1
+ GROUP BY
+ city,
+ date_trunc(sale_date, 'MONTH')
+ """, "(sale_date)")
+
+ mv_rewrite_all_success_without_check_chosen("""
+ SELECT
+ date_trunc(sale_date, 'MONTH') AS sale_date,
+ SUM(daily_city_amount) AS monthly_city_amount
+ FROM
+ (SELECT
+ city,
+ sale_date,
+ SUM(amount) AS daily_city_amount
+ FROM
+ sales_partitioned
+ GROUP BY
+ city, sale_date) as t
+ GROUP BY
+ date_trunc(sale_date, 'MONTH');
+ """, ["zz_mtmv1", "zz_mtmv2"])
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
