This is an automated email from the ASF dual-hosted git repository.
morrySnow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 9adfd2d8a1f [fix](mv) Compensate complete invalid roll-up partition
buckets (#67882)
9adfd2d8a1f is described below
commit 9adfd2d8a1fda119e9f309ce27d2f8612563015d
Author: morrySnow <[email protected]>
AuthorDate: Mon Sep 14 19:03:05 2026 +0800
[fix](mv) Compensate complete invalid roll-up partition buckets (#67882)
## Problem
During partition-union rewrite, a roll-up materialized-view partition
can cover multiple base-table partitions. When that MV partition is
invalid and removed as a whole, compensating only the base partitions
that overlap the raw query partition set can leave the replacement scope
smaller than the removed MV bucket and produce incorrect aggregate
results.
## Root cause
The compensation calculation used the intersection between an invalid MV
partition's complete base-partition mapping and the raw query partitions
as both the relevance test and the compensation set. This lost the
atomic relationship between one MV partition and every base partition
mapped to it.
## Reproduction
Use one valid MV partition mapped to `p3`, one invalid roll-up partition
mapped to `{p1, p2}`, and another invalid partition mapped to `{p4,
p5}`. For a raw query partition set `{p1, p3}`, the old calculation
compensated only `p1`, even though the complete intersecting roll-up
bucket was removed. The disjoint `{p4, p5}` bucket must still remain
outside compensation.
## Fix
Use the intersection only to decide whether an invalid MV partition is
relevant to the query. Once relevant, add its complete base-partition
mapping to compensation. Invalid MV partitions with no intersection
remain excluded, preserving the existing query-range guard. The existing
multi-PCT-table merge behavior is unchanged.
## Tests
- `./run-fe-ut.sh --run
org.apache.doris.nereids.rules.exploration.mv.PartitionCompensatorTest`
(13 tests passed)
- `DISABLE_BUILD_UI=ON ./build.sh --fe` (passed; the host's Node.js 16
cannot build the current UI, so the supported no-UI FE build path was
used)
- `./run-regression-test.sh --run -s mtmv_range_date_part_up_rewrite
...` (1 suite passed)
---
.../rules/exploration/mv/PartitionCompensator.java | 7 +-
.../exploration/mv/PartitionCompensatorTest.java | 78 ++++++++++++++++++++++
2 files changed, 83 insertions(+), 2 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java
index a8613cf3dc6..13bd6973555 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java
@@ -200,8 +200,11 @@ public class PartitionCompensator {
// Base table partition maybe deleted, need not union
continue;
}
- Sets.intersection(baseTablePartitions,
queryUsedBaseTablePartitionNameSet)
- .copyInto(baseTableNeedUnionPartitionNameSet);
+ if (!Sets.intersection(baseTablePartitions,
queryUsedBaseTablePartitionNameSet).isEmpty()) {
+ // An MV partition is the atomic unit removed from the
rewritten plan. If any base
+ // partition in its roll-up bucket is used by the query,
compensate the whole bucket.
+ baseTableNeedUnionPartitionNameSet.addAll(baseTablePartitions);
+ }
}
// If related base table creates partitions or mv is created with ttl,
need base table union
Sets.difference(queryUsedBaseTablePartitionNameSet,
mvValidBaseTablePartitionNameSet)
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensatorTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensatorTest.java
index 0ad6d8b49ed..e3ab9688e5f 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensatorTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensatorTest.java
@@ -487,6 +487,84 @@ public class PartitionCompensatorTest extends
TestWithFeService {
.forEach(v -> Assertions.assertEquals(expectedUnion, v));
}
+ @SuppressWarnings("unchecked")
+ @Test
+ public void
testCalcInvalidPartitionsCompensatesWholeIntersectingRollupBucket()
+ throws Exception {
+ DatabaseIf<?> baseDb = mockDatabase("cat", 1L, "db", 2L);
+ MTMVRelatedTableIf relatedTable = mockRelatedTableIf(
+ "base_t", 10L, ImmutableList.of("cat", "db", "base_t"),
baseDb);
+ BaseColInfo colInfo = new BaseColInfo("dt", new
BaseTableInfo(relatedTable));
+
+ DatabaseIf<?> mvDb = mockDatabase("internal", 3L, "mv_db", 4L);
+ MTMV mtmv = Mockito.mock(MTMV.class);
+ Mockito.when(mtmv.getName()).thenReturn("mv1");
+ Mockito.when(mtmv.getId()).thenReturn(100L);
+ Mockito.when(mtmv.getDatabase()).thenReturn(mvDb);
+ Mockito.when(mtmv.selectNonEmptyPartitionIds(ArgumentMatchers.any(),
ArgumentMatchers.any()))
+ .thenReturn(ImmutableList.of(1L));
+
+ long validMvPartitionId = 101L;
+ long partiallyStaleMvPartitionId = 102L;
+ long disjointMvPartitionId = 103L;
+ Partition validMvPartition = mockPartition(validMvPartitionId,
"mv_valid");
+ Partition partiallyStaleMvPartition =
mockPartition(partiallyStaleMvPartitionId, "mv_partially_stale");
+ Partition disjointMvPartition = mockPartition(disjointMvPartitionId,
"mv_disjoint");
+
Mockito.when(mtmv.getPartition(validMvPartitionId)).thenReturn(validMvPartition);
+
Mockito.when(mtmv.getPartition(partiallyStaleMvPartitionId)).thenReturn(partiallyStaleMvPartition);
+
Mockito.when(mtmv.getPartition(disjointMvPartitionId)).thenReturn(disjointMvPartition);
+
+ PartitionInfo mvPartitionInfo = Mockito.mock(PartitionInfo.class);
+ Mockito.when(mtmv.getPartitionInfo()).thenReturn(mvPartitionInfo);
+
Mockito.when(mvPartitionInfo.getType()).thenReturn(PartitionType.RANGE);
+ MTMVPartitionInfo mvPctInfo = Mockito.mock(MTMVPartitionInfo.class);
+ Mockito.when(mtmv.getMvPartitionInfo()).thenReturn(mvPctInfo);
+
Mockito.when(mvPctInfo.getPctTables()).thenReturn(ImmutableSet.of(relatedTable));
+
Mockito.when(mvPctInfo.getPctInfos()).thenReturn(ImmutableList.of(colInfo));
+
+ Map<String, Set<String>> relatedPartitionMapping = new HashMap<>();
+ relatedPartitionMapping.put("mv_valid", ImmutableSet.of("p3"));
+ relatedPartitionMapping.put("mv_partially_stale",
ImmutableSet.of("p1", "p2"));
+ relatedPartitionMapping.put("mv_disjoint", ImmutableSet.of("p4",
"p5"));
+ Map<MTMVRelatedTableIf, Map<String, Set<String>>> partitionMappings =
new HashMap<>();
+ partitionMappings.put(relatedTable, relatedPartitionMapping);
+
+ AsyncMaterializationContext matCtx =
Mockito.mock(AsyncMaterializationContext.class);
+ Mockito.when(matCtx.getMtmv()).thenReturn(mtmv);
+
Mockito.when(matCtx.calculatePartitionMappings(ArgumentMatchers.any())).thenReturn(partitionMappings);
+
+ Map<BaseTableInfo, Collection<Partition>> canRewriteMap = new
HashMap<>();
+ canRewriteMap.put(new BaseTableInfo(mtmv),
ImmutableList.of(validMvPartition));
+ StatementContext stmtCtx = Mockito.mock(StatementContext.class);
+
Mockito.when(stmtCtx.getMvCanRewritePartitionsMap()).thenReturn(canRewriteMap);
+ CascadesContext cascadesCtx = Mockito.mock(CascadesContext.class);
+ Mockito.when(cascadesCtx.getStatementContext()).thenReturn(stmtCtx);
+
+ LogicalOlapScan selectedMvScan = Mockito.mock(LogicalOlapScan.class);
+ Mockito.when(selectedMvScan.getTable()).thenReturn(mtmv);
+ Mockito.when(selectedMvScan.getSelectedPartitionIds())
+ .thenReturn(ImmutableList.of(
+ validMvPartitionId, partiallyStaleMvPartitionId,
disjointMvPartitionId));
+ Plan rewrittenPlan = Mockito.mock(Plan.class);
+ Mockito.when(rewrittenPlan.collectToList(ArgumentMatchers.any()))
+ .thenReturn(ImmutableList.of(selectedMvScan));
+
+ // The query touches p1 in the stale roll-up bucket and p3 in a valid
bucket. Once the stale
+ // MV partition is removed, its complete {p1, p2} mapping must be
compensated atomically.
+ // The completely disjoint {p4, p5} bucket is removed from the MV scan
but is not compensated.
+ Map<List<String>, Set<String>> queryUsedPartitions = new HashMap<>();
+ queryUsedPartitions.put(relatedTable.getFullQualifiers(),
ImmutableSet.of("p1", "p3"));
+
+ Pair<Map<BaseTableInfo, Set<String>>, Map<BaseColInfo, Set<String>>>
result =
+ PartitionCompensator.calcInvalidPartitions(
+ queryUsedPartitions, rewrittenPlan, matCtx,
cascadesCtx);
+
+ Assertions.assertNotNull(result);
+ Assertions.assertEquals(ImmutableSet.of("mv_partially_stale",
"mv_disjoint"),
+ result.key().get(new BaseTableInfo(mtmv)));
+ Assertions.assertEquals(ImmutableSet.of("p1", "p2"),
result.value().get(colInfo));
+ }
+
@SuppressWarnings("unchecked")
@Test
public void
testCalcInvalidPartitionsDoesNotCompensateBasePartitionsUnusedByQuery()
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
