This is an automated email from the ASF dual-hosted git repository.
morrySnow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 2ed8b169387 [fix](mv) Distinguish partition compensation from UNION
ALL rewrite (#66445)
2ed8b169387 is described below
commit 2ed8b169387c28d6ba1b3f21f70d4d973adcc4a2
Author: foxtail463 <[email protected]>
AuthorDate: Wed Aug 12 17:27:36 2026 +0800
[fix](mv) Distinguish partition compensation from UNION ALL rewrite (#66445)
Problem Summary:
MV rewrite treated removing invalid MV partitions as requiring UNION
ALL, causing
valid rewrites to be rejected.
Solution:
Separate partition compensation detection from base-table partition
union requirements.
Only check UNION ALL capability when base-table compensation is needed.
---------
Co-authored-by: yangtao555 <[email protected]>
---
.../mv/AbstractMaterializedViewRule.java | 24 ++--
...terializedViewAggregateOnNoneAggregateRule.java | 4 +-
.../rules/exploration/mv/PartitionCompensator.java | 7 -
.../union_rewrite/partition_curd_union_rewrite.out | 7 +
.../partition_curd_union_rewrite.groovy | 143 +++++++++++++++++++++
5 files changed, 162 insertions(+), 23 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
index 38c4fade7ff..5b24abedbbb 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
@@ -375,17 +375,20 @@ public abstract class AbstractMaterializedViewRule
implements ExplorationRuleFac
// if mv can not offer any partition for query, query
rewrite bail out to avoid cycle run
return rewriteResults;
}
- boolean partitionNeedUnion =
PartitionCompensator.needUnionRewrite(invalidPartitions, cascadesContext);
- boolean canUnionRewrite = canUnionRewrite(queryPlan,
- (AsyncMaterializationContext) materializationContext,
cascadesContext);
- if (partitionNeedUnion && !canUnionRewrite) {
+ boolean needRemoveMvPartitions =
!invalidPartitions.key().isEmpty();
+ boolean needBaseTableUnion =
!invalidPartitions.value().isEmpty();
+ if (needBaseTableUnion && !canUnionRewrite(queryPlan,
+ (AsyncMaterializationContext) materializationContext,
cascadesContext)) {
materializationContext.recordFailReason(queryStructInfo,
"need compensate union all, but can not, because
the query structInfo",
() -> String.format("mv partition info is %s, and
the query plan is %s",
mtmv.getMvPartitionInfo(),
queryPlan.treeString()));
return rewriteResults;
}
- if (partitionNeedUnion) {
+ if (needRemoveMvPartitions) {
+ rewrittenPlan = rewrittenPlan.accept(new
PartitionRemover(), invalidPartitions.key());
+ }
+ if (needBaseTableUnion) {
Pair<Plan, Boolean> planAndNeedAddFilterPair =
StructInfo.addFilterOnTableScan(queryPlan,
invalidPartitions.value(), cascadesContext);
if (planAndNeedAddFilterPair == null) {
@@ -395,15 +398,8 @@ public abstract class AbstractMaterializedViewRule
implements ExplorationRuleFac
invalidPartitions,
queryPlan.treeString()));
continue;
}
- if (invalidPartitions.value().isEmpty() ||
!planAndNeedAddFilterPair.value()) {
- // if invalid base table filter is empty or doesn't
need to add filter on base table,
- // only need remove mv invalid partition
- rewrittenPlan = rewrittenPlan.accept(new
PartitionRemover(), invalidPartitions.key());
- } else {
- // For rewrittenPlan which contains materialized view
should remove invalid partition ids
- List<Plan> children = Lists.newArrayList(
- rewrittenPlan.accept(new PartitionRemover(),
invalidPartitions.key()),
- planAndNeedAddFilterPair.key());
+ if (planAndNeedAddFilterPair.value()) {
+ List<Plan> children =
Lists.newArrayList(rewrittenPlan, planAndNeedAddFilterPair.key());
// Union query materialized view and source table
rewrittenPlan = new LogicalUnion(Qualifier.ALL,
queryPlan.getOutput().stream().map(NamedExpression.class::cast)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewAggregateOnNoneAggregateRule.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewAggregateOnNoneAggregateRule.java
index 883b6651ae7..7155f661ee7 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewAggregateOnNoneAggregateRule.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewAggregateOnNoneAggregateRule.java
@@ -105,8 +105,8 @@ public class MaterializedViewAggregateOnNoneAggregateRule
extends AbstractMateri
Pair<Map<BaseTableInfo, Set<String>>, Map<BaseColInfo, Set<String>>>
invalidPartitions
= super.calcInvalidPartitions(queryUsedBaseTablePartitionMap,
rewrittenPlan, cascadesContext,
materializationContext);
- if (PartitionCompensator.needUnionRewrite(invalidPartitions,
cascadesContext)) {
- // if query use some invalid partition in mv, bail out
+ if (invalidPartitions != null && !invalidPartitions.value().isEmpty())
{
+ // Aggregate-on-non-aggregate rewrite does not support base-table
union compensation.
return null;
}
return invalidPartitions;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java
index 9064049d3b8..7b07130aa59 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java
@@ -220,13 +220,6 @@ public class PartitionCompensator {
return Pair.of(mvPartitionNeedRemoveNameMap,
baseTablePartitionNeedUnionNameMap);
}
- public static boolean needUnionRewrite(
- Pair<Map<BaseTableInfo, Set<String>>, Map<BaseColInfo,
Set<String>>> invalidPartitions,
- CascadesContext cascadesContext) {
- return invalidPartitions != null
- && (!invalidPartitions.key().values().isEmpty() ||
!invalidPartitions.value().values().isEmpty());
- }
-
/**
* Check if need union compensate or not
* If query base table all partitions with ALL_PARTITIONS or
ALL_PARTITIONS_LIST, should not do union compensate
diff --git
a/regression-test/data/nereids_rules_p0/mv/union_rewrite/partition_curd_union_rewrite.out
b/regression-test/data/nereids_rules_p0/mv/union_rewrite/partition_curd_union_rewrite.out
new file mode 100644
index 00000000000..a3e98f21cb0
--- /dev/null
+++
b/regression-test/data/nereids_rules_p0/mv/union_rewrite/partition_curd_union_rewrite.out
@@ -0,0 +1,7 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !aggregate_without_partition_column_after_partition_delete --
+1 20
+
+-- !aggregate_on_detail_mv_after_partition_delete --
+1 1
+
diff --git
a/regression-test/suites/nereids_rules_p0/mv/union_rewrite/partition_curd_union_rewrite.groovy
b/regression-test/suites/nereids_rules_p0/mv/union_rewrite/partition_curd_union_rewrite.groovy
index 43c90c0df89..d5a1ac82872 100644
---
a/regression-test/suites/nereids_rules_p0/mv/union_rewrite/partition_curd_union_rewrite.groovy
+++
b/regression-test/suites/nereids_rules_p0/mv/union_rewrite/partition_curd_union_rewrite.groovy
@@ -231,4 +231,147 @@ suite ("partition_curd_union_rewrite") {
mv_rewrite_success(partition_sql, mv_name,
is_partition_statistics_ready(db, ["lineitem", "orders", mv_name]))
compare_res(partition_sql + order_by_stmt)
+
+ // Use exactly two base partitions. After p2 is dropped, p2 is stale only
in the MV: no
+ // surviving base partition needs UNION ALL compensation. The query omits
the MV partition column d,
+ // so canUnionRewrite() must be false while the rewrite still succeeds by
removing stale MV p2.
+ sql "DROP MATERIALIZED VIEW IF EXISTS partition_compensation_mv"
+ sql "DROP TABLE IF EXISTS partition_compensation_mv"
+ sql "DROP TABLE IF EXISTS partition_compensation_base"
+ sql """
+ CREATE TABLE partition_compensation_base (
+ id int not null,
+ k int not null,
+ d date not null,
+ v int not null
+ )
+ DUPLICATE KEY(id, k, d)
+ PARTITION BY RANGE(d) (
+ PARTITION p1 VALUES [('2024-01-01'), ('2024-01-02')),
+ PARTITION p2 VALUES [('2024-01-02'), ('2024-01-03'))
+ )
+ DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES ('replication_num' = '1')
+ """
+ sql """
+ INSERT INTO partition_compensation_base VALUES
+ (1, 1, '2024-01-01', 1),
+ (2, 1, '2024-01-01', 1),
+ (3, 1, '2024-01-01', 1),
+ (4, 1, '2024-01-01', 1),
+ (5, 1, '2024-01-01', 1),
+ (6, 1, '2024-01-01', 1),
+ (7, 1, '2024-01-01', 1),
+ (8, 1, '2024-01-01', 1),
+ (9, 1, '2024-01-01', 1),
+ (10, 1, '2024-01-01', 1),
+ (11, 1, '2024-01-01', 1),
+ (12, 1, '2024-01-01', 1),
+ (13, 1, '2024-01-01', 1),
+ (14, 1, '2024-01-01', 1),
+ (15, 1, '2024-01-01', 1),
+ (16, 1, '2024-01-01', 1),
+ (17, 1, '2024-01-01', 1),
+ (18, 1, '2024-01-01', 1),
+ (19, 1, '2024-01-01', 1),
+ (20, 1, '2024-01-01', 1),
+ (21, 1, '2024-01-02', 1)
+ """
+
+ def partition_compensation_mv_name = "partition_compensation_mv"
+ def aggregate_without_partition_column_sql = """
+ select k, sum(v) as sum_v
+ from partition_compensation_base
+ group by k
+ """
+ sql """
+ CREATE MATERIALIZED VIEW ${partition_compensation_mv_name}
+ BUILD IMMEDIATE REFRESH AUTO ON MANUAL
+ PARTITION BY(d)
+ DISTRIBUTED BY RANDOM BUCKETS 1
+ PROPERTIES ('replication_num' = '1')
+ AS
+ select d, k, sum(v) as sum_v
+ from partition_compensation_base
+ group by d, k
+ """
+ waitingMTMVTaskFinished(getJobName(db, partition_compensation_mv_name))
+ sql """
+ analyze table partition_compensation_base with sync;
+ analyze table ${partition_compensation_mv_name} with sync;
+ """
+
+ sql "ALTER TABLE partition_compensation_base DROP PARTITION p2 FORCE"
+ waitingPartitionIsExpected(partition_compensation_mv_name,
"p_20240102_20240103", false)
+ mv_rewrite_success(aggregate_without_partition_column_sql,
partition_compensation_mv_name,
+ is_partition_statistics_ready(db, ["partition_compensation_base",
partition_compensation_mv_name]))
+ explain {
+ sql "memo plan ${aggregate_without_partition_column_sql}"
+ notContains "PhysicalUnion"
+ }
+ order_qt_aggregate_without_partition_column_after_partition_delete """
+ ${aggregate_without_partition_column_sql}
+ order by k
+ """
+
+ // Aggregate-on-detail rewrite has the same removal-only partition state.
It should remove the stale
+ // detail MV partition and must not require a base-table UNION ALL branch.
+ sql "DROP MATERIALIZED VIEW IF EXISTS partition_compensation_detail_mv"
+ sql "DROP TABLE IF EXISTS partition_compensation_detail_mv"
+ sql "DROP TABLE IF EXISTS partition_compensation_detail_base"
+ sql """
+ CREATE TABLE partition_compensation_detail_base (
+ id int not null,
+ k int not null,
+ d date not null,
+ v int not null
+ )
+ DUPLICATE KEY(id, k, d)
+ PARTITION BY RANGE(d) (
+ PARTITION p1 VALUES [('2024-02-01'), ('2024-02-02')),
+ PARTITION p2 VALUES [('2024-02-02'), ('2024-02-03'))
+ )
+ DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES ('replication_num' = '1')
+ """
+ sql """
+ INSERT INTO partition_compensation_detail_base VALUES
+ (1, 1, '2024-02-01', 1),
+ (2, 1, '2024-02-02', 1)
+ """
+
+ def aggregate_on_detail_mv_sql = """
+ select k, sum(v) as sum_v
+ from partition_compensation_detail_base
+ group by k
+ """
+ sql """
+ CREATE MATERIALIZED VIEW partition_compensation_detail_mv
+ BUILD IMMEDIATE REFRESH AUTO ON MANUAL
+ PARTITION BY(d)
+ DISTRIBUTED BY RANDOM BUCKETS 1
+ PROPERTIES ('replication_num' = '1')
+ AS
+ select id, k, d, v
+ from partition_compensation_detail_base
+ """
+ waitingMTMVTaskFinished(getJobName(db, "partition_compensation_detail_mv"))
+ sql """
+ analyze table partition_compensation_detail_base with sync;
+ analyze table partition_compensation_detail_mv with sync;
+ """
+
+ sql "ALTER TABLE partition_compensation_detail_base DROP PARTITION p2
FORCE"
+ waitingPartitionIsExpected("partition_compensation_detail_mv",
"p_20240202_20240203", false)
+ mv_rewrite_success(aggregate_on_detail_mv_sql,
"partition_compensation_detail_mv",
+ is_partition_statistics_ready(
+ db, ["partition_compensation_detail_base",
"partition_compensation_detail_mv"]))
+ explain {
+ sql "memo plan ${aggregate_on_detail_mv_sql}"
+ notContains "PhysicalUnion"
+ }
+ order_qt_aggregate_on_detail_mv_after_partition_delete """
+ ${aggregate_on_detail_mv_sql}
+ order by k
+ """
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
