This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 5eafeee67d1 [fix](test) Fix mv case wrongly and fix mv rewrite fail
when sync mv used all agg table group by field (#55878)
5eafeee67d1 is described below
commit 5eafeee67d1ecb7d3bcb2d38aaf490deeff1846d
Author: seawinde <[email protected]>
AuthorDate: Tue Sep 16 14:48:45 2025 +0800
[fix](test) Fix mv case wrongly and fix mv rewrite fail when sync mv used
all agg table group by field (#55878)
fix cases as fllowing which use wrong check method
/Users/wusi/IdeaProjects/doris/regression-test/suites/nereids_syntax_p1/mv/aggregate/agg_sync_mv.groovy
/Users/wusi/IdeaProjects/doris/regression-test/suites/nereids_syntax_p0/mv/aggregate/agg_sync_mv.groovy
and fix rewrite wrongly when sync is created when used all agg table group
by field
/Users/wusi/IdeaProjects/doris/regression-test/suites/nereids_syntax_p1/mv/aggregate/agg_sync_mv.groovy
---
.../java/org/apache/doris/mtmv/MTMVPlanUtil.java | 7 +-
.../mv/PreMaterializedViewRewriter.java | 1 +
.../mv/PreMaterializedViewRewriterTest.java | 17 +++++
.../suites/mv_p0/agg_table_mv/agg_table_mv.groovy | 83 ++++++++++++++++++++++
.../mv/aggregate/agg_sync_mv.groovy | 2 +-
.../mv/aggregate/agg_sync_mv.groovy | 2 +-
6 files changed, 107 insertions(+), 5 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java
b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java
index 6653e988c50..bff8080a9c3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java
@@ -97,14 +97,15 @@ public class MTMVPlanUtil {
ctx.getSessionVariable().skipStorageEngineMerge = false;
ctx.getSessionVariable().showHiddenColumns = false;
ctx.getSessionVariable().allowModifyMaterializedViewData = true;
- // Temporary disable constant propagation to lose hitting the mv
- // Disable add default limit rule to avoid refresh data wrong
+ // Rules disabled during materialized view plan generation. These
rules can cause significant plan changes,
+ // which may affect transparent query rewriting by mv
List<RuleType> disableRules = Arrays.asList(
RuleType.COMPRESSED_MATERIALIZE_AGG,
RuleType.COMPRESSED_MATERIALIZE_SORT,
RuleType.ELIMINATE_CONST_JOIN_CONDITION,
RuleType.CONSTANT_PROPAGATION,
- RuleType.ADD_DEFAULT_LIMIT
+ RuleType.ADD_DEFAULT_LIMIT,
+ RuleType.ELIMINATE_GROUP_BY
);
ctx.getSessionVariable().setDisableNereidsRules(
disableRules.stream().map(RuleType::name).collect(Collectors.joining(",")));
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PreMaterializedViewRewriter.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PreMaterializedViewRewriter.java
index db8290b95f8..d6b47cabd63 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PreMaterializedViewRewriter.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PreMaterializedViewRewriter.java
@@ -69,6 +69,7 @@ public class PreMaterializedViewRewriter {
NEED_PRE_REWRITE_RULE_TYPES.set(RuleType.PUSH_DOWN_VIRTUAL_COLUMNS_INTO_OLAP_SCAN.ordinal());
NEED_PRE_REWRITE_RULE_TYPES.set(RuleType.DISTINCT_AGGREGATE_SPLIT.ordinal());
NEED_PRE_REWRITE_RULE_TYPES.set(RuleType.PROCESS_SCALAR_AGG_MUST_USE_MULTI_DISTINCT.ordinal());
+
NEED_PRE_REWRITE_RULE_TYPES.set(RuleType.ELIMINATE_GROUP_BY_KEY_BY_UNIFORM.ordinal());
}
/**
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/PreMaterializedViewRewriterTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/PreMaterializedViewRewriterTest.java
index 0022e352d7f..11fc0a27de8 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/PreMaterializedViewRewriterTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/PreMaterializedViewRewriterTest.java
@@ -23,7 +23,10 @@ import org.apache.doris.nereids.jobs.cascades.DeriveStatsJob;
import org.apache.doris.nereids.jobs.cascades.OptimizeGroupJob;
import org.apache.doris.nereids.memo.Group;
import org.apache.doris.nereids.memo.Memo;
+import org.apache.doris.nereids.rules.RuleType;
+import
org.apache.doris.nereids.rules.exploration.mv.InitMaterializationContextHook;
import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewUtils;
+import
org.apache.doris.nereids.rules.exploration.mv.PreMaterializedViewRewriter;
import
org.apache.doris.nereids.rules.exploration.mv.PreMaterializedViewRewriter.PreRewriteStrategy;
import org.apache.doris.nereids.rules.exploration.mv.StructInfo;
import org.apache.doris.nereids.sqltest.SqlTestBase;
@@ -2939,6 +2942,20 @@ public class PreMaterializedViewRewriterTest extends
SqlTestBase {
checkIfEquals(originalSql, equivalentSqlList);
}
+ /**
+ * Test pre-materialized view rewrite need pre-rewrite when
ELIMINATE_GROUP_BY_KEY_BY_UNIFORM applied
+ * */
+ @Test
+ public void testNeedPreRewrite() {
+ CascadesContext cascadesContext =
MemoTestUtils.createCascadesContext("select T1.id from T1");
+ StatementContext statementContext =
cascadesContext.getConnectContext().getStatementContext();
+ statementContext.setForceRecordTmpPlan(true);
+
statementContext.ruleSetApplied(RuleType.ELIMINATE_GROUP_BY_KEY_BY_UNIFORM);
+
statementContext.getPlannerHooks().add(InitMaterializationContextHook.INSTANCE);
+
statementContext.getTmpPlanForMvRewrite().add(cascadesContext.getRewritePlan());
+
Assertions.assertTrue(PreMaterializedViewRewriter.needPreRewrite(cascadesContext));
+ }
+
private void checkIfEquals(String originalSql, List<String>
equivalentSqlList) {
// init original cascades context
CascadesContext originalCascadesContext = initOriginal(originalSql);
diff --git a/regression-test/suites/mv_p0/agg_table_mv/agg_table_mv.groovy
b/regression-test/suites/mv_p0/agg_table_mv/agg_table_mv.groovy
new file mode 100644
index 00000000000..0ff7dfe3818
--- /dev/null
+++ b/regression-test/suites/mv_p0/agg_table_mv/agg_table_mv.groovy
@@ -0,0 +1,83 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.codehaus.groovy.runtime.IOGroovyMethods
+
+suite ("agg_table_mv") {
+
+ String db = context.config.getDbNameByFile(context.file)
+ sql "use ${db}"
+
+ sql """set enable_nereids_planner=true;"""
+ sql """ DROP TABLE IF EXISTS orders_agg; """
+ // this mv rewrite would not be rewritten in RBO phase, so set TRY_IN_RBO
explicitly to make case stable
+ sql "set pre_materialized_view_rewrite_strategy = TRY_IN_RBO"
+
+ sql """
+ CREATE TABLE `orders_agg` (
+ `o_orderkey` BIGINT not NULL,
+ `o_custkey` INT not NULL,
+ `o_orderdate` DATE not null,
+ `o_orderstatus` VARCHAR(1) replace,
+ `o_totalprice` DECIMAL(15, 2) sum,
+ `o_orderpriority` VARCHAR(15) replace,
+ `o_clerk` VARCHAR(15) replace,
+ `o_shippriority` INT sum,
+ `o_comment` VARCHAR(79) replace
+ ) ENGINE=OLAP
+ aggregate KEY(`o_orderkey`, `o_custkey`, `o_orderdate`)
+ COMMENT 'OLAP'
+ auto partition by range (date_trunc(`o_orderdate`, 'day')) ()
+ DISTRIBUTED BY HASH(`o_orderkey`) BUCKETS 96
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1"
+ );
+ """
+
+ sql """
+ insert into orders_agg values
+ (2, 1, '2023-10-17', 'k', 99.5, 'a', 'b', 1, 'yy'),
+ (1, 2, '2023-10-17', 'o', 109.2, 'c','d',2, 'mm'),
+ (3, 3, '2023-10-19', null, 99.5, 'a', 'b', 1, 'yy'),
+ (1, 2, '2023-10-20', 'o', null, 'a', 'b', 1, 'yy'),
+ (2, 3, '2023-10-21', 'k', 109.2, null,'d',2, 'mm'),
+ (3, 1, '2023-10-22', 'k', 99.5, 'a', null, 1, 'yy'),
+ (1, 3, '2023-10-19', 'o', 99.5, 'a', 'b', null, 'yy'),
+ (2, 1, '2023-10-18', 'o', 109.2, 'c','d',2, null),
+ (3, 2, '2023-10-17', 'k', 99.5, 'a', 'b', 1, 'yy'),
+ (4, 5, '2023-10-19', 'k', 99.5, 'a', 'b', 1, 'yy');
+ """
+
+ create_sync_mv(db, "orders_agg", "agg_mv_name_3", """
+ select o_orderdatE as a_o_orderdatE, o_custkey as a_o_custkey,
o_orderkey as a_o_orderkey,
+ sum(o_totalprice) as sum_total
+ from orders_agg
+ group by
+ o_orderdatE,
+ o_custkey,
+ o_orderkey;
+ """)
+
+ mv_rewrite_success("""
+ select o_custkey, o_orderkey,
+ sum(o_totalprice) as sum_total
+ from orders_agg
+ group by
+ o_custkey,
+ o_orderkey
+ """, "agg_mv_name_3")
+}
diff --git
a/regression-test/suites/nereids_syntax_p0/mv/aggregate/agg_sync_mv.groovy
b/regression-test/suites/nereids_syntax_p0/mv/aggregate/agg_sync_mv.groovy
index 044941a007d..6f0a366027b 100644
--- a/regression-test/suites/nereids_syntax_p0/mv/aggregate/agg_sync_mv.groovy
+++ b/regression-test/suites/nereids_syntax_p0/mv/aggregate/agg_sync_mv.groovy
@@ -224,7 +224,7 @@ suite("agg_sync_mv") {
qt_select_ndv """select id, ndv(kint) from agg_mv_test group by id order
by id;"""
sql """drop materialized view if exists mv_sync32 on agg_mv_test;"""
createMV("""create materialized view mv_sync32 as select id as f1,
ndv(kint) from agg_mv_test group by id order by id;""")
- mv_rewrite_success("select id, ndv(kint) from agg_mv_test group by id
order by id;", "mv_sync32")
+ mv_rewrite_any_success("select id, ndv(kint) from agg_mv_test group by id
order by id;", ["mv_sync32", "mv_sync3"])
qt_select_ndv_mv """select id, ndv(kint) from agg_mv_test group by id
order by id;"""
qt_select_covar """select id, covar(kint, kint) from agg_mv_test group by
id order by id;"""
diff --git
a/regression-test/suites/nereids_syntax_p1/mv/aggregate/agg_sync_mv.groovy
b/regression-test/suites/nereids_syntax_p1/mv/aggregate/agg_sync_mv.groovy
index 812aa35a7f3..3aced30b00f 100644
--- a/regression-test/suites/nereids_syntax_p1/mv/aggregate/agg_sync_mv.groovy
+++ b/regression-test/suites/nereids_syntax_p1/mv/aggregate/agg_sync_mv.groovy
@@ -331,7 +331,7 @@ suite("agg_sync_mv") {
qt_select_ndv """select id, ndv(kint) from agg_mv_test group by id order
by id;"""
sql """drop materialized view if exists mv_sync32 on agg_mv_test;"""
createMV("""create materialized view mv_sync32 as select id as f1,
ndv(kint) as xx1 from agg_mv_test group by id order by id;""")
- mv_rewrite_success("select id, ndv(kint) from agg_mv_test group by id
order by id;", "mv_sync32")
+ mv_rewrite_any_success("select id, ndv(kint) from agg_mv_test group by id
order by id;", ["mv_sync32", "mv_sync3"])
qt_select_ndv_mv """select id, ndv(kint) from agg_mv_test group by id
order by id;"""
qt_select_covar """select id, covar(kint, kint) from agg_mv_test group by
id order by id;"""
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
