This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 4bd106b535e [test](mtmv) Fix sync mv not partition in rewrite test and
some other test problems (#46546)
4bd106b535e is described below
commit 4bd106b535e28ddaf2e3d8d93f75698ccd947d4d
Author: seawinde <[email protected]>
AuthorDate: Wed Jan 15 20:34:13 2025 +0800
[test](mtmv) Fix sync mv not partition in rewrite test and some other test
problems (#46546)
### What problem does this PR solve?
Fix sync mv not partition in rewrite because thought mv is already built
wrongly
In the `createMV()` method of `Suite.groovy`, it checks whether the most
recent materialized view of the current database's last table has been
built successfully. If a database has two tables, the synchronization
materialized view of the second table may not be completed, which could
incorrectly lead to the assumption that the build is complete.
To address this issue, modify the test case to ensure that there is at
most one table per database.
---
.../doris/nereids/stats/StatsCalculator.java | 4 +-
.../{unique => unique_rewrite}/unique_rewrite.out | 0
.../nereids_syntax_p0/mv/newMv/multi_slot4.out | 14 +++-
.../show/test_show_create_materialized_view.out | 2 +-
.../org/apache/doris/regression/suite/Suite.groovy | 96 +++++++++++++++-------
.../suites/auth_call/test_ddl_mv_auth.groovy | 4 +-
.../suites/auth_p0/test_select_column_auth.groovy | 2 +-
.../test_mv_case/test_mv_case.groovy | 13 +--
.../create_view_nereids/create_view_use_mv.groovy | 4 +-
.../suites/mtmv_p0/test_iceberg_mtmv.groovy | 28 +++----
.../mtmv_p0/test_paimon_olap_rewrite_mtmv.groovy | 8 ++
.../suites/mv_p0/no_await/no_await.groovy | 39 +++++----
.../unique_rewrite.groovy | 0
.../mv/date_trunc/mv_with_date_trunc.groovy | 2 +-
.../mv/nested_mtmv/nested_mtmv.groovy | 6 +-
.../nereids_syntax_p0/mv/newMv/multi_slot4.groovy | 10 ++-
.../show/test_show_create_materialized_view.groovy | 7 +-
.../test_dup_mv_schema_change.groovy | 4 +-
.../test_uniq_mv_schema_change.groovy | 21 +----
19 files changed, 158 insertions(+), 106 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
index 64019818f5d..5daf3bb254f 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
@@ -501,8 +501,8 @@ public class StatsCalculator extends
DefaultPlanVisitor<Statistics, Void> {
// mv is selected, return its estimated stats
Optional<Statistics> optStats =
cascadesContext.getStatementContext()
.getStatistics(((Relation) olapScan).getRelationId());
- LOG.info("computeOlapScan optStats isPresent {}, tableRowCount is
{}",
- optStats.isPresent(), tableRowCount);
+ LOG.info("computeOlapScan optStats isPresent {}, tableRowCount is
{}, table name is {}",
+ optStats.isPresent(), tableRowCount,
olapTable.getQualifiedName());
if (optStats.isPresent()) {
double selectedPartitionsRowCount =
getSelectedPartitionRowCount(olapScan, tableRowCount);
LOG.info("computeOlapScan optStats is {},
selectedPartitionsRowCount is {}", optStats.get(),
diff --git a/regression-test/data/mv_p0/unique/unique_rewrite.out
b/regression-test/data/mv_p0/unique_rewrite/unique_rewrite.out
similarity index 100%
rename from regression-test/data/mv_p0/unique/unique_rewrite.out
rename to regression-test/data/mv_p0/unique_rewrite/unique_rewrite.out
diff --git a/regression-test/data/nereids_syntax_p0/mv/newMv/multi_slot4.out
b/regression-test/data/nereids_syntax_p0/mv/newMv/multi_slot4.out
index 5b500067986..264a653fd3a 100644
--- a/regression-test/data/nereids_syntax_p0/mv/newMv/multi_slot4.out
+++ b/regression-test/data/nereids_syntax_p0/mv/newMv/multi_slot4.out
@@ -1,14 +1,22 @@
-- This file is automatically generated. You should know what you did if you
want to edit this
-- !select_star --
-4 -4 -4 d
+-4 -4 -4 d
+-4 -4 -4 d
+1 1 1 a
+1 1 1 a
1 1 1 a
2 2 2 b
+2 2 2 b
+2 2 2 b
+3 -3 \N c
+3 -3 \N c
3 -3 \N c
3 -3 \N c
-- !select_mv --
--3 1
-2 7
-3 9
+-3 3
+2 21
+3 27
4 \N
diff --git
a/regression-test/data/query_p0/show/test_show_create_materialized_view.out
b/regression-test/data/query_p0/show/test_show_create_materialized_view.out
index 040ebf56f29..1d874bb14d3 100644
--- a/regression-test/data/query_p0/show/test_show_create_materialized_view.out
+++ b/regression-test/data/query_p0/show/test_show_create_materialized_view.out
@@ -1,4 +1,4 @@
-- This file is automatically generated. You should know what you did if you
want to edit this
-- !cmd --
-table_for_mv_test mv_show_create_materialized_view CREATE
MATERIALIZED VIEW mv_show_create_materialized_view AS\n SELECT id,
name, SUM(value) AS total_value\n FROM table_for_mv_test\n
GROUP BY id, name;\n
+table_for_mv_test mv_show_create_materialized_view \n
CREATE MATERIALIZED VIEW mv_show_create_materialized_view \n AS \n
SELECT id, name, SUM(value) AS total_value\n FROM
table_for_mv_test\n GROUP BY id, name\n
diff --git
a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
index 5cbed97829c..155cb5868bc 100644
---
a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
+++
b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
@@ -742,10 +742,16 @@ class Suite implements GroovyInterceptable {
return result
}
+ // Should use create_sync_mv, this method only check the sync mv in
current db
+ // If has multi sync mv in db, may make mistake
+ @Deprecated
void createMV(String sql) {
(new CreateMVAction(context, sql)).run()
}
+ // Should use create_sync_mv, this method only check the sync mv in
current db
+ // If has multi sync mv in db, may make mistake
+ @Deprecated
void createMV(String sql, String expection) {
(new CreateMVAction(context, sql, expection)).run()
}
@@ -1475,80 +1481,101 @@ class Suite implements GroovyInterceptable {
return debugPoint
}
- void waitingMTMVTaskFinishedByMvName(String mvName) {
+ def waitingMTMVTaskFinishedByMvName = { mvName, dbName = context.dbName ->
Thread.sleep(2000);
- String showTasks = "select
TaskId,JobId,JobName,MvId,Status,MvName,MvDatabaseName,ErrorMsg from
tasks('type'='mv') where MvName = '${mvName}' order by CreateTime ASC"
+ String showTasks = "select
TaskId,JobId,JobName,MvId,Status,MvName,MvDatabaseName,ErrorMsg from
tasks('type'='mv') where MvDatabaseName = '${dbName}' and MvName = '${mvName}'
order by CreateTime DESC LIMIT 1"
String status = "NULL"
List<List<Object>> result
long startTime = System.currentTimeMillis()
long timeoutTimestamp = startTime + 5 * 60 * 1000 // 5 min
- do {
+ List<String> toCheckTaskRow = new ArrayList<>();
+ while (timeoutTimestamp > System.currentTimeMillis() && (status !=
"SUCCESS")) {
result = sql(showTasks)
- logger.info("result: " + result.toString())
- if (!result.isEmpty()) {
- status = result.last().get(4)
- }
+ logger.info("current db is " + dbName + ", showTasks is " +
showTasks)
+ if (result.isEmpty()) {
+ logger.info("waitingMTMVTaskFinishedByMvName toCheckTaskRow is
empty")
+ Thread.sleep(1000);
+ continue;
+ }
+ toCheckTaskRow = result.get(0);
+ status = toCheckTaskRow.get(4)
logger.info("The state of ${showTasks} is ${status}")
Thread.sleep(1000);
- } while (timeoutTimestamp > System.currentTimeMillis() && (status ==
'PENDING' || status == 'RUNNING' || status == 'NULL'))
+ }
if (status != "SUCCESS") {
logger.info("status is not success")
}
Assert.assertEquals("SUCCESS", status)
def show_tables = sql """
- show tables from ${result.last().get(6)};
+ show tables from ${toCheckTaskRow.get(6)};
"""
- def db_id = getDbId(result.last().get(6))
- def table_id = getTableId(result.last().get(6), mvName)
+ def db_id = getDbId(toCheckTaskRow.get(6))
+ def table_id = getTableId(toCheckTaskRow.get(6), mvName)
logger.info("waitingMTMVTaskFinished analyze mv name is " + mvName
- + ", db name is " + result.last().get(6)
+ + ", db name is " + toCheckTaskRow.get(6)
+ ", show_tables are " + show_tables
+ ", db_id is " + db_id
+ ", table_id " + table_id)
- sql "analyze table ${result.last().get(6)}.${mvName} with sync;"
+ sql "analyze table ${toCheckTaskRow.get(6)}.${mvName} with sync;"
}
- void waitingMTMVTaskFinishedByMvNameAllowCancel(String mvName) {
+ def waitingMTMVTaskFinishedByMvNameAllowCancel = {mvName, dbName =
context.dbName ->
Thread.sleep(2000);
- String showTasks = "select
TaskId,JobId,JobName,MvId,Status,MvName,MvDatabaseName,ErrorMsg from
tasks('type'='mv') where MvName = '${mvName}' order by CreateTime ASC"
+ String showTasks = "select
TaskId,JobId,JobName,MvId,Status,MvName,MvDatabaseName,ErrorMsg from
tasks('type'='mv') where MvDatabaseName = '${dbName}' and MvName = '${mvName}'
order by CreateTime DESC LIMIT 1"
+
String status = "NULL"
List<List<Object>> result
long startTime = System.currentTimeMillis()
long timeoutTimestamp = startTime + 5 * 60 * 1000 // 5 min
- do {
+ List<String> toCheckTaskRow = new ArrayList<>();
+ while (timeoutTimestamp > System.currentTimeMillis() && (status !=
"SUCCESS")) {
result = sql(showTasks)
- logger.info("result: " + result.toString())
- if (!result.isEmpty()) {
- status = result.last().get(4)
- }
+ logger.info("current db is " + dbName + ", showTasks result: " +
result.toString())
+ if (result.isEmpty()) {
+ logger.info("waitingMTMVTaskFinishedByMvName toCheckTaskRow is
empty")
+ Thread.sleep(1000);
+ continue;
+ }
+ toCheckTaskRow = result.get(0)
+ status = toCheckTaskRow.get(4)
logger.info("The state of ${showTasks} is ${status}")
Thread.sleep(1000);
- } while (timeoutTimestamp > System.currentTimeMillis() && (status ==
'PENDING' || status == 'RUNNING' || status == 'NULL' || status == 'CANCELED'))
+ }
if (status != "SUCCESS") {
logger.info("status is not success")
assertTrue(result.toString().contains("same table"))
}
// Need to analyze materialized view for cbo to choose the
materialized view accurately
- logger.info("waitingMTMVTaskFinished analyze mv name is " +
result.last().get(5))
- sql "analyze table ${result.last().get(6)}.${mvName} with sync;"
+ logger.info("waitingMTMVTaskFinished analyze mv name is " +
toCheckTaskRow.get(5))
+ sql "analyze table ${toCheckTaskRow.get(6)}.${mvName} with sync;"
}
- void waitingMVTaskFinishedByMvName(String dbName, String tableName) {
+ void waitingMVTaskFinishedByMvName(String dbName, String tableName, String
indexName) {
Thread.sleep(2000)
- String showTasks = "SHOW ALTER TABLE MATERIALIZED VIEW from ${dbName}
where TableName='${tableName}' ORDER BY CreateTime ASC"
+ String showTasks = "SHOW ALTER TABLE MATERIALIZED VIEW from ${dbName}
where TableName='${tableName}' ORDER BY CreateTime DESC"
String status = "NULL"
List<List<Object>> result
long startTime = System.currentTimeMillis()
long timeoutTimestamp = startTime + 5 * 60 * 1000 // 5 min
- do {
+ List<String> toCheckTaskRow = new ArrayList<>();
+ while (timeoutTimestamp > System.currentTimeMillis() && (status !=
'FINISHED')) {
result = sql(showTasks)
- logger.info("result: " + result.toString())
- if (!result.isEmpty()) {
- status = result.last().get(8)
+ logger.info("crrent db is " + dbName + ", showTasks result: " +
result.toString())
+ // just consider current db
+ for (List<String> taskRow : result) {
+ if (taskRow.get(5).equals(indexName)) {
+ toCheckTaskRow = taskRow;
+ }
+ }
+ if (toCheckTaskRow.isEmpty()) {
+ logger.info("waitingMVTaskFinishedByMvName toCheckTaskRow is
empty")
+ Thread.sleep(1000);
+ continue;
}
+ status = toCheckTaskRow.get(8)
logger.info("The state of ${showTasks} is ${status}")
Thread.sleep(1000);
- } while (timeoutTimestamp > System.currentTimeMillis() && (status !=
'FINISHED'))
+ }
if (status != "FINISHED") {
logger.info("status is not success")
}
@@ -1925,6 +1952,15 @@ class Suite implements GroovyInterceptable {
return isReady
}
+ def create_sync_mv = { db, table_name, mv_name, mv_sql ->
+ sql """DROP MATERIALIZED VIEW IF EXISTS ${mv_name} ON ${table_name};"""
+ sql"""
+ CREATE MATERIALIZED VIEW ${mv_name}
+ AS ${mv_sql}
+ """
+ waitingMVTaskFinishedByMvName(db, table_name, mv_name)
+ }
+
def create_async_mv = { db, mv_name, mv_sql ->
sql """DROP MATERIALIZED VIEW IF EXISTS ${db}.${mv_name}"""
diff --git a/regression-test/suites/auth_call/test_ddl_mv_auth.groovy
b/regression-test/suites/auth_call/test_ddl_mv_auth.groovy
index 4dbf54fdf0d..0701abb8a8b 100644
--- a/regression-test/suites/auth_call/test_ddl_mv_auth.groovy
+++ b/regression-test/suites/auth_call/test_ddl_mv_auth.groovy
@@ -80,9 +80,9 @@ suite("test_ddl_mv_auth","p0,auth_call") {
connect(user, "${pwd}", context.config.jdbcUrl) {
sql """use ${dbName}"""
sql """create materialized view ${mvName} as select username from
${dbName}.${tableName};"""
- waitingMVTaskFinishedByMvName(dbName, tableName)
+ waitingMVTaskFinishedByMvName(dbName, tableName, mvName)
sql """alter table ${dbName}.${tableName} add rollup
${rollupName}(username)"""
- waitingMVTaskFinishedByMvName(dbName, tableName)
+ waitingMVTaskFinishedByMvName(dbName, tableName, rollupName)
def mv_res = sql """desc ${dbName}.${tableName} all;"""
logger.info("mv_res: " + mv_res)
diff --git a/regression-test/suites/auth_p0/test_select_column_auth.groovy
b/regression-test/suites/auth_p0/test_select_column_auth.groovy
index 36cc2a0a09c..1e93981966d 100644
--- a/regression-test/suites/auth_p0/test_select_column_auth.groovy
+++ b/regression-test/suites/auth_p0/test_select_column_auth.groovy
@@ -69,7 +69,7 @@ suite("test_select_column_auth","p0,auth") {
(3, "333");
"""
sql """refresh MATERIALIZED VIEW ${dbName}.${mtmv_name} auto"""
- waitingMTMVTaskFinishedByMvName(mtmv_name)
+ waitingMTMVTaskFinishedByMvName(mtmv_name, dbName)
sql """grant select_priv on regression_test to ${user}"""
diff --git
a/regression-test/suites/correctness_p0/test_mv_case/test_mv_case.groovy
b/regression-test/suites/correctness_p0/test_mv_case/test_mv_case.groovy
index 8e548eb27ea..eae3d1fb359 100644
--- a/regression-test/suites/correctness_p0/test_mv_case/test_mv_case.groovy
+++ b/regression-test/suites/correctness_p0/test_mv_case/test_mv_case.groovy
@@ -16,6 +16,7 @@
// under the License.
suite("test_mv_case") {
+
sql """drop table if exists test_table_aaa2;"""
sql """CREATE TABLE `test_table_aaa2` (
`ordernum` varchar(65533) NOT NULL ,
@@ -29,7 +30,7 @@ suite("test_mv_case") {
"replication_allocation" = "tag.location.default: 1"
);"""
sql """DROP MATERIALIZED VIEW IF EXISTS ods_zn_dnt_max1 ON
test_table_aaa2;"""
- createMV("""create materialized view ods_zn_dnt_max1 as
+ create_sync_mv(context.dbName, "test_table_aaa2", "ods_zn_dnt_max1", """
select ordernum,max(dnt) as dnt from test_table_aaa2
group by ordernum
ORDER BY ordernum;""")
@@ -92,7 +93,7 @@ suite("test_mv_case") {
)
"""
sql """insert into tb1 select id,map_agg(a, b) from(select 123 id,3 a,'5'
b union all select 123 id, 6 a, '8' b) aa group by id"""
- createMV ("""CREATE MATERIALIZED VIEW mv1 BUILD IMMEDIATE REFRESH COMPLETE
ON SCHEDULE EVERY 10 MINUTE DUPLICATE KEY(info_id) DISTRIBUTED BY
HASH(`info_id`) BUCKETS 2 PROPERTIES (
+ sql"""CREATE MATERIALIZED VIEW mv1 BUILD IMMEDIATE REFRESH COMPLETE ON
SCHEDULE EVERY 10 MINUTE DUPLICATE KEY(info_id) DISTRIBUTED BY HASH(`info_id`)
BUCKETS 2 PROPERTIES (
"replication_allocation" = "tag.location.default: 1",
"min_load_replica_num" = "-1",
"is_being_synced" = "false",
@@ -112,8 +113,9 @@ suite("test_mv_case") {
cast(a.id as bigint) info_id,
map_infos
from
- tb1 a;""")
- createMV ("""CREATE MATERIALIZED VIEW mv2 BUILD IMMEDIATE REFRESH COMPLETE
ON SCHEDULE EVERY 10 MINUTE DUPLICATE KEY(info_id) DISTRIBUTED BY
HASH(`info_id`) BUCKETS 2 PROPERTIES (
+ tb1 a;"""
+ waitingMTMVTaskFinishedByMvName("mv1")
+ sql """CREATE MATERIALIZED VIEW mv2 BUILD IMMEDIATE REFRESH COMPLETE ON
SCHEDULE EVERY 10 MINUTE DUPLICATE KEY(info_id) DISTRIBUTED BY HASH(`info_id`)
BUCKETS 2 PROPERTIES (
"replication_allocation" = "tag.location.default: 1",
"min_load_replica_num" = "-1",
"is_being_synced" = "false",
@@ -132,6 +134,7 @@ suite("test_mv_case") {
info_id,
map_infos
from
- mv1 a;""")
+ mv1 a;"""
+ waitingMTMVTaskFinishedByMvName("mv2")
qt_select_mv """ select * from mv2 """
}
diff --git
a/regression-test/suites/ddl_p0/create_view_nereids/create_view_use_mv.groovy
b/regression-test/suites/ddl_p0/create_view_nereids/create_view_use_mv.groovy
index 295b195aa58..7b6069968f4 100644
---
a/regression-test/suites/ddl_p0/create_view_nereids/create_view_use_mv.groovy
+++
b/regression-test/suites/ddl_p0/create_view_nereids/create_view_use_mv.groovy
@@ -48,8 +48,8 @@ suite("create_view_use_mv") {
(3, 1, 1, 2, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', null, 'c', 'd',
'xxxxxxxxx', '2023-10-19'),
(1, 3, 2, 2, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17',
'a', 'b', 'yyyyyyyyy', '2023-10-17');"""
- createMV("""
- CREATE MATERIALIZED VIEW t_mv_mv AS select
+ create_sync_mv(context.dbName, "orders", "t_mv_mv", """
+ select
o_orderkey,
sum(o_totalprice) as sum_total,
max(o_totalprice) as max_total,
diff --git a/regression-test/suites/mtmv_p0/test_iceberg_mtmv.groovy
b/regression-test/suites/mtmv_p0/test_iceberg_mtmv.groovy
index 36c0d3f120e..8dd16ea571f 100644
--- a/regression-test/suites/mtmv_p0/test_iceberg_mtmv.groovy
+++ b/regression-test/suites/mtmv_p0/test_iceberg_mtmv.groovy
@@ -107,35 +107,35 @@ suite("test_iceberg_mtmv",
"p0,external,iceberg,external_docker,external_docker_
sql """insert into ${catalog_name}.${icebergDb}.${icebergTable1}
values ('2024-10-26 01:02:03', 1), ('2024-10-27 01:02:03', 2), ('2024-10-27
21:02:03', 3)"""
sql """CREATE MATERIALIZED VIEW ${mvName1} BUILD DEFERRED REFRESH AUTO
ON MANUAL partition by(`ts`) DISTRIBUTED BY RANDOM BUCKETS 2 PROPERTIES
('replication_num' = '1') as SELECT * FROM
${catalog_name}.${icebergDb}.${icebergTable1}"""
sql """REFRESH MATERIALIZED VIEW ${mvName1} complete"""
- waitingMTMVTaskFinishedByMvName(mvName1)
+ waitingMTMVTaskFinishedByMvName(mvName1, dbName)
qt_test_ts_refresh1 "select * from ${mvName1} order by value"
sql """insert into ${catalog_name}.${icebergDb}.${icebergTable1}
values ('2024-10-26 21:02:03', 4)"""
sql """REFRESH MATERIALIZED VIEW ${mvName1} auto"""
- waitingMTMVTaskFinishedByMvName(mvName1)
+ waitingMTMVTaskFinishedByMvName(mvName1, dbName)
qt_test_ts_refresh2 """select * from ${mvName1} order by value"""
sql """insert into ${catalog_name}.${icebergDb}.${icebergTable1}
values ('2024-10-26 01:22:03', 5), ('2024-10-27 01:12:03', 6);"""
sql """REFRESH MATERIALIZED VIEW ${mvName1}
partitions(p_20241026000000_20241027000000);"""
- waitingMTMVTaskFinishedByMvName(mvName1)
+ waitingMTMVTaskFinishedByMvName(mvName1, dbName)
qt_test_ts_refresh3 """select * from ${mvName1} order by value"""
sql """REFRESH MATERIALIZED VIEW ${mvName1} auto"""
- waitingMTMVTaskFinishedByMvName(mvName1)
+ waitingMTMVTaskFinishedByMvName(mvName1, dbName)
qt_test_ts_refresh4 """select * from ${mvName1} order by value"""
sql """insert into ${catalog_name}.${icebergDb}.${icebergTable1}
values ('2024-10-28 01:22:03', 7);"""
sql """REFRESH MATERIALIZED VIEW ${mvName1}
partitions(p_20241026000000_20241027000000);"""
- waitingMTMVTaskFinishedByMvName(mvName1)
+ waitingMTMVTaskFinishedByMvName(mvName1, dbName)
qt_test_ts_refresh5 """select * from ${mvName1} order by value"""
sql """REFRESH MATERIALIZED VIEW ${mvName1} auto"""
- waitingMTMVTaskFinishedByMvName(mvName1)
+ waitingMTMVTaskFinishedByMvName(mvName1, dbName)
qt_test_ts_refresh6 """select * from ${mvName1} order by value"""
sql """insert into ${catalog_name}.${icebergDb}.${icebergTable1}
values (null, 8);"""
sql """REFRESH MATERIALIZED VIEW ${mvName1} auto"""
- waitingMTMVTaskFinishedByMvName(mvName1)
+ waitingMTMVTaskFinishedByMvName(mvName1, dbName)
qt_test_ts_refresh_null """select * from ${mvName1} order by value"""
def showPartitionsResult = sql """show partitions from ${mvName1}"""
@@ -176,25 +176,25 @@ suite("test_iceberg_mtmv",
"p0,external,iceberg,external_docker,external_docker_
sql """insert into ${catalog_name}.${icebergDb}.${icebergTable2}
values ('2024-08-26', 1), ('2024-09-17', 2), ('2024-09-27', 3);"""
sql """CREATE MATERIALIZED VIEW ${mvName2} BUILD DEFERRED REFRESH AUTO
ON MANUAL partition by(`d`) DISTRIBUTED BY RANDOM BUCKETS 2 PROPERTIES
('replication_num' = '1') as SELECT * FROM
${catalog_name}.${icebergDb}.${icebergTable2}"""
sql """REFRESH MATERIALIZED VIEW ${mvName2} complete"""
- waitingMTMVTaskFinishedByMvName(mvName2)
+ waitingMTMVTaskFinishedByMvName(mvName2, dbName)
qt_test_d_refresh1 "select * from ${mvName2} order by value"
sql """insert into ${catalog_name}.${icebergDb}.${icebergTable2}
values ('2024-09-01', 4);"""
sql """REFRESH MATERIALIZED VIEW ${mvName2} auto"""
- waitingMTMVTaskFinishedByMvName(mvName2)
+ waitingMTMVTaskFinishedByMvName(mvName2, dbName)
qt_test_d_refresh2 "select * from ${mvName2} order by value"
sql """insert into ${catalog_name}.${icebergDb}.${icebergTable2}
values ('2024-08-22', 5), ('2024-09-30', 6);"""
sql """REFRESH MATERIALIZED VIEW ${mvName2}
partitions(p_20240801_20240901);"""
- waitingMTMVTaskFinishedByMvName(mvName2)
+ waitingMTMVTaskFinishedByMvName(mvName2, dbName)
qt_test_d_refresh3 "select * from ${mvName2} order by value"
sql """REFRESH MATERIALIZED VIEW ${mvName2}
partitions(p_20240901_20241001);"""
- waitingMTMVTaskFinishedByMvName(mvName2)
+ waitingMTMVTaskFinishedByMvName(mvName2, dbName)
qt_test_d_refresh4 "select * from ${mvName2} order by value"
sql """insert into ${catalog_name}.${icebergDb}.${icebergTable2}
values ('2024-10-28', 7);"""
sql """REFRESH MATERIALIZED VIEW ${mvName2} auto"""
- waitingMTMVTaskFinishedByMvName(mvName2)
+ waitingMTMVTaskFinishedByMvName(mvName2, dbName)
qt_test_d_refresh5 "select * from ${mvName2} order by value"
showPartitionsResult = sql """show partitions from ${mvName2}"""
@@ -240,7 +240,7 @@ suite("test_iceberg_mtmv",
"p0,external,iceberg,external_docker,external_docker_
// refresh one partiton
sql """REFRESH MATERIALIZED VIEW ${mvName}
partitions(p_20240101000000_20240102000000);"""
- waitingMTMVTaskFinishedByMvName(mvName)
+ waitingMTMVTaskFinishedByMvName(mvName, dbName)
order_qt_refresh_one_partition "SELECT * FROM ${mvName} "
def explainOnePartition = sql """ explain ${mvSql} """
logger.info("explainOnePartition: " + explainOnePartition.toString())
@@ -250,7 +250,7 @@ suite("test_iceberg_mtmv",
"p0,external,iceberg,external_docker,external_docker_
//refresh auto
sql """REFRESH MATERIALIZED VIEW ${mvName} auto"""
- waitingMTMVTaskFinishedByMvName(mvName)
+ waitingMTMVTaskFinishedByMvName(mvName, dbName)
order_qt_refresh_auto "SELECT * FROM ${mvName} "
def explainAllPartition = sql """ explain ${mvSql}; """
logger.info("explainAllPartition: " + explainAllPartition.toString())
diff --git
a/regression-test/suites/mtmv_p0/test_paimon_olap_rewrite_mtmv.groovy
b/regression-test/suites/mtmv_p0/test_paimon_olap_rewrite_mtmv.groovy
index a3ac1c048d3..7a77cdc4590 100644
--- a/regression-test/suites/mtmv_p0/test_paimon_olap_rewrite_mtmv.groovy
+++ b/regression-test/suites/mtmv_p0/test_paimon_olap_rewrite_mtmv.groovy
@@ -87,6 +87,10 @@ suite("test_paimon_olap_rewrite_mtmv",
"p0,external,mtmv,external_docker,externa
def explainOnePartition = sql """ explain ${mvSql} """
logger.info("explainOnePartition: " + explainOnePartition.toString())
+
+ def explain_memo_plan = sql """ explain memo plan ${mvSql} """
+ logger.info("explain_memo_plan: " + explain_memo_plan.toString())
+
assertTrue(explainOnePartition.toString().contains("VUNION"))
order_qt_refresh_one_partition_rewrite "${mvSql}"
@@ -104,6 +108,10 @@ suite("test_paimon_olap_rewrite_mtmv",
"p0,external,mtmv,external_docker,externa
def explainAllPartition = sql """ explain ${mvSql}; """
logger.info("explainAllPartition: " + explainAllPartition.toString())
+
+ def explainMemoPlan = sql """ explain memo plan ${mvSql}; """
+ logger.info("explainMemoPlan: " + explainMemoPlan.toString())
+
assertTrue(explainAllPartition.toString().contains("VOlapScanNode"))
order_qt_refresh_all_partition_rewrite "${mvSql}"
diff --git a/regression-test/suites/mv_p0/no_await/no_await.groovy
b/regression-test/suites/mv_p0/no_await/no_await.groovy
index 866e4fdd5d3..3eab03aa7e4 100644
--- a/regression-test/suites/mv_p0/no_await/no_await.groovy
+++ b/regression-test/suites/mv_p0/no_await/no_await.groovy
@@ -19,6 +19,8 @@ import org.codehaus.groovy.runtime.IOGroovyMethods
suite ("no_await") {
+ String db = context.config.getDbNameByFile(context.file)
+
def tblName = "agg_have_dup_base_no_await"
def waitDrop = {
def try_times = 1000
@@ -58,92 +60,93 @@ suite ("no_await") {
"""
sql "insert into ${tblName} select e1, -4, -4, 'd' from (select 1 k1) as t
lateral view explode_numbers(10000) tmp1 as e1;"
// do not await
- sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from
${tblName} group by k1;"
+ create_sync_mv(db, tblName, "k12s3m", """select k1,sum(k2),max(k2) from
${tblName} group by k1;""")
+
sql "insert into ${tblName} select -4, -4, -4, \'d\'"
qt_mv "select sum(k1) from ${tblName}"
waitDrop()
- sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from
${tblName} group by k1;"
+ create_sync_mv(db, tblName, "k12s3m", """select k1,sum(k2),max(k2) from
${tblName} group by k1;""")
sql "insert into ${tblName} select -4, -4, -4, \'d\'"
qt_mv "select sum(k1) from ${tblName}"
waitDrop()
- sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from
${tblName} group by k1;"
+ create_sync_mv(db, tblName, "k12s3m", """select k1,sum(k2),max(k2) from
${tblName} group by k1;""")
sql "insert into ${tblName} select -4, -4, -4, \'d\'"
qt_mv "select sum(k1) from ${tblName}"
waitDrop()
- sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from
${tblName} group by k1;"
+ create_sync_mv(db, tblName, "k12s3m", """select k1,sum(k2),max(k2) from
${tblName} group by k1;""")
sql "insert into ${tblName} select -4, -4, -4, \'d\'"
qt_mv "select sum(k1) from ${tblName}"
waitDrop()
- sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from
${tblName} group by k1;"
+ create_sync_mv(db, tblName, "k12s3m", """select k1,sum(k2),max(k2) from
${tblName} group by k1;""")
sql "insert into ${tblName} select -4, -4, -4, \'d\'"
qt_mv "select sum(k1) from ${tblName}"
waitDrop()
- sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from
${tblName} group by k1;"
+ create_sync_mv(db, tblName, "k12s3m", """select k1,sum(k2),max(k2) from
${tblName} group by k1;""")
sql "insert into ${tblName} select -4, -4, -4, \'d\'"
qt_mv "select sum(k1) from ${tblName}"
waitDrop()
- sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from
${tblName} group by k1;"
+ create_sync_mv(db, tblName, "k12s3m", """select k1,sum(k2),max(k2) from
${tblName} group by k1;""")
sql "insert into ${tblName} select -4, -4, -4, \'d\'"
qt_mv "select sum(k1) from ${tblName}"
waitDrop()
- sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from
${tblName} group by k1;"
+ create_sync_mv(db, tblName, "k12s3m", """select k1,sum(k2),max(k2) from
${tblName} group by k1;""")
sql "insert into ${tblName} select -4, -4, -4, \'d\'"
qt_mv "select sum(k1) from ${tblName}"
waitDrop()
- sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from
${tblName} group by k1;"
+ create_sync_mv(db, tblName, "k12s3m", """select k1,sum(k2),max(k2) from
${tblName} group by k1;""")
sql "insert into ${tblName} select -4, -4, -4, \'d\'"
qt_mv "select sum(k1) from ${tblName}"
waitDrop()
- sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from
${tblName} group by k1;"
+ create_sync_mv(db, tblName, "k12s3m", """select k1,sum(k2),max(k2) from
${tblName} group by k1;""")
sql "insert into ${tblName} select -4, -4, -4, \'d\'"
qt_mv "select sum(k1) from ${tblName}"
waitDrop()
- sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from
${tblName} group by k1;"
+ create_sync_mv(db, tblName, "k12s3m", """select k1,sum(k2),max(k2) from
${tblName} group by k1;""")
sql "insert into ${tblName} select -4, -4, -4, \'d\'"
qt_mv "select sum(k1) from ${tblName}"
waitDrop()
- sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from
${tblName} group by k1;"
+ create_sync_mv(db, tblName, "k12s3m", """select k1,sum(k2),max(k2) from
${tblName} group by k1;""")
sql "insert into ${tblName} select -4, -4, -4, \'d\'"
qt_mv "select sum(k1) from ${tblName}"
waitDrop()
- sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from
${tblName} group by k1;"
+ create_sync_mv(db, tblName, "k12s3m", """select k1,sum(k2),max(k2) from
${tblName} group by k1;""")
sql "insert into ${tblName} select -4, -4, -4, \'d\'"
qt_mv "select sum(k1) from ${tblName}"
waitDrop()
- sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from
${tblName} group by k1;"
+ create_sync_mv(db, tblName, "k12s3m", """select k1,sum(k2),max(k2) from
${tblName} group by k1;""")
sql "insert into ${tblName} select -4, -4, -4, \'d\'"
qt_mv "select sum(k1) from ${tblName}"
waitDrop()
- sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from
${tblName} group by k1;"
+ create_sync_mv(db, tblName, "k12s3m", """select k1,sum(k2),max(k2) from
${tblName} group by k1;""")
sql "insert into ${tblName} select -4, -4, -4, \'d\'"
qt_mv "select sum(k1) from ${tblName}"
waitDrop()
- sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from
${tblName} group by k1;"
+ create_sync_mv(db, tblName, "k12s3m", """select k1,sum(k2),max(k2) from
${tblName} group by k1;""")
sql "insert into ${tblName} select -4, -4, -4, \'d\'"
qt_mv "select sum(k1) from ${tblName}"
waitDrop()
- sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from
${tblName} group by k1;"
+ create_sync_mv(db, tblName, "k12s3m", """select k1,sum(k2),max(k2) from
${tblName} group by k1;""")
sql "insert into ${tblName} select -4, -4, -4, \'d\'"
qt_mv "select sum(k1) from ${tblName}"
waitDrop()
- sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from
${tblName} group by k1;"
+ create_sync_mv(db, tblName, "k12s3m", """select k1,sum(k2),max(k2) from
${tblName} group by k1;""")
sql "insert into ${tblName} select -4, -4, -4, \'d\'"
qt_mv "select sum(k1) from ${tblName}"
}
diff --git a/regression-test/suites/mv_p0/unique/unique_rewrite.groovy
b/regression-test/suites/mv_p0/unique_rewrite/unique_rewrite.groovy
similarity index 100%
rename from regression-test/suites/mv_p0/unique/unique_rewrite.groovy
rename to regression-test/suites/mv_p0/unique_rewrite/unique_rewrite.groovy
diff --git
a/regression-test/suites/nereids_rules_p0/mv/date_trunc/mv_with_date_trunc.groovy
b/regression-test/suites/nereids_rules_p0/mv/date_trunc/mv_with_date_trunc.groovy
index 58b2e7bbdd8..e88f4132eac 100644
---
a/regression-test/suites/nereids_rules_p0/mv/date_trunc/mv_with_date_trunc.groovy
+++
b/regression-test/suites/nereids_rules_p0/mv/date_trunc/mv_with_date_trunc.groovy
@@ -1447,7 +1447,7 @@ suite("mv_with_date_trunc") {
logger.info("lineitem table stats: " + result)
result = sql """show index stats lineitem lineitem"""
logger.info("lineitem index stats: " + result)
- mv_rewrite_success(query4_0, "mv4_0")
+ mv_rewrite_success(query4_0, "mv4_0", true,
is_partition_statistics_ready(db, ["lineitem", "mv4_0"]))
order_qt_query4_0_after "${query4_0}"
sql """ DROP MATERIALIZED VIEW IF EXISTS mv4_0"""
diff --git
a/regression-test/suites/nereids_rules_p0/mv/nested_mtmv/nested_mtmv.groovy
b/regression-test/suites/nereids_rules_p0/mv/nested_mtmv/nested_mtmv.groovy
index 4870ec99e65..1972c2d505b 100644
--- a/regression-test/suites/nereids_rules_p0/mv/nested_mtmv/nested_mtmv.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/nested_mtmv/nested_mtmv.groovy
@@ -742,10 +742,12 @@ suite("nested_mtmv") {
mv_rewrite_any_success(sql_2, [mv_1, mv_2])
compare_res(sql_2 + " order by 1,2,3,4,5,6,7,8,9,10,11,12,13")
- mv_rewrite_any_success(sql_3, [mv_3, mv_4])
+ // level 1 maybe use mv_1 and mv_2, this also meets expectation
+ mv_rewrite_any_success(sql_3, [mv_3, mv_4, mv_1, mv_2])
compare_res(sql_3 + " order by 1,2,3,4,5,6,7,8,9,10,11,12,13")
- mv_rewrite_any_success(sql_4, [mv_3, mv_4])
+ // level 1 maybe use mv_1 and mv_2, this also meets expectation
+ mv_rewrite_any_success(sql_4, [mv_3, mv_4, mv_1, mv_2])
compare_res(sql_4 + " order by 1,2,3,4,5,6,7,8,9,10,11,12,13")
mv_rewrite_any_success(sql_5, [mv_3, mv_4, mv_5])
diff --git
a/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot4.groovy
b/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot4.groovy
index 7a49dbc4b37..32e1684fdc9 100644
--- a/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot4.groovy
+++ b/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot4.groovy
@@ -33,21 +33,29 @@ suite ("multi_slot4") {
"""
sql "insert into multi_slot4 select 1,1,1,'a';"
+ sql "insert into multi_slot4 select 1,1,1,'a';"
+ sql "insert into multi_slot4 select 1,1,1,'a';"
+ sql "insert into multi_slot4 select 2,2,2,'b';"
+ sql "insert into multi_slot4 select 2,2,2,'b';"
sql "insert into multi_slot4 select 2,2,2,'b';"
sql "insert into multi_slot4 select 3,-3,null,'c';"
sql "insert into multi_slot4 select 3,-3,null,'c';"
+ sql "insert into multi_slot4 select 3,-3,null,'c';"
+ sql "insert into multi_slot4 select 3,-3,null,'c';"
createMV ("create materialized view k1p2ap3ps as select
k1+1,sum(abs(k2+2)+k3+3) from multi_slot4 group by k1+1;")
sleep(3000)
+ sql "insert into multi_slot4 select -4,-4,-4,'d';"
+ sql "insert into multi_slot4 select -4,-4,-4,'d';"
sql "insert into multi_slot4 select -4,-4,-4,'d';"
sql "SET experimental_enable_nereids_planner=true"
sql "SET enable_fallback_to_original_planner=false"
sql "analyze table multi_slot4 with sync;"
- sql """alter table multi_slot4 modify column k1 set stats
('row_count'='5');"""
+ sql """alter table multi_slot4 modify column k1 set stats
('row_count'='13');"""
sql """set enable_stats=false;"""
diff --git
a/regression-test/suites/query_p0/show/test_show_create_materialized_view.groovy
b/regression-test/suites/query_p0/show/test_show_create_materialized_view.groovy
index 9550a7fec3d..56f5d655255 100644
---
a/regression-test/suites/query_p0/show/test_show_create_materialized_view.groovy
+++
b/regression-test/suites/query_p0/show/test_show_create_materialized_view.groovy
@@ -30,12 +30,11 @@ suite("test_show_create_materialized_view",
"query,arrow_flight_sql") {
DISTRIBUTED BY HASH(id) BUCKETS 5
PROPERTIES ("replication_num" = "1");
"""
-
- createMV("""CREATE MATERIALIZED VIEW ${mvName} AS
+
+ create_sync_mv(context.dbName, tableName, mvName, """
SELECT id, name, SUM(value) AS total_value
FROM ${tableName}
- GROUP BY id, name;
- """)
+ GROUP BY id, name""")
checkNereidsExecute("""SHOW CREATE MATERIALIZED VIEW ${mvName} ON
${tableName};""")
qt_cmd("""SHOW CREATE MATERIALIZED VIEW ${mvName} ON ${tableName};""")
diff --git
a/regression-test/suites/schema_change_p0/test_dup_mv_schema_change.groovy
b/regression-test/suites/schema_change_p0/test_dup_mv_schema_change.groovy
index 713c470436e..a6ad20ec623 100644
--- a/regression-test/suites/schema_change_p0/test_dup_mv_schema_change.groovy
+++ b/regression-test/suites/schema_change_p0/test_dup_mv_schema_change.groovy
@@ -70,7 +70,7 @@ suite ("test_dup_mv_schema_change") {
"""
//add materialized view
- createMV("create materialized view mv1 as select date, user_id, city,
age from ${tableName};")
+ create_sync_mv(context.dbName, tableName, "mv1", """select date,
user_id, city, age from ${tableName}""")
// alter and test light schema change
if (!isCloudMode()) {
@@ -78,7 +78,7 @@ suite ("test_dup_mv_schema_change") {
}
//add materialized view
- createMV("create materialized view mv2 as select date, user_id, city,
age, cost from ${tableName};")
+ create_sync_mv(context.dbName, tableName, "mv2", """select date,
user_id, city, age, cost from ${tableName}""")
sql """ INSERT INTO ${tableName} VALUES
(2, '2017-10-01', 'Beijing', 10, 1, '2020-01-02',
'2020-01-02', '2020-01-02', 1, 31, 21)
diff --git
a/regression-test/suites/schema_change_p0/test_uniq_mv_schema_change.groovy
b/regression-test/suites/schema_change_p0/test_uniq_mv_schema_change.groovy
index eba6036c30a..1d8fdd4d1e7 100644
--- a/regression-test/suites/schema_change_p0/test_uniq_mv_schema_change.groovy
+++ b/regression-test/suites/schema_change_p0/test_uniq_mv_schema_change.groovy
@@ -21,20 +21,7 @@ import org.awaitility.Awaitility
suite ("test_uniq_mv_schema_change") {
def tableName = "schema_change_uniq_mv_regression_test"
- def getMVJobState = { tbName ->
- def jobStateResult = sql """ SHOW ALTER TABLE MATERIALIZED VIEW
WHERE TableName='${tbName}' ORDER BY CreateTime DESC LIMIT 1 """
- return jobStateResult[0][8]
- }
- def waitForJob = (tbName, timeout) -> {
- Awaitility.await().atMost(timeout,
TimeUnit.SECONDS).with().pollDelay(100, TimeUnit.MILLISECONDS).await().until(()
-> {
- String result = getMVJobState(tbName)
- if (result == "FINISHED") {
- return true;
- }
- return false;
- });
- // when timeout awaitlity will raise a exception.
- }
+
try {
String backend_id;
@@ -78,8 +65,7 @@ suite ("test_uniq_mv_schema_change") {
//add materialized view
def mvName = "mv1"
- sql "create materialized view ${mvName} as select user_id, date, city,
age, sex from ${tableName};"
- waitForJob(tableName, 3000)
+ create_sync_mv(context.dbName, tableName, mvName, """select user_id, date,
city, age, sex from ${tableName}""")
// alter and test light schema change
if (!isCloudMode()) {
@@ -88,8 +74,7 @@ suite ("test_uniq_mv_schema_change") {
//add materialized view
def mvName2 = "mv2"
- sql "create materialized view ${mvName2} as select user_id, date, city,
age, sex, cost from ${tableName};"
- waitForJob(tableName, 3000)
+ create_sync_mv(context.dbName, tableName, mvName2, """select user_id,
date, city, age, sex, cost from ${tableName};""")
sql """ INSERT INTO ${tableName} VALUES
(2, '2017-10-01', 'Beijing', 10, 1, '2020-01-02', '2020-01-02',
'2020-01-02', 1, 31, 21)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
