This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 1c58147f7ed [fix](mtmv)fix in the scenario of recreating a table, the
materialized view may assume that the data has not changed (#41762)
1c58147f7ed is described below
commit 1c58147f7ed634fc925d1b319a1d2adc9e45fa7a
Author: zhangdong <[email protected]>
AuthorDate: Wed Oct 23 18:37:31 2024 +0800
[fix](mtmv)fix in the scenario of recreating a table, the materialized view
may assume that the data has not changed (#41762)
snapshot of table,only compare tableName and version,when recerate
table,and if the version remains unchanged 。
MV will assume that the data has not changed
---
.../java/org/apache/doris/catalog/OlapTable.java | 2 +-
.../doris/mtmv/MTMVRefreshPartitionSnapshot.java | 34 ++++++++
.../data/mtmv_p0/test_recreate_table_mtmv.out | 13 ++++
.../suites/mtmv_p0/test_recreate_table_mtmv.groovy | 91 ++++++++++++++++++++++
4 files changed, 139 insertions(+), 1 deletion(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
index 1ad1df536d5..2d97bdc6fbc 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
@@ -3281,7 +3281,7 @@ public class OlapTable extends Table implements
MTMVRelatedTableIf, GsonPostProc
public MTMVSnapshotIf getTableSnapshot(MTMVRefreshContext context) {
Map<Long, Long> tableVersions =
context.getBaseVersions().getTableVersions();
long visibleVersion = tableVersions.containsKey(id) ?
tableVersions.get(id) : getVisibleVersion();
- return new MTMVVersionSnapshot(visibleVersion);
+ return new MTMVVersionSnapshot(visibleVersion, id);
}
@Override
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRefreshPartitionSnapshot.java
b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRefreshPartitionSnapshot.java
index fa17ed76666..a8de5b6597b 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRefreshPartitionSnapshot.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRefreshPartitionSnapshot.java
@@ -20,6 +20,7 @@ package org.apache.doris.mtmv;
import org.apache.doris.catalog.MTMV;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Partition;
+import org.apache.doris.catalog.TableIf;
import org.apache.doris.common.AnalysisException;
import com.google.common.collect.Maps;
@@ -89,6 +90,13 @@ public class MTMVRefreshPartitionSnapshot {
} catch (Throwable e) {
LOG.warn("MTMV compatibleTables failed, mtmv: {}", mtmv.getName(),
e);
}
+
+ try {
+ // snapshot add tableId resolve problem of recreate table
+ compatibleTablesSnapshot();
+ } catch (Throwable e) {
+ LOG.warn("MTMV compatibleTables failed, mtmv: {}", mtmv.getName(),
e);
+ }
}
private void compatiblePartitions(MTMV mtmv) throws AnalysisException {
@@ -116,6 +124,32 @@ public class MTMVRefreshPartitionSnapshot {
return false;
}
+ private void compatibleTablesSnapshot() {
+ if (!checkHasDataWithoutTableId()) {
+ return;
+ }
+ for (Entry<BaseTableInfo, MTMVSnapshotIf> entry :
tablesInfo.entrySet()) {
+ MTMVVersionSnapshot versionSnapshot = (MTMVVersionSnapshot)
entry.getValue();
+ if (versionSnapshot.getId() == 0) {
+ try {
+ TableIf table = MTMVUtil.getTable(entry.getKey());
+ versionSnapshot.setId(table.getId());
+ } catch (AnalysisException e) {
+ LOG.warn("MTMV compatibleTablesSnapshot failed, can not
get table by: {}", entry.getKey());
+ }
+ }
+ }
+ }
+
+ private boolean checkHasDataWithoutTableId() {
+ for (MTMVSnapshotIf snapshot : tablesInfo.values()) {
+ if (snapshot instanceof MTMVVersionSnapshot &&
((MTMVVersionSnapshot) snapshot).getId() == 0) {
+ return true;
+ }
+ }
+ return false;
+ }
+
private void compatibleTables(MTMV mtmv) {
if (tables.size() == tablesInfo.size()) {
return;
diff --git a/regression-test/data/mtmv_p0/test_recreate_table_mtmv.out
b/regression-test/data/mtmv_p0/test_recreate_table_mtmv.out
new file mode 100644
index 00000000000..688518a2a4d
--- /dev/null
+++ b/regression-test/data/mtmv_p0/test_recreate_table_mtmv.out
@@ -0,0 +1,13 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !mv1 --
+21 mysql test1
+
+-- !mv2 --
+mysql 1
+
+-- !mv1_recreate --
+21 doris test1
+
+-- !mv2_recreate --
+doris 1
+
diff --git a/regression-test/suites/mtmv_p0/test_recreate_table_mtmv.groovy
b/regression-test/suites/mtmv_p0/test_recreate_table_mtmv.groovy
new file mode 100644
index 00000000000..e9f7f339a0c
--- /dev/null
+++ b/regression-test/suites/mtmv_p0/test_recreate_table_mtmv.groovy
@@ -0,0 +1,91 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.junit.Assert;
+
+suite("test_recreate_table_mtmv","mtmv") {
+ String suiteName = "test_recreate_table_mtmv"
+ String tableName1 = "${suiteName}_table1"
+ String tableName2 = "${suiteName}_table2"
+ String mvName1 = "${suiteName}_mv1"
+ String mvName2 = "${suiteName}_mv2"
+ sql """drop table if exists `${tableName1}`"""
+ sql """drop table if exists `${tableName2}`"""
+ sql """drop materialized view if exists ${mvName1};"""
+ sql """drop materialized view if exists ${mvName2};"""
+
+ sql """
+ CREATE TABLE `${tableName1}` (
+ `id` int NULL,
+ `first_name` varchar(255) NULL,
+ `last_name` varchar(255) NULL
+ ) ENGINE=OLAP
+ AGGREGATE KEY(`id`, `first_name`, `last_name`)
+ COMMENT 'OLAP'
+ DISTRIBUTED BY HASH(`id`) BUCKETS 2
+ PROPERTIES ('replication_num' = '1') ;
+ """
+ sql """
+ INSERT INTO `${tableName1}` (`id`, `first_name`, `last_name`) VALUES
(21, 'mysql', 'test1');
+ """
+ sql """
+ CREATE TABLE `${tableName2}` (
+ `id` int NULL,
+ `first_name` varchar(255) NULL,
+ `last_name` varchar(255) NULL
+ ) ENGINE=OLAP
+ AGGREGATE KEY(`id`, `first_name`, `last_name`)
+ COMMENT 'OLAP'
+ DISTRIBUTED BY HASH(`id`) BUCKETS 2
+ PROPERTIES ('replication_num' = '1') ;
+ """
+ sql """
+ INSERT INTO `${tableName2}` (`id`, `first_name`, `last_name`) VALUES
(21, 'doris', 'test1');
+ """
+ sql """
+ CREATE MATERIALIZED VIEW IF NOT EXISTS ${mvName1} BUILD IMMEDIATE
REFRESH AUTO ON COMMIT DISTRIBUTED BY RANDOM BUCKETS 2
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1"
+ ) AS (SELECT * FROM ${tableName1} );
+ """
+ waitingMTMVTaskFinishedByMvName(mvName1);
+ order_qt_mv1 "SELECT * FROM ${mvName1}"
+ sql """
+ CREATE MATERIALIZED VIEW IF NOT EXISTS ${mvName2} BUILD IMMEDIATE
REFRESH AUTO ON COMMIT DISTRIBUTED BY RANDOM BUCKETS 2
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1"
+ ) AS (SELECT first_name,count(last_name) FROM ${mvName1} GROUP BY
first_name);
+ """
+ waitingMTMVTaskFinishedByMvName(mvName2);
+ order_qt_mv2 "SELECT * FROM ${mvName2}"
+ sql """drop materialized view if exists ${mvName1};"""
+ sql """
+ CREATE MATERIALIZED VIEW IF NOT EXISTS ${mvName1} BUILD IMMEDIATE
REFRESH AUTO ON COMMIT DISTRIBUTED BY RANDOM BUCKETS 2
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1"
+ ) AS (SELECT * FROM ${tableName2} );
+ """
+ waitingMTMVTaskFinishedByMvName(mvName1);
+ order_qt_mv1_recreate "SELECT * FROM ${mvName1}"
+ waitingMTMVTaskFinishedByMvName(mvName2);
+ order_qt_mv2_recreate "SELECT * FROM ${mvName2}"
+
+ sql """drop table if exists `${tableName1}`"""
+ sql """drop table if exists `${tableName2}`"""
+ sql """drop materialized view if exists ${mvName1};"""
+ sql """drop materialized view if exists ${mvName2};"""
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
