This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 9b4354fcb7b [fix](mtmv) second level MTMV always refresh all partition
by mistake… (#39167)
9b4354fcb7b is described below
commit 9b4354fcb7b0d0aeabb5a447fcba9c56fc68e3d1
Author: zhangdong <[email protected]>
AuthorDate: Sat Aug 10 18:26:56 2024 +0800
[fix](mtmv) second level MTMV always refresh all partition by mistake…
(#39167)
pick:
https://github.com/apache/doris/pull/38698
---
.../apache/doris/job/extensions/mtmv/MTMVTask.java | 10 ++++---
.../org/apache/doris/mtmv/MTMVPartitionUtil.java | 5 ++--
.../org/apache/doris/mtmv/MTMVRewriteUtil.java | 2 +-
.../main/java/org/apache/doris/mtmv/MTMVUtil.java | 2 +-
.../apache/doris/mtmv/MTMVPartitionUtilTest.java | 2 +-
.../data/mtmv_p0/test_multi_level_mtmv.out | 6 +++++
.../suites/mtmv_p0/test_multi_level_mtmv.groovy | 31 +++++++++++++++++-----
7 files changed, 43 insertions(+), 15 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java
b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java
index b50d755951f..0207301c2d2 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java
@@ -196,7 +196,7 @@ public class MTMVTask extends AbstractTask {
.subList(start, end > needRefreshPartitions.size() ?
needRefreshPartitions.size() : end));
// need get names before exec
Map<String, MTMVRefreshPartitionSnapshot>
execPartitionSnapshots = MTMVPartitionUtil
- .generatePartitionSnapshots(mtmv,
relation.getBaseTables(), execPartitionNames,
+ .generatePartitionSnapshots(mtmv,
relation.getBaseTablesOneLevel(), execPartitionNames,
partitionMappings);
exec(ctx, execPartitionNames, tableWithPartKey);
completedPartitions.addAll(execPartitionNames);
@@ -288,7 +288,7 @@ public class MTMVTask extends AbstractTask {
* @throws DdlException
*/
private void refreshHmsTable() throws AnalysisException, DdlException {
- for (BaseTableInfo tableInfo : relation.getBaseTables()) {
+ for (BaseTableInfo tableInfo : relation.getBaseTablesOneLevel()) {
TableIf tableIf = MTMVUtil.getTable(tableInfo);
if (tableIf instanceof HMSExternalTable) {
HMSExternalTable hmsTable = (HMSExternalTable) tableIf;
@@ -450,7 +450,8 @@ public class MTMVTask extends AbstractTask {
// check if data is fresh
// We need to use a newly generated relationship and cannot retrieve
it using mtmv.getRelation()
// to avoid rebuilding the baseTable and causing a change in the
tableId
- boolean fresh = MTMVPartitionUtil.isMTMVSync(mtmv,
relation.getBaseTables(), mtmv.getExcludedTriggerTables(),
+ boolean fresh = MTMVPartitionUtil.isMTMVSync(mtmv,
relation.getBaseTablesOneLevel(),
+ mtmv.getExcludedTriggerTables(),
partitionMappings);
if (fresh) {
return Lists.newArrayList();
@@ -461,7 +462,8 @@ public class MTMVTask extends AbstractTask {
}
// We need to use a newly generated relationship and cannot retrieve
it using mtmv.getRelation()
// to avoid rebuilding the baseTable and causing a change in the
tableId
- return MTMVPartitionUtil.getMTMVNeedRefreshPartitions(mtmv,
relation.getBaseTables(), partitionMappings);
+ return MTMVPartitionUtil.getMTMVNeedRefreshPartitions(mtmv,
relation.getBaseTablesOneLevel(),
+ partitionMappings);
}
public MTMVTaskContext getTaskContext() {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java
b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java
index 6595afb70f7..9bf0d82146d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java
@@ -201,7 +201,8 @@ public class MTMVPartitionUtil {
return false;
}
try {
- return isMTMVSync(mtmv, mtmvRelation.getBaseTables(),
Sets.newHashSet(), mtmv.calculatePartitionMappings());
+ return isMTMVSync(mtmv, mtmvRelation.getBaseTablesOneLevel(),
Sets.newHashSet(),
+ mtmv.calculatePartitionMappings());
} catch (AnalysisException e) {
LOG.warn("isMTMVSync failed: ", e);
return false;
@@ -254,7 +255,7 @@ public class MTMVPartitionUtil {
Set<String> relatedPartitionNames)
throws AnalysisException {
List<String> res = Lists.newArrayList();
- for (BaseTableInfo baseTableInfo : mtmv.getRelation().getBaseTables())
{
+ for (BaseTableInfo baseTableInfo :
mtmv.getRelation().getBaseTablesOneLevel()) {
TableIf table = MTMVUtil.getTable(baseTableInfo);
if (!(table instanceof MTMVRelatedTableIf)) {
continue;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java
b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java
index 3516e75427e..5392313ba62 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java
@@ -71,7 +71,7 @@ public class MTMVRewriteUtil {
partitionMappings = mtmv.calculatePartitionMappings();
}
if (MTMVPartitionUtil.isMTMVPartitionSync(mtmv,
partition.getName(),
- partitionMappings.get(partition.getName()),
mtmvRelation.getBaseTables(),
+ partitionMappings.get(partition.getName()),
mtmvRelation.getBaseTablesOneLevel(),
Sets.newHashSet())) {
res.add(partition);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVUtil.java
b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVUtil.java
index 1fd56d7b6e4..4868ef94a1b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVUtil.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVUtil.java
@@ -85,7 +85,7 @@ public class MTMVUtil {
* @return
*/
public static boolean mtmvContainsExternalTable(MTMV mtmv) {
- Set<BaseTableInfo> baseTables = mtmv.getRelation().getBaseTables();
+ Set<BaseTableInfo> baseTables =
mtmv.getRelation().getBaseTablesOneLevel();
for (BaseTableInfo baseTableInfo : baseTables) {
if (baseTableInfo.getCtlId() !=
InternalCatalog.INTERNAL_CATALOG_ID) {
return true;
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVPartitionUtilTest.java
b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVPartitionUtilTest.java
index c125e548d06..d40e6db5d44 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVPartitionUtilTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVPartitionUtilTest.java
@@ -111,7 +111,7 @@ public class MTMVPartitionUtilTest {
minTimes = 0;
result = true;
- relation.getBaseTables();
+ relation.getBaseTablesOneLevel();
minTimes = 0;
result = baseTables;
diff --git a/regression-test/data/mtmv_p0/test_multi_level_mtmv.out
b/regression-test/data/mtmv_p0/test_multi_level_mtmv.out
index 3d8a1e1b43f..7543b21ffa7 100644
--- a/regression-test/data/mtmv_p0/test_multi_level_mtmv.out
+++ b/regression-test/data/mtmv_p0/test_multi_level_mtmv.out
@@ -5,6 +5,12 @@
-- !mv2 --
1 1
+-- !mv1_should_one_partition --
+["p_2"]
+
+-- !mv2_should_one_partition --
+["p_2"]
+
-- !status1 --
multi_level_mtmv1 SCHEMA_CHANGE SUCCESS
diff --git a/regression-test/suites/mtmv_p0/test_multi_level_mtmv.groovy
b/regression-test/suites/mtmv_p0/test_multi_level_mtmv.groovy
index 81d4b78a3d7..55689b74148 100644
--- a/regression-test/suites/mtmv_p0/test_multi_level_mtmv.groovy
+++ b/regression-test/suites/mtmv_p0/test_multi_level_mtmv.groovy
@@ -29,6 +29,11 @@ suite("test_multi_level_mtmv") {
k1 int,
k2 int
)
+ PARTITION BY LIST(`k1`)
+ (
+ PARTITION `p1` VALUES IN ('1'),
+ PARTITION `p2` VALUES IN ('2')
+ )
DISTRIBUTED BY HASH(k1) BUCKETS 10
PROPERTIES (
"replication_num" = "1"
@@ -40,34 +45,48 @@ suite("test_multi_level_mtmv") {
sql """
CREATE MATERIALIZED VIEW ${mv1}
- BUILD DEFERRED REFRESH COMPLETE ON MANUAL
+ BUILD DEFERRED REFRESH AUTO ON MANUAL
+ partition by(k1)
DISTRIBUTED BY RANDOM BUCKETS 2
PROPERTIES ('replication_num' = '1')
AS
SELECT * FROM ${tableName};
"""
- def jobName1 = getJobName("regression_test_mtmv_p0", mv1);
sql """
REFRESH MATERIALIZED VIEW ${mv1} AUTO
"""
- waitingMTMVTaskFinished(jobName1)
+ waitingMTMVTaskFinishedByMvName(mv1)
order_qt_mv1 "select * from ${mv1}"
sql """
CREATE MATERIALIZED VIEW ${mv2}
- BUILD DEFERRED REFRESH COMPLETE ON MANUAL
+ BUILD DEFERRED REFRESH AUTO ON MANUAL
+ partition by(k1)
DISTRIBUTED BY RANDOM BUCKETS 2
PROPERTIES ('replication_num' = '1')
AS
SELECT * FROM ${mv1};
"""
- def jobName2 = getJobName("regression_test_mtmv_p0", mv2);
sql """
REFRESH MATERIALIZED VIEW ${mv2} AUTO
"""
- waitingMTMVTaskFinished(jobName2)
+ waitingMTMVTaskFinishedByMvName(mv2)
order_qt_mv2 "select * from ${mv2}"
+ sql """
+ INSERT INTO ${tableName} VALUES(2,2);
+ """
+ sql """
+ REFRESH MATERIALIZED VIEW ${mv1} AUTO
+ """
+ waitingMTMVTaskFinishedByMvName(mv1)
+ order_qt_mv1_should_one_partition "select NeedRefreshPartitions from
tasks('type'='mv') where MvName = '${mv1}' order by CreateTime desc limit 1"
+ sql """
+ REFRESH MATERIALIZED VIEW ${mv2} AUTO
+ """
+ waitingMTMVTaskFinishedByMvName(mv2)
+ order_qt_mv2_should_one_partition "select NeedRefreshPartitions from
tasks('type'='mv') where MvName = '${mv2}' order by CreateTime desc limit 1"
+
// drop table
sql """
drop table ${tableName}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
