This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new c971c733db2 branch-3.0:[enhance](mtmv)cache table snapshot in refresh
context (#50855) (#51494)
c971c733db2 is described below
commit c971c733db2084c9a98af3d9718daf64765e0ce7
Author: zhangdong <[email protected]>
AuthorDate: Thu Jun 12 10:15:19 2025 +0800
branch-3.0:[enhance](mtmv)cache table snapshot in refresh context (#50855)
(#51494)
pick: https://github.com/apache/doris/pull/50855
---
.../org/apache/doris/mtmv/MTMVPartitionUtil.java | 27 ++++++++++++++++++++--
.../org/apache/doris/mtmv/MTMVRefreshContext.java | 10 ++++++++
.../apache/doris/mtmv/MTMVPartitionUtilTest.java | 21 +++++++++++++++++
3 files changed, 56 insertions(+), 2 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java
b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java
index b078a6d9aa9..8da00cdeeb3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java
@@ -488,11 +488,34 @@ public class MTMVPartitionUtil {
if (!baseTable.needAutoRefresh()) {
return true;
}
- MTMVSnapshotIf baseTableCurrentSnapshot =
baseTable.getTableSnapshot(context, Optional.empty());
+ MTMVSnapshotIf baseTableCurrentSnapshot =
getTableSnapshotFromContext(baseTable, context);
return mtmv.getRefreshSnapshot()
.equalsWithBaseTable(mtmvPartitionName, new
BaseTableInfo(baseTable), baseTableCurrentSnapshot);
}
+ /**
+ * Try context first, then load via getTableSnapshot and cache
+ *
+ * @param mtmvRelatedTableIf Base table of materialized views
+ * @param context The context data persists for the duration of either a
refresh task
+ * or a transparent rewrite operation
+ * @return The snapshot information of the MTMV
+ * @throws AnalysisException
+ */
+ public static MTMVSnapshotIf
getTableSnapshotFromContext(MTMVRelatedTableIf mtmvRelatedTableIf,
+ MTMVRefreshContext context)
+ throws AnalysisException {
+ BaseTableInfo baseTableInfo = new BaseTableInfo(mtmvRelatedTableIf);
+ Map<BaseTableInfo, MTMVSnapshotIf> baseTableSnapshotCache =
context.getBaseTableSnapshotCache();
+ if (baseTableSnapshotCache.containsKey(baseTableInfo)) {
+ return baseTableSnapshotCache.get(baseTableInfo);
+ }
+ MTMVSnapshotIf baseTableCurrentSnapshot =
mtmvRelatedTableIf.getTableSnapshot(context,
+ Optional.empty());
+ baseTableSnapshotCache.put(baseTableInfo, baseTableCurrentSnapshot);
+ return baseTableCurrentSnapshot;
+ }
+
/**
* Generate updated snapshots of partitions to determine if they are
synchronized
*
@@ -539,7 +562,7 @@ public class MTMVPartitionUtil {
continue;
}
refreshPartitionSnapshot.addTableSnapshot(baseTableInfo,
- ((MTMVRelatedTableIf) table).getTableSnapshot(context,
Optional.empty()));
+ getTableSnapshotFromContext((MTMVRelatedTableIf) table,
context));
}
return refreshPartitionSnapshot;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRefreshContext.java
b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRefreshContext.java
index 3d611b5e852..c59abd9ebdc 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRefreshContext.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRefreshContext.java
@@ -20,6 +20,8 @@ package org.apache.doris.mtmv;
import org.apache.doris.catalog.MTMV;
import org.apache.doris.common.AnalysisException;
+import com.google.common.collect.Maps;
+
import java.util.Map;
import java.util.Set;
@@ -27,6 +29,10 @@ public class MTMVRefreshContext {
private MTMV mtmv;
private Map<String, Set<String>> partitionMappings;
private MTMVBaseVersions baseVersions;
+ // Within the same context, repeated fetches of the same table's snapshot
must return consistent values.
+ // Hence, the results are cached at this stage.
+ // The value is loaded/cached on the first fetch
+ private Map<BaseTableInfo, MTMVSnapshotIf> baseTableSnapshotCache =
Maps.newHashMap();
public MTMVRefreshContext(MTMV mtmv) {
this.mtmv = mtmv;
@@ -44,6 +50,10 @@ public class MTMVRefreshContext {
return baseVersions;
}
+ public Map<BaseTableInfo, MTMVSnapshotIf> getBaseTableSnapshotCache() {
+ return baseTableSnapshotCache;
+ }
+
public static MTMVRefreshContext buildContext(MTMV mtmv) throws
AnalysisException {
MTMVRefreshContext context = new MTMVRefreshContext(mtmv);
context.partitionMappings = mtmv.calculatePartitionMappings();
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVPartitionUtilTest.java
b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVPartitionUtilTest.java
index c2e1777f796..d90b43482eb 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVPartitionUtilTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVPartitionUtilTest.java
@@ -38,6 +38,7 @@ import org.junit.Before;
import org.junit.Test;
import java.util.List;
+import java.util.Map;
import java.util.Optional;
import java.util.Set;
@@ -92,6 +93,10 @@ public class MTMVPartitionUtilTest {
minTimes = 0;
result = versions;
+ context.getBaseTableSnapshotCache();
+ minTimes = 0;
+ result = Maps.newHashMap();
+
mtmv.getPartitions();
minTimes = 0;
result = Lists.newArrayList(p1);
@@ -296,4 +301,20 @@ public class MTMVPartitionUtilTest {
Assert.assertFalse(MTMVPartitionUtil.isTableNamelike(new
TableName("db1"), tableNameToCheck));
Assert.assertFalse(MTMVPartitionUtil.isTableNamelike(new
TableName("ctl1"), tableNameToCheck));
}
+
+ @Test
+ public void testGetTableSnapshotFromContext() throws AnalysisException {
+ Map<BaseTableInfo, MTMVSnapshotIf> cache = Maps.newHashMap();
+ new Expectations() {
+ {
+ context.getBaseTableSnapshotCache();
+ minTimes = 0;
+ result = cache;
+ }
+ };
+ Assert.assertTrue(cache.isEmpty());
+ MTMVPartitionUtil.getTableSnapshotFromContext(baseOlapTable, context);
+ Assert.assertEquals(1, cache.size());
+ Assert.assertEquals(baseSnapshotIf, cache.values().iterator().next());
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
