This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.0 by this push:
new 681d0b890fc branch-4.0: [Opt](cloud) cache table version for cloud
mode #59339 (#59442)
681d0b890fc is described below
commit 681d0b890fca54d1fd97001852fe938c748bda12
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Mon Jan 12 14:20:13 2026 +0800
branch-4.0: [Opt](cloud) cache table version for cloud mode #59339 (#59442)
Cherry-picked from #59339
Co-authored-by: bobhan1 <[email protected]>
---
.../java/org/apache/doris/catalog/OlapTable.java | 39 +++++++
.../java/org/apache/doris/qe/SessionVariable.java | 4 +
.../org/apache/doris/catalog/OlapTableTest.java | 122 +++++++++++++++++++++
3 files changed, 165 insertions(+)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
index 7607d2c8be7..0ac334fd2b3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
@@ -234,6 +234,11 @@ public class OlapTable extends Table implements
MTMVRelatedTableIf, GsonPostProc
// Ensures only one creation task runs for a given partition at a time.
private ConcurrentHashMap<String, CompletableFuture<Void>>
partitionCreationFutures = new ConcurrentHashMap<>();
+ // Cache for table version in cloud mode
+ // This value is set when get the table version from meta-service, 0 means
version is not cached yet
+ private long lastTableVersionCachedTimeMs = 0;
+ private long cachedTableVersion = -1;
+
public OlapTable() {
// for persist
super(TableType.OLAP);
@@ -3280,11 +3285,43 @@ public class OlapTable extends Table implements
MTMVRelatedTableIf, GsonPostProc
}
}
+ public boolean isCachedTableVersionExpired() {
+ // -1 means no cache yet, need to fetch from MS
+ if (cachedTableVersion == -1) {
+ return true;
+ }
+ ConnectContext ctx = ConnectContext.get();
+ if (ctx == null) {
+ return true;
+ }
+ long cacheExpirationMs =
ctx.getSessionVariable().cloudTableVersionCacheTtlMs;
+ if (cacheExpirationMs <= 0) { // always expired
+ return true;
+ }
+ return System.currentTimeMillis() - lastTableVersionCachedTimeMs >
cacheExpirationMs;
+ }
+
+ public void setCachedTableVersion(long version) {
+ if (version > cachedTableVersion) {
+ cachedTableVersion = version;
+ lastTableVersionCachedTimeMs = System.currentTimeMillis();
+ }
+ }
+
+ public long getCachedTableVersion() {
+ return cachedTableVersion;
+ }
+
public long getVisibleVersion() throws RpcException {
if (Config.isNotCloudMode()) {
return tableAttributes.getVisibleVersion();
}
+ // check if cache is not expired
+ if (!isCachedTableVersionExpired()) {
+ return getCachedTableVersion();
+ }
+
// get version rpc
Cloud.GetVersionRequest request = Cloud.GetVersionRequest.newBuilder()
.setRequestIp(FrontendOptions.getLocalHostAddressCached())
@@ -3309,6 +3346,8 @@ public class OlapTable extends Table implements
MTMVRelatedTableIf, GsonPostProc
if (version == 0) {
version = 1;
}
+ // update cache
+ setCachedTableVersion(version);
return version;
} catch (RpcException e) {
LOG.warn("get version from meta service failed", e);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index ba96a4a739e..7c5e6cca68c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -767,6 +767,8 @@ public class SessionVariable implements Serializable,
Writable {
public static final String DISABLE_EMPTY_PARTITION_PRUNE =
"disable_empty_partition_prune";
public static final String CLOUD_PARTITION_VERSION_CACHE_TTL_MS =
"cloud_partition_version_cache_ttl_ms";
+ public static final String CLOUD_TABLE_VERSION_CACHE_TTL_MS =
+ "cloud_table_version_cache_ttl_ms";
// CLOUD_VARIABLES_BEGIN
public static final String ENABLE_MATCH_WITHOUT_INVERTED_INDEX =
"enable_match_without_inverted_index";
@@ -2795,6 +2797,8 @@ public class SessionVariable implements Serializable,
Writable {
public boolean disableEmptyPartitionPrune = false;
@VariableMgr.VarAttr(name = CLOUD_PARTITION_VERSION_CACHE_TTL_MS)
public static long cloudPartitionVersionCacheTtlMs = 0;
+ @VariableMgr.VarAttr(name = CLOUD_TABLE_VERSION_CACHE_TTL_MS)
+ public long cloudTableVersionCacheTtlMs = 0;
// CLOUD_VARIABLES_END
// fetch remote schema rpc timeout
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/catalog/OlapTableTest.java
b/fe/fe-core/src/test/java/org/apache/doris/catalog/OlapTableTest.java
index 0c983332ee5..befa306e186 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/catalog/OlapTableTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/OlapTableTest.java
@@ -20,10 +20,14 @@ package org.apache.doris.catalog;
import org.apache.doris.analysis.IndexDef;
import org.apache.doris.analysis.UserIdentity;
import org.apache.doris.catalog.TableIf.TableType;
+import org.apache.doris.cloud.proto.Cloud;
+import org.apache.doris.cloud.rpc.VersionHelper;
+import org.apache.doris.common.Config;
import org.apache.doris.common.FeConstants;
import org.apache.doris.common.io.FastByteArrayOutputStream;
import org.apache.doris.common.util.UnitTestUtil;
import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.SessionVariable;
import org.apache.doris.resource.Tag;
import org.apache.doris.resource.computegroup.ComputeGroup;
import org.apache.doris.system.Backend;
@@ -283,4 +287,122 @@ public class OlapTableTest {
ConnectContext.remove();
}
+
+ @Test
+ public void testTableVersionCacheWithRpc() throws Exception {
+ // Mock cloud mode
+ new MockUp<Config>() {
+ @Mock
+ public boolean isNotCloudMode() {
+ return false;
+ }
+ };
+
+ // Create table and database
+ final Database db = new Database(1L, "test_db");
+
+ // Create a custom OlapTable that overrides getDatabase()
+ OlapTable table = new OlapTable() {
+ @Override
+ public Database getDatabase() {
+ return db;
+ }
+ };
+ table.id = 1000L;
+
+ // Mock VersionHelper.getVersionFromMeta()
+ final long[] versions = {100L, 200L, 300L};
+ final int[] callCount = {0};
+
+ new MockUp<VersionHelper>() {
+ @Mock
+ public Cloud.GetVersionResponse
getVersionFromMeta(Cloud.GetVersionRequest req) {
+ Cloud.GetVersionResponse.Builder builder =
Cloud.GetVersionResponse.newBuilder();
+ builder.setStatus(Cloud.MetaServiceResponseStatus.newBuilder()
+ .setCode(Cloud.MetaServiceCode.OK).build());
+ builder.setVersion(versions[callCount[0]]);
+ callCount[0]++;
+ return builder.build();
+ }
+ };
+
+ // Create ConnectContext with SessionVariable
+ ConnectContext ctx = new ConnectContext();
+ ctx.setSessionVariable(new SessionVariable());
+ ctx.setThreadLocalInfo();
+
+ try {
+ // Test 1: Initial state with TTL set, should still call RPC for
first time
+ ctx.getSessionVariable().cloudTableVersionCacheTtlMs = 100000; //
Set long TTL
+ Assert.assertEquals(-1, table.getCachedTableVersion()); // Initial
state
+ Assert.assertTrue(table.isCachedTableVersionExpired()); // Should
be expired due to -1
+
+ long ver0 = table.getVisibleVersion();
+ Assert.assertEquals(100, ver0); // Should get from MS
+ Assert.assertEquals(1, callCount[0]); // First RPC call
+ Assert.assertEquals(100, table.getCachedTableVersion()); // Cache
updated
+
+ // Second call should use cache
+ long ver0Again = table.getVisibleVersion();
+ Assert.assertEquals(100, ver0Again); // Should use cached version
+ Assert.assertEquals(1, callCount[0]); // No new RPC call
+
+ // Test 2: Disable cache (TTL = 0), should always call RPC
+ ctx.getSessionVariable().cloudTableVersionCacheTtlMs = 0;
+ long ver1 = table.getVisibleVersion();
+ Assert.assertEquals(200, ver1);
+ Assert.assertEquals(2, callCount[0]); // Second RPC call
+
+ long ver2 = table.getVisibleVersion();
+ Assert.assertEquals(300, ver2);
+ Assert.assertEquals(3, callCount[0]); // Third RPC call
+ Assert.assertEquals(300, table.getCachedTableVersion()); // Cache
updated to 300
+
+ // Test 3: Enable cache with long TTL, should use cached version
+ ctx.getSessionVariable().cloudTableVersionCacheTtlMs = 100000; //
100 seconds
+ table.setCachedTableVersion(350); // Set cache to a larger version
+ long ver3 = table.getVisibleVersion();
+ Assert.assertEquals(350, ver3); // Should return cached version
(350)
+ Assert.assertEquals(3, callCount[0]); // No new RPC call
+
+ // Test 4: Test setCachedTableVersion only updates when version is
greater
+ ctx.getSessionVariable().cloudTableVersionCacheTtlMs = 500; //
500ms TTL
+
+ // At this point, cache is 350 from Test 3
+ // Set a larger version to 400
+ table.setCachedTableVersion(400);
+ Assert.assertEquals(400, table.getCachedTableVersion());
+ Assert.assertFalse(table.isCachedTableVersionExpired()); // Not
expired yet
+
+ Thread.sleep(300); // Sleep 300ms
+
+ // Try to set a smaller version (380), should NOT update version
or timestamp
+ table.setCachedTableVersion(380);
+ Assert.assertEquals(400, table.getCachedTableVersion()); //
Version should remain 400
+
+ Thread.sleep(300); // Total 600ms since setCachedTableVersion(400)
+ // Cache should be expired (600ms > 500ms TTL)
+ // If timestamp was incorrectly reset by
setCachedTableVersion(380), cache would not be expired
+ Assert.assertTrue(table.isCachedTableVersionExpired());
+
+ // Test 5: Setting a greater version should update both version
and timestamp
+ ctx.getSessionVariable().cloudTableVersionCacheTtlMs = 500; //
500ms TTL
+ table.setCachedTableVersion(500); // Set to 500
+ Assert.assertEquals(500, table.getCachedTableVersion());
+ Assert.assertFalse(table.isCachedTableVersionExpired()); // Not
expired
+
+ Thread.sleep(300); // Sleep 300ms
+
+ // Set a greater version (550), should update both version and
timestamp
+ table.setCachedTableVersion(550);
+ Assert.assertEquals(550, table.getCachedTableVersion()); //
Version updated to 550
+ Assert.assertFalse(table.isCachedTableVersionExpired()); //
Timestamp reset, not expired yet
+
+ Thread.sleep(300); // Sleep another 300ms (total 600ms from first
setCachedTableVersion(500), but only 300ms from setCachedTableVersion(550))
+ Assert.assertFalse(table.isCachedTableVersionExpired()); // Still
not expired (300ms < 500ms TTL)
+
+ } finally {
+ ConnectContext.remove();
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
