This is an automated email from the ASF dual-hosted git repository.
gavinchou pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 95e086fde30 [opt](systable) Improve information_schema.partitions
getVisibleVersion latency under cloud mode (#58515)
95e086fde30 is described below
commit 95e086fde3096afb557a2a6783d25cec411d62f6
Author: koarz <[email protected]>
AuthorDate: Fri Jan 30 02:05:57 2026 +0800
[opt](systable) Improve information_schema.partitions getVisibleVersion
latency under cloud mode (#58515)
Added a new `SessionVariable`:
`cloud_partitions_table_use_cached_visible_version`. By executing `set
cloud_partitions_table_use_cached_visible_version = true`, you can
control whether to use the cached visible version when reading the
`partitions` system table.
The `cloud_partitions_table_use_cached_visible_version` default value is
true
Accessing the SessionVariable requires the ConnectContext. However,
ConnectContext is thread-local. Since the RPC request from the BE
creates a new thread (which does not contain the ConnectContext), we
need to retrieve the correct ConnectContext using the original
thread_id.
---------
Co-authored-by: Gavin Chou <[email protected]>
---
.../exec/schema_scanner/schema_partitions_scanner.cpp | 3 +++
.../java/org/apache/doris/qe/SessionVariable.java | 12 ++++++++++++
.../apache/doris/tablefunction/MetadataGenerator.java | 19 +++++++++++++++----
gensrc/thrift/FrontendService.thrift | 1 +
4 files changed, 31 insertions(+), 4 deletions(-)
diff --git a/be/src/exec/schema_scanner/schema_partitions_scanner.cpp
b/be/src/exec/schema_scanner/schema_partitions_scanner.cpp
index 5d091c15769..b2803ac8c98 100644
--- a/be/src/exec/schema_scanner/schema_partitions_scanner.cpp
+++ b/be/src/exec/schema_scanner/schema_partitions_scanner.cpp
@@ -126,6 +126,9 @@ Status
SchemaPartitionsScanner::get_onedb_info_from_fe(int64_t dbId) {
schema_table_request_params.__set_current_user_ident(*_param->common_param->current_user_ident);
schema_table_request_params.__set_catalog(*_param->common_param->catalog);
schema_table_request_params.__set_dbId(dbId);
+ if (_param->common_param->thread_id > 0) {
+
schema_table_request_params.__set_thread_id(_param->common_param->thread_id);
+ }
schema_table_request_params.__set_time_zone(_timezone);
TFetchSchemaTableDataRequest request;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index 8695ee2bd44..8920ddcc332 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -752,6 +752,9 @@ public class SessionVariable implements Serializable,
Writable {
public static final String ENABLE_EXTENDED_REGEX = "enable_extended_regex";
+ public static final String
CLOUD_PARTITIONS_TABLE_USE_CACHED_VISIBLE_VERSION =
+ "cloud_partitions_table_use_cached_visible_version";
+
// NOTE: if you want to add some debug variables, please disable sql cache
in `CacheAnalyzer.commonCacheCondition`,
// and set affectQueryResult=true
public static final List<String> DEBUG_VARIABLES = ImmutableList.of(
@@ -3294,6 +3297,11 @@ public class SessionVariable implements Serializable,
Writable {
)
public int defaultVariantSparseHashShardCount = 0;
+ @VariableMgr.VarAttr(name =
CLOUD_PARTITIONS_TABLE_USE_CACHED_VISIBLE_VERSION, needForward = false,
+ description = {"partitions系统表的visible_version列在cloud模式是否使用cached",
+ "Whether cache is used for the visible_version column"
+ + "in the partitions system table on cloud mode"})
+ public boolean cloudPartitionsTableUseCachedVisibleVersion = true;
@VariableMgr.VarAttr(
name = DEFAULT_VARIANT_ENABLE_DOC_MODE,
@@ -5970,6 +5978,10 @@ public class SessionVariable implements Serializable,
Writable {
return defaultVariantSparseHashShardCount;
}
+ public boolean getCloudPartitionsTableUseCachedVisibleVersion() {
+ return cloudPartitionsTableUseCachedVisibleVersion;
+ }
+
public boolean getDefaultVariantEnableDocMode() {
return defaultVariantEnableDocMode;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java
b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java
index 4a4026ee313..34d1afd60bd 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java
@@ -83,6 +83,7 @@ import org.apache.doris.qe.QeProcessorImpl;
import org.apache.doris.qe.QeProcessorImpl.QueryInfo;
import org.apache.doris.qe.VariableMgr;
import org.apache.doris.resource.workloadgroup.WorkloadGroupMgr;
+import org.apache.doris.service.ExecuteEnv;
import org.apache.doris.system.Backend;
import org.apache.doris.system.SystemInfoService;
import org.apache.doris.thrift.FrontendService;
@@ -1595,8 +1596,8 @@ public class MetadataGenerator {
return result;
}
- private static void partitionsForInternalCatalog(UserIdentity
currentUserIdentity,
- CatalogIf catalog, DatabaseIf database, List<TableIf> tables,
List<TRow> dataBatch, String timeZone) {
+ private static void partitionsForInternalCatalog(UserIdentity
currentUserIdentity, CatalogIf catalog,
+ DatabaseIf database, List<TableIf> tables, List<TRow> dataBatch,
String timeZone, Long threadId) {
for (TableIf table : tables) {
if (!(table instanceof OlapTable)) {
continue;
@@ -1683,7 +1684,16 @@ public class MetadataGenerator {
trow.addToColumnValue(new
TCell().setIntVal(partition.getDistributionInfo()
.getBucketNum())); // BUCKET_NUM
trow.addToColumnValue(new
TCell().setLongVal(partition.getCommittedVersion())); // COMMITTED_VERSION
- trow.addToColumnValue(new
TCell().setLongVal(partition.getVisibleVersion())); // VISIBLE_VERSION
+ ConnectContext ctx =
+
ExecuteEnv.getInstance().getScheduler().getContext(threadId.intValue());
+ boolean useCachedVisibleVersion = ctx != null
+ &&
ctx.getSessionVariable().getCloudPartitionsTableUseCachedVisibleVersion();
+ if (useCachedVisibleVersion) {
+ trow.addToColumnValue(
+ new
TCell().setLongVal(partition.getCachedVisibleVersion())); // VISIBLE_VERSION
+ } else {
+ trow.addToColumnValue(new
TCell().setLongVal(partition.getVisibleVersion())); // VISIBLE_VERSION
+ }
if (partitionInfo.getType() == PartitionType.RANGE
|| partitionInfo.getType() == PartitionType.LIST) {
List<Column> partitionColumns =
partitionInfo.getPartitionColumns();
@@ -1756,6 +1766,7 @@ public class MetadataGenerator {
TFetchSchemaTableDataResult result = new TFetchSchemaTableDataResult();
Long dbId = params.getDbId();
String clg = params.getCatalog();
+ Long threadId = params.getThreadId();
List<TRow> dataBatch = Lists.newArrayList();
CatalogIf catalog =
Env.getCurrentEnv().getCatalogMgr().getCatalog(clg);
if (catalog == null) {
@@ -1778,7 +1789,7 @@ public class MetadataGenerator {
List<TableIf> tables = database.getTables();
if (catalog instanceof InternalCatalog) {
// only olap tables
- partitionsForInternalCatalog(currentUserIdentity, catalog,
database, tables, dataBatch, timezone);
+ partitionsForInternalCatalog(currentUserIdentity, catalog,
database, tables, dataBatch, timezone, threadId);
} else if (catalog instanceof ExternalCatalog) {
partitionsForExternalCatalog(currentUserIdentity, catalog,
database, tables, dataBatch, timezone);
}
diff --git a/gensrc/thrift/FrontendService.thrift
b/gensrc/thrift/FrontendService.thrift
index 1232cc0f9a1..b6b9b4555c5 100644
--- a/gensrc/thrift/FrontendService.thrift
+++ b/gensrc/thrift/FrontendService.thrift
@@ -860,6 +860,7 @@ struct TSchemaTableRequestParams {
5: optional i64 dbId // used for table specific queries
6: optional string time_zone // used for DATETIME field
7: optional string frontend_conjuncts
+ 8: optional i64 thread_id // mysql connection id for fetching
ConnectContext if needed
}
struct TFetchSchemaTableDataRequest {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
