This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 0b6350e802a branch-3.0: [Enhancement](schema_table)Improve the field
information of information_schema partitions (#51520)
0b6350e802a is described below
commit 0b6350e802ae9d09bca844168e598d1ad9bfeb3a
Author: koarz <[email protected]>
AuthorDate: Wed Jun 11 11:02:35 2025 +0800
branch-3.0: [Enhancement](schema_table)Improve the field information of
information_schema partitions (#51520)
### What problem does this PR solve?
pick https://github.com/apache/doris/pull/51025 to branch-3.0
---
be/src/exec/schema_scanner/schema_helper.cpp | 9 +++
be/src/exec/schema_scanner/schema_helper.h | 6 ++
.../schema_scanner/schema_partitions_scanner.cpp | 32 +++++++--
.../schema_scanner/schema_partitions_scanner.h | 1 +
.../java/org/apache/doris/catalog/SchemaTable.java | 23 ++++++-
.../doris/tablefunction/MetadataGenerator.java | 74 +++++++++++++++++++--
.../data/query_p0/system/test_query_sys_tables.out | Bin 5606 -> 6150 bytes
7 files changed, 130 insertions(+), 15 deletions(-)
diff --git a/be/src/exec/schema_scanner/schema_helper.cpp
b/be/src/exec/schema_scanner/schema_helper.cpp
index 7b76c3ffde4..2c985d11306 100644
--- a/be/src/exec/schema_scanner/schema_helper.cpp
+++ b/be/src/exec/schema_scanner/schema_helper.cpp
@@ -153,4 +153,13 @@ Status SchemaHelper::fetch_routine_load_job(const
std::string& ip, const int32_t
});
}
+Status SchemaHelper::fetch_schema_table_data(const std::string& ip, const
int32_t port,
+ const
TFetchSchemaTableDataRequest& request,
+ TFetchSchemaTableDataResult*
result) {
+ return ThriftRpcHelper::rpc<FrontendServiceClient>(
+ ip, port, [&request, &result](FrontendServiceConnection& client) {
+ client->fetchSchemaTableData(*result, request);
+ });
+}
+
} // namespace doris
diff --git a/be/src/exec/schema_scanner/schema_helper.h
b/be/src/exec/schema_scanner/schema_helper.h
index fc5e42af803..729166724f7 100644
--- a/be/src/exec/schema_scanner/schema_helper.h
+++ b/be/src/exec/schema_scanner/schema_helper.h
@@ -30,6 +30,8 @@ class TDescribeTablesParams;
class TDescribeTablesResult;
class TFetchRoutineLoadJobRequest;
class TFetchRoutineLoadJobResult;
+class TFetchSchemaTableDataRequest;
+class TFetchSchemaTableDataResult;
class TGetDbsParams;
class TGetDbsResult;
class TGetTablesParams;
@@ -92,6 +94,10 @@ public:
static Status fetch_routine_load_job(const std::string& ip, const int32_t
port,
const TFetchRoutineLoadJobRequest&
request,
TFetchRoutineLoadJobResult* result);
+
+ static Status fetch_schema_table_data(const std::string& ip, const int32_t
port,
+ const TFetchSchemaTableDataRequest&
request,
+ TFetchSchemaTableDataResult* result);
};
} // namespace doris
diff --git a/be/src/exec/schema_scanner/schema_partitions_scanner.cpp
b/be/src/exec/schema_scanner/schema_partitions_scanner.cpp
index 459715fd628..f2ba4abaddc 100644
--- a/be/src/exec/schema_scanner/schema_partitions_scanner.cpp
+++ b/be/src/exec/schema_scanner/schema_partitions_scanner.cpp
@@ -38,6 +38,7 @@ class Block;
std::vector<SchemaScanner::ColumnDesc>
SchemaPartitionsScanner::_s_tbls_columns = {
// name, type, size, is_null
+ {"PARTITION_ID", TYPE_BIGINT, sizeof(int64_t), true},
{"TABLE_CATALOG", TYPE_VARCHAR, sizeof(StringRef), true},
{"TABLE_SCHEMA", TYPE_VARCHAR, sizeof(StringRef), true},
{"TABLE_NAME", TYPE_VARCHAR, sizeof(StringRef), false},
@@ -63,6 +64,21 @@ std::vector<SchemaScanner::ColumnDesc>
SchemaPartitionsScanner::_s_tbls_columns
{"PARTITION_COMMENT", TYPE_STRING, sizeof(StringRef), false},
{"NODEGROUP", TYPE_VARCHAR, sizeof(StringRef), true},
{"TABLESPACE_NAME", TYPE_VARCHAR, sizeof(StringRef), true},
+ {"LOCAL_DATA_SIZE", TYPE_STRING, sizeof(StringRef), true},
+ {"REMOTE_DATA_SIZE", TYPE_STRING, sizeof(StringRef), true},
+ {"STATE", TYPE_STRING, sizeof(StringRef), true},
+ {"REPLICA_ALLOCATION", TYPE_STRING, sizeof(StringRef), true},
+ {"REPLICA_NUM", TYPE_INT, sizeof(int32_t), true},
+ {"STORAGE_POLICY", TYPE_STRING, sizeof(StringRef), true},
+ {"STORAGE_MEDIUM", TYPE_STRING, sizeof(StringRef), true},
+ {"COOLDOWN_TIME_MS", TYPE_STRING, sizeof(StringRef), true},
+ {"LAST_CONSISTENCY_CHECK_TIME", TYPE_STRING, sizeof(StringRef), true},
+ {"BUCKET_NUM", TYPE_INT, sizeof(int32_t), true},
+ {"COMMITTED_VERSION", TYPE_BIGINT, sizeof(int64_t), true},
+ {"VISIBLE_VERSION", TYPE_BIGINT, sizeof(int64_t), true},
+ {"PARTITION_KEY", TYPE_STRING, sizeof(StringRef), true},
+ {"RANGE", TYPE_STRING, sizeof(StringRef), true},
+ {"DISTRIBUTION", TYPE_STRING, sizeof(StringRef), true},
};
SchemaPartitionsScanner::SchemaPartitionsScanner()
@@ -115,13 +131,13 @@ Status
SchemaPartitionsScanner::get_onedb_info_from_fe(int64_t dbId) {
TFetchSchemaTableDataResult result;
- RETURN_IF_ERROR(ThriftRpcHelper::rpc<FrontendServiceClient>(
- master_addr.hostname, master_addr.port,
- [&request, &result](FrontendServiceConnection& client) {
- client->fetchSchemaTableData(result, request);
- },
- _rpc_timeout_ms));
+
RETURN_IF_ERROR(SchemaHelper::fetch_schema_table_data(master_addr.hostname,
master_addr.port,
+ request, &result));
+ RETURN_IF_ERROR(fill_db_partitions(result));
+ return Status::OK();
+}
+Status
SchemaPartitionsScanner::fill_db_partitions(TFetchSchemaTableDataResult&
result) {
Status status(Status::create(result.status));
if (!status.ok()) {
LOG(WARNING) << "fetch table options from FE failed, errmsg=" <<
status;
@@ -137,7 +153,7 @@ Status
SchemaPartitionsScanner::get_onedb_info_from_fe(int64_t dbId) {
data_type->create_column(), data_type,
_s_tbls_columns[i].name));
}
_partitions_block->reserve(_block_rows_limit);
- if (result_data.size() > 0) {
+ if (!result_data.empty()) {
int col_size = result_data[0].column_value.size();
if (col_size != _s_tbls_columns.size()) {
return Status::InternalError<false>("table options schema is not
match for FE and BE");
@@ -174,6 +190,8 @@ Status
SchemaPartitionsScanner::get_next_block_internal(vectorized::Block* block
return Status::InternalError("input pointer is nullptr.");
}
+ SCOPED_TIMER(_fill_block_timer);
+
if ((_partitions_block == nullptr) || (_row_idx == _total_rows)) {
if (_db_index < _db_result.db_ids.size()) {
RETURN_IF_ERROR(get_onedb_info_from_fe(_db_result.db_ids[_db_index]));
diff --git a/be/src/exec/schema_scanner/schema_partitions_scanner.h
b/be/src/exec/schema_scanner/schema_partitions_scanner.h
index 3c246f36eec..d068b366681 100644
--- a/be/src/exec/schema_scanner/schema_partitions_scanner.h
+++ b/be/src/exec/schema_scanner/schema_partitions_scanner.h
@@ -44,6 +44,7 @@ public:
private:
Status get_onedb_info_from_fe(int64_t dbId);
+ Status fill_db_partitions(TFetchSchemaTableDataResult& result);
bool check_and_mark_eos(bool* eos) const;
int _block_rows_limit = 4096;
int _db_index = 0;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java
index 2158bcacc9e..31a80d3daf6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java
@@ -297,7 +297,8 @@ public class SchemaTable extends Table {
.column("EXTRA",
ScalarType.createVarchar(256)).build()))
.put("partitions",
new SchemaTable(SystemIdGenerator.getNextId(),
"partitions", TableType.SCHEMA,
- builder().column("TABLE_CATALOG",
ScalarType.createVarchar(64))
+ builder().column("PARTITION_ID",
ScalarType.createType(PrimitiveType.BIGINT))
+ .column("TABLE_CATALOG",
ScalarType.createVarchar(64))
.column("TABLE_SCHEMA",
ScalarType.createVarchar(64))
.column("TABLE_NAME",
ScalarType.createVarchar(64))
.column("PARTITION_NAME",
ScalarType.createVarchar(64))
@@ -322,7 +323,23 @@ public class SchemaTable extends Table {
.column("CHECKSUM",
ScalarType.createType(PrimitiveType.BIGINT))
.column("PARTITION_COMMENT",
ScalarType.createStringType())
.column("NODEGROUP",
ScalarType.createVarchar(256))
- .column("TABLESPACE_NAME",
ScalarType.createVarchar(268)).build()))
+ .column("TABLESPACE_NAME",
ScalarType.createVarchar(268))
+ .column("LOCAL_DATA_SIZE",
ScalarType.createStringType())
+ .column("REMOTE_DATA_SIZE",
ScalarType.createStringType())
+ .column("STATE",
ScalarType.createStringType())
+ .column("REPLICA_ALLOCATION",
ScalarType.createStringType())
+ .column("REPLICA_NUM",
ScalarType.createType(PrimitiveType.INT))
+ .column("STORAGE_POLICY",
ScalarType.createStringType())
+ .column("STORAGE_MEDIUM",
ScalarType.createStringType())
+ .column("COOLDOWN_TIME_MS",
ScalarType.createStringType())
+ .column("LAST_CONSISTENCY_CHECK_TIME",
ScalarType.createStringType())
+ .column("BUCKET_NUM",
ScalarType.createType(PrimitiveType.INT))
+ .column("COMMITTED_VERSION",
ScalarType.createType(PrimitiveType.BIGINT))
+ .column("VISIBLE_VERSION",
ScalarType.createType(PrimitiveType.BIGINT))
+ .column("PARTITION_KEY",
ScalarType.createStringType())
+ .column("RANGE",
ScalarType.createStringType())
+ .column("DISTRIBUTION",
ScalarType.createStringType())
+ .build()))
// Compatible with Datagrip
.put("column_privileges",
new SchemaTable(SystemIdGenerator.getNextId(),
"column_privileges", TableType.SCHEMA,
@@ -419,7 +436,7 @@ public class SchemaTable extends Table {
.column("DATA_TYPEDTD_IDENDS",
ScalarType.createVarchar(64))
.build()))
.put("metadata_name_ids", new
SchemaTable(SystemIdGenerator.getNextId(),
- "metadata_name_ids", TableType.SCHEMA,
+ "metadata_name_ids", TableType.SCHEMA,
builder().column("CATALOG_ID",
ScalarType.createType(PrimitiveType.BIGINT))
.column("CATALOG_NAME",
ScalarType.createVarchar(FN_REFLEN))
.column("DATABASE_ID",
ScalarType.createType(PrimitiveType.BIGINT))
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java
b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java
index e6d479105fa..1420e03f375 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java
@@ -19,6 +19,7 @@ package org.apache.doris.tablefunction;
import org.apache.doris.analysis.UserIdentity;
import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.DataProperty;
import org.apache.doris.catalog.Database;
import org.apache.doris.catalog.DatabaseIf;
import org.apache.doris.catalog.DistributionInfo;
@@ -28,6 +29,7 @@ import org.apache.doris.catalog.HashDistributionInfo;
import org.apache.doris.catalog.MTMV;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Partition;
+import org.apache.doris.catalog.PartitionInfo;
import org.apache.doris.catalog.PartitionItem;
import org.apache.doris.catalog.PartitionType;
import org.apache.doris.catalog.ScalarType;
@@ -43,6 +45,7 @@ import org.apache.doris.common.Pair;
import org.apache.doris.common.UserException;
import org.apache.doris.common.proc.FrontendsProcNode;
import org.apache.doris.common.proc.PartitionsProcDir;
+import org.apache.doris.common.util.DebugUtil;
import org.apache.doris.common.util.NetUtils;
import org.apache.doris.common.util.TimeUtils;
import org.apache.doris.common.util.Util;
@@ -96,6 +99,7 @@ import org.apache.doris.thrift.TStatusCode;
import org.apache.doris.thrift.TTasksMetadataParams;
import org.apache.doris.thrift.TUserIdentity;
+import com.google.common.base.Joiner;
import com.google.common.base.Stopwatch;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableMap;
@@ -1363,9 +1367,12 @@ public class MetadataGenerator {
olapTable.readLock();
try {
Collection<Partition> allPartitions =
olapTable.getAllPartitions();
-
+ PartitionInfo partitionInfo = olapTable.getPartitionInfo();
+ Joiner joiner = Joiner.on(", ");
for (Partition partition : allPartitions) {
TRow trow = new TRow();
+ long partitionId = partition.getId();
+ trow.addToColumnValue(new
TCell().setLongVal(partitionId)); // PARTITION_ID
trow.addToColumnValue(new
TCell().setStringVal(catalog.getName())); // TABLE_CATALOG
trow.addToColumnValue(new
TCell().setStringVal(database.getFullName())); // TABLE_SCHEMA
trow.addToColumnValue(new
TCell().setStringVal(table.getName())); // TABLE_NAME
@@ -1375,17 +1382,17 @@ public class MetadataGenerator {
trow.addToColumnValue(new TCell().setIntVal(0));
//PARTITION_ORDINAL_POSITION (not available)
trow.addToColumnValue(new TCell().setIntVal(0));
//SUBPARTITION_ORDINAL_POSITION (not available)
trow.addToColumnValue(new TCell().setStringVal(
-
olapTable.getPartitionInfo().getType().toString())); // PARTITION_METHOD
+ partitionInfo.getType().toString())); //
PARTITION_METHOD
trow.addToColumnValue(new TCell().setStringVal("NULL"));
// SUBPARTITION_METHOD(always null)
- PartitionItem item =
olapTable.getPartitionInfo().getItem(partition.getId());
- if ((olapTable.getPartitionInfo().getType() ==
PartitionType.UNPARTITIONED) || (item == null)) {
+ PartitionItem item = partitionInfo.getItem(partitionId);
+ if ((partitionInfo.getType() ==
PartitionType.UNPARTITIONED) || (item == null)) {
trow.addToColumnValue(new
TCell().setStringVal("NULL")); // if unpartitioned, its null
trow.addToColumnValue(
new TCell().setStringVal("NULL")); //
SUBPARTITION_EXPRESSION (always null)
trow.addToColumnValue(new
TCell().setStringVal("NULL")); // PARITION DESC, its null
} else {
trow.addToColumnValue(new TCell().setStringVal(
- olapTable.getPartitionInfo()
+ partitionInfo
.getDisplayPartitionColumns().toString())); // PARTITION_EXPRESSION
trow.addToColumnValue(
new TCell().setStringVal("NULL")); //
SUBPARTITION_EXPRESSION (always null)
@@ -1406,6 +1413,63 @@ public class MetadataGenerator {
trow.addToColumnValue(new TCell().setStringVal("")); //
PARTITION_COMMENT (not available)
trow.addToColumnValue(new TCell().setStringVal("")); //
NODEGROUP (not available)
trow.addToColumnValue(new TCell().setStringVal("")); //
TABLESPACE_NAME (not available)
+
+ Pair<Double, String> sizePair =
DebugUtil.getByteUint(partition.getDataSize(false));
+ String readableDateSize =
DebugUtil.DECIMAL_FORMAT_SCALE_3.format(sizePair.first) + " "
+ + sizePair.second;
+ trow.addToColumnValue(new
TCell().setStringVal(readableDateSize)); // LOCAL_DATA_SIZE
+ sizePair =
DebugUtil.getByteUint(partition.getRemoteDataSize());
+ readableDateSize =
DebugUtil.DECIMAL_FORMAT_SCALE_3.format(sizePair.first) + " "
+ + sizePair.second;
+ trow.addToColumnValue(new
TCell().setStringVal(readableDateSize)); // REMOTE_DATA_SIZE
+ trow.addToColumnValue(new
TCell().setStringVal(partition.getState().toString())); // STATE
+ trow.addToColumnValue(new
TCell().setStringVal(partitionInfo.getReplicaAllocation(partitionId)
+ .toCreateStmt())); // REPLICA_ALLOCATION
+ trow.addToColumnValue(new
TCell().setIntVal(partitionInfo.getReplicaAllocation(partitionId)
+ .getTotalReplicaNum())); // REPLICA_NUM
+ trow.addToColumnValue(new
TCell().setStringVal(partitionInfo
+ .getStoragePolicy(partitionId))); // STORAGE_POLICY
+ DataProperty dataProperty =
partitionInfo.getDataProperty(partitionId);
+ trow.addToColumnValue(new
TCell().setStringVal(dataProperty.getStorageMedium()
+ .name())); // STORAGE_MEDIUM
+ trow.addToColumnValue(new
TCell().setStringVal(TimeUtils.longToTimeString(dataProperty
+ .getCooldownTimeMs()))); // COOLDOWN_TIME_MS
+ trow.addToColumnValue(new
TCell().setStringVal(TimeUtils.longToTimeString(partition
+ .getLastCheckTime()))); //
LAST_CONSISTENCY_CHECK_TIME
+ trow.addToColumnValue(new
TCell().setIntVal(partition.getDistributionInfo()
+ .getBucketNum())); // BUCKET_NUM
+ trow.addToColumnValue(new
TCell().setLongVal(partition.getCommittedVersion())); // COMMITTED_VERSION
+ trow.addToColumnValue(new
TCell().setLongVal(partition.getVisibleVersion())); // VISIBLE_VERSION
+ if (partitionInfo.getType() == PartitionType.RANGE
+ || partitionInfo.getType() == PartitionType.LIST) {
+ List<Column> partitionColumns =
partitionInfo.getPartitionColumns();
+ List<String> colNames = new ArrayList<>();
+ for (Column column : partitionColumns) {
+ colNames.add(column.getName());
+ }
+ String colNamesStr = joiner.join(colNames);
+ trow.addToColumnValue(new
TCell().setStringVal(colNamesStr)); // PARTITION_KEY
+ trow.addToColumnValue(new
TCell().setStringVal(partitionInfo
+ .getPartitionRangeString(partitionId))); //
RANGE
+ } else {
+ trow.addToColumnValue(new TCell().setStringVal(""));
// PARTITION_KEY
+ trow.addToColumnValue(new TCell().setStringVal(""));
// RANGE
+ }
+ DistributionInfo distributionInfo =
partition.getDistributionInfo();
+ if (distributionInfo.getType() ==
DistributionInfoType.HASH) {
+ HashDistributionInfo hashDistributionInfo =
(HashDistributionInfo) distributionInfo;
+ List<Column> distributionColumns =
hashDistributionInfo.getDistributionColumns();
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < distributionColumns.size(); i++) {
+ if (i != 0) {
+ sb.append(", ");
+ }
+ sb.append(distributionColumns.get(i).getName());
+ }
+ trow.addToColumnValue(new
TCell().setStringVal(sb.toString())); // DISTRIBUTION
+ } else {
+ trow.addToColumnValue(new
TCell().setStringVal("RANDOM")); // DISTRIBUTION
+ }
dataBatch.add(trow);
}
} finally {
diff --git a/regression-test/data/query_p0/system/test_query_sys_tables.out
b/regression-test/data/query_p0/system/test_query_sys_tables.out
index 648959e2ce4..b47d2865409 100644
Binary files a/regression-test/data/query_p0/system/test_query_sys_tables.out
and b/regression-test/data/query_p0/system/test_query_sys_tables.out differ
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
