This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new fa06c7517ae [feature](meta) Add thrift rpc to get db/table/backend
meta (#25943)
fa06c7517ae is described below
commit fa06c7517ae20bd28f26d915b6d0fe8edac41b02
Author: Jack Drogon <[email protected]>
AuthorDate: Fri Oct 27 10:12:58 2023 +0800
[feature](meta) Add thrift rpc to get db/table/backend meta (#25943)
Signed-off-by: Jack Drogon <[email protected]>
---
.../main/java/org/apache/doris/catalog/Env.java | 92 +++++++++++
.../apache/doris/service/FrontendServiceImpl.java | 174 +++++++++++++++++++++
gensrc/thrift/FrontendService.thrift | 111 +++++++++++++
gensrc/thrift/Types.thrift | 3 +
4 files changed, 380 insertions(+)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
index 9a6d13552d6..26b9bbd2e12 100755
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
@@ -251,7 +251,16 @@ import org.apache.doris.task.PriorityMasterTaskExecutor;
import org.apache.doris.thrift.BackendService;
import org.apache.doris.thrift.TCompressionType;
import org.apache.doris.thrift.TFrontendInfo;
+import org.apache.doris.thrift.TGetMetaDBMeta;
+import org.apache.doris.thrift.TGetMetaIndexMeta;
+import org.apache.doris.thrift.TGetMetaPartitionMeta;
+import org.apache.doris.thrift.TGetMetaReplicaMeta;
+import org.apache.doris.thrift.TGetMetaResult;
+import org.apache.doris.thrift.TGetMetaTableMeta;
+import org.apache.doris.thrift.TGetMetaTabletMeta;
import org.apache.doris.thrift.TNetworkAddress;
+import org.apache.doris.thrift.TStatus;
+import org.apache.doris.thrift.TStatusCode;
import org.apache.doris.thrift.TStorageMedium;
import org.apache.doris.transaction.DbUsedDataQuotaInfoCollector;
import org.apache.doris.transaction.GlobalTransactionMgr;
@@ -5625,6 +5634,89 @@ public class Env {
return GlobalVariable.lowerCaseTableNames == 2;
}
+ private static void getTableMeta(OlapTable olapTable, TGetMetaDBMeta
dbMeta) {
+ LOG.debug("get table meta. table: {}", olapTable.getName());
+
+ TGetMetaTableMeta tableMeta = new TGetMetaTableMeta();
+ olapTable.readLock();
+ try {
+ tableMeta.setId(olapTable.getId());
+ tableMeta.setName(olapTable.getName());
+
+ PartitionInfo tblPartitionInfo = olapTable.getPartitionInfo();
+
+ Collection<Partition> partitions = olapTable.getAllPartitions();
+ for (Partition partition : partitions) {
+ TGetMetaPartitionMeta partitionMeta = new
TGetMetaPartitionMeta();
+ long partitionId = partition.getId();
+ partitionMeta.setId(partitionId);
+ partitionMeta.setName(partition.getName());
+ String partitionRange = "";
+ if (tblPartitionInfo.getType() == PartitionType.RANGE
+ || tblPartitionInfo.getType() == PartitionType.LIST) {
+ partitionRange =
tblPartitionInfo.getItem(partitionId).getItems().toString();
+ }
+ partitionMeta.setRange(partitionRange);
+ partitionMeta.setVisibleVersion(partition.getVisibleVersion());
+ // partitionMeta.setTemp（partition.isTemp());
+
+ for (MaterializedIndex index :
partition.getMaterializedIndices(IndexExtState.ALL)) {
+ TGetMetaIndexMeta indexMeta = new TGetMetaIndexMeta();
+ indexMeta.setId(index.getId());
+
indexMeta.setName(olapTable.getIndexNameById(index.getId()));
+
+ for (Tablet tablet : index.getTablets()) {
+ TGetMetaTabletMeta tabletMeta = new
TGetMetaTabletMeta();
+ tabletMeta.setId(tablet.getId());
+
+ for (Replica replica : tablet.getReplicas()) {
+ TGetMetaReplicaMeta replicaMeta = new
TGetMetaReplicaMeta();
+ replicaMeta.setId(replica.getId());
+ replicaMeta.setBackendId(replica.getBackendId());
+ replicaMeta.setVersion(replica.getVersion());
+ tabletMeta.addToReplicas(replicaMeta);
+ }
+
+ indexMeta.addToTablets(tabletMeta);
+ }
+
+ partitionMeta.addToIndexes(indexMeta);
+ }
+ tableMeta.addToPartitions(partitionMeta);
+ }
+ dbMeta.addToTables(tableMeta);
+ } finally {
+ olapTable.readUnlock();
+ }
+ }
+
+ public static TGetMetaResult getMeta(Database db, List<Table> tables)
throws MetaNotFoundException {
+ TGetMetaResult result = new TGetMetaResult();
+ result.setStatus(new TStatus(TStatusCode.OK));
+
+ TGetMetaDBMeta dbMeta = new TGetMetaDBMeta();
+ dbMeta.setId(db.getId());
+ dbMeta.setName(db.getFullName());
+
+ if (tables == null) {
+ db.readLock();
+ tables = db.getTables();
+ db.readUnlock();
+ }
+
+ for (Table table : tables) {
+ if (table.getType() != TableType.OLAP) {
+ continue;
+ }
+
+ OlapTable olapTable = (OlapTable) table;
+ getTableMeta(olapTable, dbMeta);
+ }
+
+ result.setDbMeta(dbMeta);
+ return result;
+ }
+
public void compactTable(AdminCompactTableStmt stmt) throws DdlException {
String dbName = stmt.getDbName();
String tableName = stmt.getTblName();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
index febd638e73d..e6a883e07c7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
@@ -107,6 +107,7 @@ import org.apache.doris.thrift.TAddColumnsRequest;
import org.apache.doris.thrift.TAddColumnsResult;
import org.apache.doris.thrift.TAutoIncrementRangeRequest;
import org.apache.doris.thrift.TAutoIncrementRangeResult;
+import org.apache.doris.thrift.TBackend;
import org.apache.doris.thrift.TBeginTxnRequest;
import org.apache.doris.thrift.TBeginTxnResult;
import org.apache.doris.thrift.TBinlog;
@@ -134,6 +135,8 @@ import org.apache.doris.thrift.TFinishTaskRequest;
import org.apache.doris.thrift.TFrontendPingFrontendRequest;
import org.apache.doris.thrift.TFrontendPingFrontendResult;
import org.apache.doris.thrift.TFrontendPingFrontendStatusCode;
+import org.apache.doris.thrift.TGetBackendMetaRequest;
+import org.apache.doris.thrift.TGetBackendMetaResult;
import org.apache.doris.thrift.TGetBinlogLagResult;
import org.apache.doris.thrift.TGetBinlogRequest;
import org.apache.doris.thrift.TGetBinlogResult;
@@ -141,6 +144,10 @@ import org.apache.doris.thrift.TGetDbsParams;
import org.apache.doris.thrift.TGetDbsResult;
import org.apache.doris.thrift.TGetMasterTokenRequest;
import org.apache.doris.thrift.TGetMasterTokenResult;
+import org.apache.doris.thrift.TGetMetaDB;
+import org.apache.doris.thrift.TGetMetaRequest;
+import org.apache.doris.thrift.TGetMetaResult;
+import org.apache.doris.thrift.TGetMetaTable;
import org.apache.doris.thrift.TGetQueryStatsRequest;
import org.apache.doris.thrift.TGetSnapshotRequest;
import org.apache.doris.thrift.TGetSnapshotResult;
@@ -3331,4 +3338,171 @@ public class FrontendServiceImpl implements
FrontendService.Iface {
LOG.debug("send create partition result: {}", result);
return result;
}
+
+ public TGetMetaResult getMeta(TGetMetaRequest request) throws TException {
+ String clientAddr = getClientAddrAsString();
+ LOG.debug("receive get meta request: {}", request);
+
+ TGetMetaResult result = new TGetMetaResult();
+ TStatus status = new TStatus(TStatusCode.OK);
+ result.setStatus(status);
+ try {
+ result = getMetaImpl(request, clientAddr);
+ } catch (UserException e) {
+ LOG.warn("failed to get meta: {}", e.getMessage());
+ status.setStatusCode(TStatusCode.ANALYSIS_ERROR);
+ status.addToErrorMsgs(e.getMessage());
+ } catch (Throwable e) {
+ LOG.warn("catch unknown result.", e);
+ status.setStatusCode(TStatusCode.INTERNAL_ERROR);
+ status.addToErrorMsgs(Strings.nullToEmpty(e.getMessage()));
+ }
+ return result;
+ }
+
+ private TGetMetaResult getMetaImpl(TGetMetaRequest request, String
clientIp)
+ throws Exception {
+ // Step 1: check fields
+ if (!request.isSetUser()) {
+ throw new UserException("user is not set");
+ }
+ if (!request.isSetPasswd()) {
+ throw new UserException("passwd is not set");
+ }
+ if (!request.isSetDb()) {
+ throw new UserException("db is not set");
+ }
+
+ // Step 2: check auth
+ TGetMetaResult result = new TGetMetaResult();
+ result.setStatus(new TStatus(TStatusCode.OK));
+ Database db = null;
+ List<Table> tables = null;
+
+ String cluster = request.getCluster();
+ if (Strings.isNullOrEmpty(cluster)) {
+ cluster = SystemInfoService.DEFAULT_CLUSTER;
+ }
+ if (Strings.isNullOrEmpty(request.getToken())) {
+ TGetMetaDB getMetaDb = request.getDb();
+
+ if (getMetaDb.isSetId()) {
+ db =
Env.getCurrentInternalCatalog().getDbNullable(getMetaDb.getId());
+ } else if (getMetaDb.isSetName()) {
+ db =
Env.getCurrentInternalCatalog().getDbNullable(getMetaDb.getName());
+ }
+
+ if (db == null) {
+ LOG.warn("db not found {}", getMetaDb);
+ return result;
+ }
+
+ if (getMetaDb.isSetTables()) {
+ tables = Lists.newArrayList();
+ List<TGetMetaTable> getMetaTables = getMetaDb.getTables();
+ for (TGetMetaTable getMetaTable : getMetaTables) {
+ Table table = null;
+ if (getMetaTable.isSetId()) {
+ table = db.getTableNullable(getMetaTable.getId());
+ } else {
+ table = db.getTableNullable(getMetaTable.getName());
+ }
+
+ if (table == null) {
+ LOG.warn("table not found {}", getMetaTable);
+ continue;
+ }
+
+ tables.add(table);
+ }
+ }
+
+ if (tables == null) {
+ checkDbPasswordAndPrivs(cluster, request.getUser(),
request.getPasswd(), db.getFullName(), clientIp,
+ PrivPredicate.SELECT);
+ } else {
+ List<String> tableList = Lists.newArrayList();
+ for (Table table : tables) {
+ tableList.add(table.getName());
+ }
+ checkPasswordAndPrivs(cluster, request.getUser(),
request.getPasswd(), db.getFullName(), tableList,
+ clientIp,
+ PrivPredicate.SELECT);
+ }
+ }
+
+ // Step 3: get meta
+ try {
+ return Env.getMeta(db, tables);
+ } catch (Throwable e) {
+ throw e;
+ }
+ }
+
+ public TGetBackendMetaResult getBackendMeta(TGetBackendMetaRequest
request) throws TException {
+ String clientAddr = getClientAddrAsString();
+ LOG.debug("receive get backend meta request: {}", request);
+
+ TGetBackendMetaResult result = new TGetBackendMetaResult();
+ TStatus status = new TStatus(TStatusCode.OK);
+ result.setStatus(status);
+ try {
+ result = getBackendMetaImpl(request, clientAddr);
+ } catch (UserException e) {
+ LOG.warn("failed to get backend meta: {}", e.getMessage());
+ status.setStatusCode(TStatusCode.ANALYSIS_ERROR);
+ status.addToErrorMsgs(e.getMessage());
+ } catch (Throwable e) {
+ LOG.warn("catch unknown result.", e);
+ status.setStatusCode(TStatusCode.INTERNAL_ERROR);
+ status.addToErrorMsgs(Strings.nullToEmpty(e.getMessage()));
+ }
+
+ return result;
+ }
+
+ private TGetBackendMetaResult getBackendMetaImpl(TGetBackendMetaRequest
request, String clientAddr)
+ throws UserException {
+ // Step 1: Check fields
+ if (!request.isSetUser()) {
+ throw new UserException("user is not set");
+ }
+ if (!request.isSetPasswd()) {
+ throw new UserException("passwd is not set");
+ }
+
+ // Step 2: check auth
+ String cluster = request.getCluster();
+ if (Strings.isNullOrEmpty(cluster)) {
+ cluster = SystemInfoService.DEFAULT_CLUSTER;
+ }
+ checkPassword(request.getCluster(), request.getUser(),
request.getPasswd(), clientAddr);
+
+
+ // TODO: check getBackendMeta privilege, which privilege should be
checked?
+
+ // Step 3: get meta
+ try {
+ TGetBackendMetaResult result = new TGetBackendMetaResult();
+ result.setStatus(new TStatus(TStatusCode.OK));
+
+ final SystemInfoService systemInfoService =
Env.getCurrentSystemInfo();
+ List<Backend> backends = systemInfoService.getAllBackends();
+
+ for (Backend backend : backends) {
+ TBackend tBackend = new TBackend();
+ tBackend.setId(backend.getId());
+ tBackend.setHost(backend.getHost());
+ tBackend.setHttpPort(backend.getHttpPort());
+ tBackend.setBrpcPort(backend.getBrpcPort());
+ tBackend.setBePort(backend.getBePort());
+ tBackend.setIsAlive(backend.isAlive());
+ result.addToBackends(tBackend);
+ }
+
+ return result;
+ } catch (Throwable e) {
+ throw e;
+ }
+ }
}
diff --git a/gensrc/thrift/FrontendService.thrift
b/gensrc/thrift/FrontendService.thrift
index ee593083fe8..1e0ad2d4624 100644
--- a/gensrc/thrift/FrontendService.thrift
+++ b/gensrc/thrift/FrontendService.thrift
@@ -1183,6 +1183,113 @@ struct TCreatePartitionResult {
4: optional list<Descriptors.TNodeInfo> nodes
}
+struct TGetMetaReplica {
+ 1: optional i64 id
+}
+
+struct TGetMetaTablet {
+ 1: optional i64 id
+ 2: optional list<TGetMetaReplica> replicas
+}
+
+struct TGetMetaIndex {
+ 1: optional i64 id
+ 2: optional string name
+ 3: optional list<TGetMetaTablet> tablets
+}
+
+struct TGetMetaPartition {
+ 1: optional i64 id
+ 2: optional string name
+ 3: optional string key
+ 4: optional string range
+ 5: optional bool is_temp
+ 6: optional list<TGetMetaIndex> indexes
+}
+
+struct TGetMetaTable {
+ 1: optional i64 id
+ 2: optional string name
+ 3: optional bool in_trash
+ 4: optional list<TGetMetaPartition> partitions
+}
+
+struct TGetMetaDB {
+ 1: optional i64 id
+ 2: optional string name
+ 3: optional bool only_table_names
+ 4: optional list<TGetMetaTable> tables
+}
+
+struct TGetMetaRequest {
+ 1: optional string cluster
+ 2: optional string user
+ 3: optional string passwd
+ 4: optional string user_ip
+ 5: optional string token
+ 6: optional TGetMetaDB db
+ // trash
+}
+
+struct TGetMetaReplicaMeta {
+ 1: optional i64 id
+ 2: optional i64 backend_id
+ 3: optional i64 version
+}
+
+struct TGetMetaTabletMeta {
+ 1: optional i64 id
+ 2: optional list<TGetMetaReplicaMeta> replicas
+}
+
+struct TGetMetaIndexMeta {
+ 1: optional i64 id
+ 2: optional string name
+ 3: optional list<TGetMetaTabletMeta> tablets
+}
+
+struct TGetMetaPartitionMeta {
+ 1: optional i64 id
+ 2: optional string name
+ 3: optional string key
+ 4: optional string range
+ 5: optional i64 visible_version
+ 6: optional bool is_temp
+ 7: optional list<TGetMetaIndexMeta> indexes
+}
+
+struct TGetMetaTableMeta {
+ 1: optional i64 id
+ 2: optional string name
+ 3: optional bool in_trash
+ 4: optional list<TGetMetaPartitionMeta> partitions
+}
+
+struct TGetMetaDBMeta {
+ 1: optional i64 id
+ 2: optional string name
+ 3: optional list<TGetMetaTableMeta> tables
+}
+
+struct TGetMetaResult {
+ 1: required Status.TStatus status
+ 2: optional TGetMetaDBMeta db_meta
+}
+
+struct TGetBackendMetaRequest {
+ 1: optional string cluster
+ 2: optional string user
+ 3: optional string passwd
+ 4: optional string user_ip
+ 5: optional string token
+ 6: optional i64 backend_id
+}
+
+struct TGetBackendMetaResult {
+ 1: required Status.TStatus status
+ 2: optional list<Types.TBackend> backends
+}
+
service FrontendService {
TGetDbsResult getDbNames(1: TGetDbsParams params)
TGetTablesResult getTableNames(1: TGetTablesParams params)
@@ -1254,4 +1361,8 @@ service FrontendService {
TAutoIncrementRangeResult getAutoIncrementRange(1:
TAutoIncrementRangeRequest request)
TCreatePartitionResult createPartition(1: TCreatePartitionRequest request)
+
+ TGetMetaResult getMeta(1: TGetMetaRequest request)
+
+ TGetBackendMetaResult getBackendMeta(1: TGetBackendMetaRequest request)
}
diff --git a/gensrc/thrift/Types.thrift b/gensrc/thrift/Types.thrift
index 7da21f30bc1..69ad6009523 100644
--- a/gensrc/thrift/Types.thrift
+++ b/gensrc/thrift/Types.thrift
@@ -621,6 +621,9 @@ struct TBackend {
1: required string host
2: required TPort be_port
3: required TPort http_port
+ 4: optional TPort brpc_port
+ 5: optional bool is_alive
+ 6: optional i64 id
}
struct TReplicaInfo {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
