This is an automated email from the ASF dual-hosted git repository.
lide pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-1.2-lts by this push:
new 497ae91c6e6 [improvement](http) add show_table_data http api (#28380)
(#28631)
497ae91c6e6 is described below
commit 497ae91c6e6c8ff486cdda263366c9aadf75872b
Author: xueweizhang <[email protected]>
AuthorDate: Mon Dec 25 15:16:49 2023 +0800
[improvement](http) add show_table_data http api (#28380) (#28631)
---
.../http-actions/fe/show-table-data-action.md | 163 +++++++++++++++++++++
docs/sidebars.json | 1 +
.../http-actions/fe/show-table-data-action.md | 163 +++++++++++++++++++++
.../java/org/apache/doris/catalog/OlapTable.java | 8 +-
.../doris/httpv2/rest/RestBaseController.java | 1 +
.../org/apache/doris/httpv2/rest/ShowAction.java | 69 +++++++++
6 files changed, 403 insertions(+), 2 deletions(-)
diff --git
a/docs/en/docs/admin-manual/http-actions/fe/show-table-data-action.md
b/docs/en/docs/admin-manual/http-actions/fe/show-table-data-action.md
new file mode 100644
index 00000000000..9cdbe2c49ad
--- /dev/null
+++ b/docs/en/docs/admin-manual/http-actions/fe/show-table-data-action.md
@@ -0,0 +1,163 @@
+---
+{
+ "title": "Show Table Data Action",
+ "language": "en"
+}
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Show Table Data Action
+
+## Request
+
+`GET /api/show_table_data`
+
+## Description
+
+Used to get the data size of all tables in all databases under all internal
catalog, or the data size of the specified database or table. Unit byte.
+
+## Path parameters
+
+NULL
+
+## Query parameters
+
+* `db`
+
+ Optional. If specified, get the data size of the tables under the
specified database.
+
+* `table`
+
+ Optional. If specified, get the data size of the specified table.
+
+* `single_replica`
+
+ Optional. If specified, get the data size of the single replica of the
table.
+
+## Request body
+
+NULL
+
+## Response
+
+1. The data size of all tables in the specified database.
+
+ ```
+ {
+ "msg":"success",
+ "code":0,
+ "data":{
+ "tpch":{
+ "partsupp":9024548244,
+ "revenue0":0,
+ "customer":1906421482
+ }
+ },
+ "count":0
+ }
+ ```
+
+2. The data size of the specified table of the specified db.
+
+ ```
+ {
+ "msg":"success",
+ "code":0,
+ "data":{
+ "tpch":{
+ "partsupp":9024548244
+ }
+ },
+ "count":0
+ }
+ ```
+
+3. The data size of the single replica of the specified table of the specified
db.
+
+ ```
+ {
+ "msg":"success",
+ "code":0,
+ "data":{
+ "tpch":{
+ "partsupp":3008182748
+ }
+ },
+ "count":0
+ }
+ ```
+
+## Examples
+
+1. The data size of all tables in the specified database.
+
+ ```
+ GET /api/show_table_data?db=tpch
+
+ Response:
+ {
+ "msg":"success",
+ "code":0,
+ "data":{
+ "tpch":{
+ "partsupp":9024548244,
+ "revenue0":0,
+ "customer":1906421482
+ }
+ },
+ "count":0
+ }
+ ```
+
+2. The data size of the specified table of the specified db.
+
+ ```
+ GET /api/show_table_data?db=tpch&table=partsupp
+
+ Response:
+ {
+ "msg":"success",
+ "code":0,
+ "data":{
+ "tpch":{
+ "partsupp":9024548244
+ }
+ },
+ "count":0
+ }
+ ```
+3. The data size of the single replica of the specified table of the specified
db.
+
+ ```
+ GET /api/show_table_data?db=tpch&table=partsupp&single_replica=true
+
+ Response:
+ {
+ "msg":"success",
+ "code":0,
+ "data":{
+ "tpch":{
+ "partsupp":3008182748
+ }
+ },
+ "count":0
+ }
+ ```
diff --git a/docs/sidebars.json b/docs/sidebars.json
index 55b66394f33..0dde25b4779 100644
--- a/docs/sidebars.json
+++ b/docs/sidebars.json
@@ -1162,6 +1162,7 @@
"admin-manual/http-actions/fe/show-meta-info-action",
"admin-manual/http-actions/fe/show-proc-action",
"admin-manual/http-actions/fe/show-runtime-info-action",
+
"admin-manual/http-actions/fe/show-table-data-action",
"admin-manual/http-actions/fe/statement-execution-action",
"admin-manual/http-actions/fe/table-query-plan-action",
"admin-manual/http-actions/fe/table-row-count-action",
diff --git
a/docs/zh-CN/docs/admin-manual/http-actions/fe/show-table-data-action.md
b/docs/zh-CN/docs/admin-manual/http-actions/fe/show-table-data-action.md
new file mode 100644
index 00000000000..97bb3dcf3b8
--- /dev/null
+++ b/docs/zh-CN/docs/admin-manual/http-actions/fe/show-table-data-action.md
@@ -0,0 +1,163 @@
+---
+{
+ "title": "Show Table Data Action",
+ "language": "zh-CN"
+}
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Show Table Data Action
+
+## Request
+
+`GET /api/show_table_data`
+
+## Description
+
+用于获取所有internal源下所有数据库所有表的数据量，或者指定数据库或指定表的数据量。单位字节。
+
+## Path parameters
+
+无
+
+## Query parameters
+
+* `db`
+
+ 可选。如果指定，则获取指定数据库下表的数据量。
+
+* `table`
+
+ 可选。如果指定，则获取指定表的数据量。
+
+* `single_replica`
+
+ 可选。如果指定，则获取表单副本所占用的数据量。
+
+## Request body
+
+无
+
+## Response
+
+1. 指定数据库所有表的数据量。
+
+ ```
+ {
+ "msg":"success",
+ "code":0,
+ "data":{
+ "tpch":{
+ "partsupp":9024548244,
+ "revenue0":0,
+ "customer":1906421482
+ }
+ },
+ "count":0
+ }
+ ```
+
+2. 指定数据库指定表的数据量。
+
+ ```
+ {
+ "msg":"success",
+ "code":0,
+ "data":{
+ "tpch":{
+ "partsupp":9024548244
+ }
+ },
+ "count":0
+ }
+ ```
+
+3. 指定数据库指定表单副本的数据量。
+
+ ```
+ {
+ "msg":"success",
+ "code":0,
+ "data":{
+ "tpch":{
+ "partsupp":3008182748
+ }
+ },
+ "count":0
+ }
+ ```
+
+## Examples
+
+1. 获取指定数据库的数据量
+
+ ```
+ GET /api/show_table_data?db=tpch
+
+ Response:
+ {
+ "msg":"success",
+ "code":0,
+ "data":{
+ "tpch":{
+ "partsupp":9024548244,
+ "revenue0":0,
+ "customer":1906421482
+ }
+ },
+ "count":0
+ }
+ ```
+
+2. 指定数据库指定表的数据量。
+
+ ```
+ GET /api/show_table_data?db=tpch&table=partsupp
+
+ Response:
+ {
+ "msg":"success",
+ "code":0,
+ "data":{
+ "tpch":{
+ "partsupp":9024548244
+ }
+ },
+ "count":0
+ }
+ ```
+3. 指定数据库指定表单副本的数据量。
+
+ ```
+ GET /api/show_table_data?db=tpch&table=partsupp&single_replica=true
+
+ Response:
+ {
+ "msg":"success",
+ "code":0,
+ "data":{
+ "tpch":{
+ "partsupp":3008182748
+ }
+ },
+ "count":0
+ }
+ ```
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
index d880ce05045..5c9ca6184d9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
@@ -1393,14 +1393,18 @@ public class OlapTable extends Table {
return oldPartition;
}
- public long getDataSize() {
+ public long getDataSize(boolean singleReplica) {
long dataSize = 0;
for (Partition partition : getAllPartitions()) {
- dataSize += partition.getDataSize(false);
+ dataSize += partition.getDataSize(singleReplica);
}
return dataSize;
}
+ public long getDataSize() {
+ return getDataSize(false);
+ }
+
public long getReplicaCount() {
long replicaCount = 0;
for (Partition partition : getAllPartitions()) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/RestBaseController.java
b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/RestBaseController.java
index 2f56831d446..80e2b4e2fa6 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/RestBaseController.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/RestBaseController.java
@@ -49,6 +49,7 @@ public class RestBaseController extends BaseController {
protected static final String LABEL_KEY = "label";
protected static final String TXN_ID_KEY = "txn_id";
protected static final String TXN_OPERATION_KEY = "txn_operation";
+ protected static final String SINGLE_REPLICA_KEY = "single_replica";
private static final Logger LOG =
LogManager.getLogger(RestBaseController.class);
public ActionAuthorizationInfo executeCheckPassword(HttpServletRequest
request,
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/ShowAction.java
b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/ShowAction.java
index dd3fbf48f88..659fedb7088 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/ShowAction.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/ShowAction.java
@@ -23,6 +23,7 @@ import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Table;
import org.apache.doris.catalog.TableIf.TableType;
+import org.apache.doris.cluster.ClusterNamespace;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Config;
import org.apache.doris.common.proc.ProcNodeInterface;
@@ -190,6 +191,34 @@ public class ShowAction extends RestBaseController {
return ResponseEntityBuilder.ok(oneEntry);
}
+ @RequestMapping(path = "/api/show_table_data", method = RequestMethod.GET)
+ public Object show_table_data(HttpServletRequest request,
HttpServletResponse response) {
+ String dbName = request.getParameter(DB_KEY);
+ String tableName = request.getParameter(TABLE_KEY);
+ String singleReplica = request.getParameter(SINGLE_REPLICA_KEY);
+ boolean singleReplicaBool = Boolean.parseBoolean(singleReplica);
+ Map<String, Map<String, Long>> oneEntry = Maps.newHashMap();
+ if (dbName != null) {
+ String fullDbName = getFullDbName(dbName);
+ DatabaseIf db =
Env.getCurrentInternalCatalog().getDbNullable(fullDbName);
+ if (db == null) {
+ return ResponseEntityBuilder.okWithCommonError("database " +
fullDbName + " not found.");
+ }
+ Map<String, Long> tablesEntry = getDataSizeOfTables(db, tableName,
singleReplicaBool);
+ oneEntry.put(ClusterNamespace.getNameFromFullName(fullDbName),
tablesEntry);
+ } else {
+ for (long dbId : Env.getCurrentInternalCatalog().getDbIds()) {
+ DatabaseIf db =
Env.getCurrentInternalCatalog().getDbNullable(dbId);
+ if (db == null || !(db instanceof Database) || ((Database)
db).isInfoSchemaDb()) {
+ continue;
+ }
+ Map<String, Long> tablesEntry = getDataSizeOfTables(db,
tableName, singleReplicaBool);
+
oneEntry.put(ClusterNamespace.getNameFromFullName(db.getFullName()),
tablesEntry);
+ }
+ }
+ return ResponseEntityBuilder.ok(oneEntry);
+ }
+
private Map<String, String> getHaInfo() throws IOException {
HashMap<String, String> feInfo = new HashMap<String, String>();
feInfo.put("role", Env.getCurrentEnv().getFeType().toString());
@@ -271,6 +300,46 @@ public class ShowAction extends RestBaseController {
return totalSize;
}
+ public Map<String, Long> getDataSizeOfTables(DatabaseIf db, String
tableName, boolean singleReplica) {
+ Map<String, Long> oneEntry = Maps.newHashMap();
+ db.readLock();
+ try {
+ if (Strings.isNullOrEmpty(tableName)) {
+ List<Table> tables = db.getTables();
+ for (Table table : tables) {
+ Map<String, Long> tableEntry = getDataSizeOfTable(table,
singleReplica);
+ oneEntry.putAll(tableEntry);
+ }
+ } else {
+ Table table = ((Database) db).getTableNullable(tableName);
+ if (table == null) {
+ return oneEntry;
+ }
+ Map<String, Long> tableEntry = getDataSizeOfTable(table,
singleReplica);
+ oneEntry.putAll(tableEntry);
+ }
+ } finally {
+ db.readUnlock();
+ }
+ return oneEntry;
+ }
+
+ public Map<String, Long> getDataSizeOfTable(Table table, boolean
singleReplica) {
+ Map<String, Long> oneEntry = Maps.newHashMap();
+ if (table.getType() == TableType.VIEW || table.getType() ==
TableType.ODBC) {
+ oneEntry.put(table.getName(), 0L);
+ } else if (table.getType() == TableType.OLAP) {
+ table.readLock();
+ try {
+ long tableSize = ((OlapTable)
table).getDataSize(singleReplica);
+ oneEntry.put(table.getName(), tableSize);
+ } finally {
+ table.readUnlock();
+ }
+ }
+ return oneEntry;
+ }
+
private Map<String, Long> getDataSize() {
Map<String, Long> result = new HashMap<String, Long>();
List<String> dbNames = Env.getCurrentInternalCatalog().getDbNames();
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
