This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 1a42a2acb68 [opt](fe) Optimize fe show table statistics (#35457)
1a42a2acb68 is described below
commit 1a42a2acb687a5c3255fbc76e005ac27c0d061a4
Author: Lei Zhang <[email protected]>
AuthorDate: Tue May 28 20:16:45 2024 +0800
[opt](fe) Optimize fe show table statistics (#35457)
* Remove `olapTable.readLock` when show table statistics
---
.../org/apache/doris/analysis/ShowDataStmt.java | 13 +--
.../apache/doris/catalog/CloudTabletStatMgr.java | 71 +++++-------
.../java/org/apache/doris/catalog/Database.java | 21 ++--
.../java/org/apache/doris/catalog/OlapTable.java | 119 ++++++++++++++++-----
.../org/apache/doris/catalog/TabletStatMgr.java | 37 ++++++-
.../org/apache/doris/common/proc/DbsProcDir.java | 2 +-
.../doris/metric/PrometheusMetricVisitor.java | 13 +--
7 files changed, 173 insertions(+), 103 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowDataStmt.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowDataStmt.java
index 6ea50ac2b72..51690f9ce87 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowDataStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowDataStmt.java
@@ -190,14 +190,11 @@ public class ShowDataStmt extends ShowStmt {
long tableSize = 0;
long replicaCount = 0;
long remoteSize = 0;
- olapTable.readLock();
- try {
- tableSize = olapTable.getDataSize();
- replicaCount = olapTable.getReplicaCount();
- remoteSize = olapTable.getRemoteDataSize();
- } finally {
- olapTable.readUnlock();
- }
+
+ tableSize = olapTable.getDataSize();
+ replicaCount = olapTable.getReplicaCount();
+ remoteSize = olapTable.getRemoteDataSize();
+
//|TableName|Size|ReplicaCount|RemoteSize
List<Object> row = Arrays.asList(table.getName(),
tableSize, replicaCount, remoteSize);
totalRowsObject.add(row);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/CloudTabletStatMgr.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/CloudTabletStatMgr.java
index e2375131dc5..3a75862f8d3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/CloudTabletStatMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/CloudTabletStatMgr.java
@@ -29,7 +29,6 @@ import org.apache.doris.common.Pair;
import org.apache.doris.common.util.MasterDaemon;
import org.apache.doris.rpc.RpcException;
-import lombok.Getter;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -45,8 +44,8 @@ import java.util.Map;
public class CloudTabletStatMgr extends MasterDaemon {
private static final Logger LOG =
LogManager.getLogger(CloudTabletStatMgr.class);
- // <(dbId, tableId) -> CloudTableStats>
- private volatile Map<Pair<Long, Long>, CloudTableStats> cloudTableStatsMap
= new HashMap<>();
+ // <(dbId, tableId) -> OlapTable.Statistics>
+ private volatile Map<Pair<Long, Long>, OlapTable.Statistics>
cloudTableStatsMap = new HashMap<>();
public CloudTabletStatMgr() {
super("cloud tablet stat mgr",
Config.tablet_stat_update_interval_second * 1000);
@@ -134,7 +133,7 @@ public class CloudTabletStatMgr extends MasterDaemon {
// after update replica in all backends, update index row num
start = System.currentTimeMillis();
- Map<Pair<Long, Long>, CloudTableStats> newCloudTableStatsMap = new
HashMap<>();
+ Map<Pair<Long, Long>, OlapTable.Statistics> newCloudTableStatsMap =
new HashMap<>();
for (Long dbId : dbIds) {
Database db = Env.getCurrentInternalCatalog().getDbNullable(dbId);
if (db == null) {
@@ -148,14 +147,14 @@ public class CloudTabletStatMgr extends MasterDaemon {
}
OlapTable olapTable = (OlapTable) table;
- String dbName = db.getName();
- Long tableId = table.getId();
- String tableName = table.getName();
-
Long tableDataSize = 0L;
+ Long tableTotalReplicaDataSize = 0L;
+
+ Long tableReplicaCount = 0L;
+
+ Long tableRowCount = 0L;
Long tableRowsetCount = 0L;
Long tableSegmentCount = 0L;
- Long tableRowCount = 0L;
if (!table.writeLockIfExist()) {
continue;
@@ -167,6 +166,7 @@ public class CloudTabletStatMgr extends MasterDaemon {
long indexRowCount = 0L;
for (Tablet tablet : index.getTablets()) {
long tabletDataSize = 0L;
+
long tabletRowsetCount = 0L;
long tabletSegmentCount = 0L;
long tabletRowCount = 0L;
@@ -174,6 +174,11 @@ public class CloudTabletStatMgr extends MasterDaemon {
for (Replica replica : tablet.getReplicas()) {
if (replica.getDataSize() >
tabletDataSize) {
tabletDataSize = replica.getDataSize();
+ tableTotalReplicaDataSize +=
replica.getDataSize();
+ }
+
+ if (replica.getRowCount() >
tabletRowCount) {
+ tabletRowCount = replica.getRowCount();
}
if (replica.getRowsetCount() >
tabletRowsetCount) {
@@ -184,29 +189,33 @@ public class CloudTabletStatMgr extends MasterDaemon {
tabletSegmentCount =
replica.getSegmentCount();
}
- if (replica.getRowCount() >
tabletRowCount) {
- tabletRowCount = replica.getRowCount();
- }
+ tableReplicaCount++;
}
tableDataSize += tabletDataSize;
- tableRowsetCount += tabletRowsetCount;
- tableSegmentCount += tabletSegmentCount;
- tableRowCount += tabletRowCount;
+ tableRowCount += tabletRowCount;
indexRowCount += tabletRowCount;
+
+ tableRowsetCount += tabletRowsetCount;
+ tableSegmentCount += tabletSegmentCount;
} // end for tablets
index.setRowCount(indexRowCount);
} // end for indices
} // end for partitions
+
+ olapTable.setStatistics(new
OlapTable.Statistics(db.getName(),
+ table.getName(), tableDataSize,
tableTotalReplicaDataSize, 0L,
+ tableReplicaCount, tableRowCount,
tableRowsetCount, tableSegmentCount));
LOG.debug("finished to set row num for table: {} in
database: {}",
table.getName(), db.getFullName());
} finally {
table.writeUnlock();
}
- newCloudTableStatsMap.put(Pair.of(dbId, tableId), new
CloudTableStats(dbName, tableName,
- tableDataSize, tableRowsetCount, tableSegmentCount,
tableRowCount));
+ newCloudTableStatsMap.put(Pair.of(dbId, table.getId()), new
OlapTable.Statistics(db.getName(),
+ table.getName(), tableDataSize,
tableTotalReplicaDataSize, 0L,
+ tableReplicaCount, tableRowCount, tableRowsetCount,
tableSegmentCount));
}
}
this.cloudTableStatsMap = newCloudTableStatsMap;
@@ -243,33 +252,7 @@ public class CloudTabletStatMgr extends MasterDaemon {
return response;
}
- public Map<Pair<Long, Long>, CloudTableStats> getCloudTableStatsMap() {
+ public Map<Pair<Long, Long>, OlapTable.Statistics> getCloudTableStatsMap()
{
return this.cloudTableStatsMap;
}
-
- public static class CloudTableStats {
- @Getter
- private String dbName;
- @Getter
- private String tableName;
-
- @Getter
- private Long tableDataSize;
- @Getter
- private Long tableRowsetCount;
- @Getter
- private Long tableSegmentCount;
- @Getter
- private Long tableRowCount;
-
- public CloudTableStats(String dbName, String tableName, Long
tableDataSize, Long tableRowsetCount,
- Long tableSegmentCount, Long tableRowCount) {
- this.dbName = dbName;
- this.tableName = tableName;
- this.tableDataSize = tableDataSize;
- this.tableRowsetCount = tableRowsetCount;
- this.tableSegmentCount = tableSegmentCount;
- this.tableRowCount = tableRowCount;
- }
- }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java
index 3f6d0daa060..ffc9db2a89a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java
@@ -309,18 +309,14 @@ public class Database extends MetaObject implements
Writable, DatabaseIf<Table>
}
OlapTable olapTable = (OlapTable) table;
- olapTable.readLock();
- try {
- usedDataSize = usedDataSize + olapTable.getDataSize();
- usedRemoteDataSize = usedRemoteDataSize +
olapTable.getRemoteDataSize();
- } finally {
- olapTable.readUnlock();
- }
+ usedDataSize = usedDataSize + olapTable.getDataSize();
+ usedRemoteDataSize = usedRemoteDataSize +
olapTable.getRemoteDataSize();
+
}
return Pair.of(usedDataSize, usedRemoteDataSize);
}
- public long getReplicaCountWithLock() {
+ public long getReplicaCount() {
readLock();
try {
long usedReplicaCount = 0;
@@ -330,12 +326,7 @@ public class Database extends MetaObject implements
Writable, DatabaseIf<Table>
}
OlapTable olapTable = (OlapTable) table;
- olapTable.readLock();
- try {
- usedReplicaCount = usedReplicaCount +
olapTable.getReplicaCount();
- } finally {
- olapTable.readUnlock();
- }
+ usedReplicaCount = usedReplicaCount +
olapTable.getReplicaCount();
}
return usedReplicaCount;
} finally {
@@ -344,7 +335,7 @@ public class Database extends MetaObject implements
Writable, DatabaseIf<Table>
}
public long getReplicaQuotaLeftWithLock() {
- long leftReplicaQuota = replicaQuotaSize - getReplicaCountWithLock();
+ long leftReplicaQuota = replicaQuotaSize - getReplicaCount();
return Math.max(leftReplicaQuota, 0L);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
index a223d72bc5e..950da6d5fed 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
@@ -88,6 +88,7 @@ import com.google.common.collect.Maps;
import com.google.common.collect.Range;
import com.google.common.collect.Sets;
import com.google.gson.annotations.SerializedName;
+import lombok.Getter;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -185,6 +186,8 @@ public class OlapTable extends Table implements
MTMVRelatedTableIf {
private AutoIncrementGenerator autoIncrementGenerator;
+ private volatile Statistics statistics = new Statistics();
+
public OlapTable() {
// for persist
super(TableType.OLAP);
@@ -1801,34 +1804,6 @@ public class OlapTable extends Table implements
MTMVRelatedTableIf {
return oldPartition;
}
- public long getDataSize(boolean singleReplica) {
- long dataSize = 0;
- for (Partition partition : getAllPartitions()) {
- dataSize += partition.getDataSize(singleReplica);
- }
- return dataSize;
- }
-
- public long getDataSize() {
- return getDataSize(false);
- }
-
- public long getRemoteDataSize() {
- long remoteDataSize = 0;
- for (Partition partition : getAllPartitions()) {
- remoteDataSize += partition.getRemoteDataSize();
- }
- return remoteDataSize;
- }
-
- public long getReplicaCount() {
- long replicaCount = 0;
- for (Partition partition : getAllPartitions()) {
- replicaCount += partition.getReplicaCount();
- }
- return replicaCount;
- }
-
public void checkNormalStateForAlter() throws DdlException {
if (state != OlapTableState.NORMAL) {
throw new DdlException("Table[" + name + "]'s state(" +
state.toString()
@@ -2875,4 +2850,92 @@ public class OlapTable extends Table implements
MTMVRelatedTableIf {
}
return null;
}
+
+ public void setStatistics(Statistics statistics) {
+ this.statistics = statistics;
+ }
+
+ public static class Statistics {
+ @Getter
+ private String dbName;
+ @Getter
+ private String tableName;
+
+ @Getter
+ private Long dataSize; // single replica data size
+ @Getter
+ private Long totalReplicaDataSize;
+
+ @Getter
+ private Long remoteDataSize; // single replica remote data size
+
+ @Getter
+ private Long replicaCount;
+
+ @Getter
+ private Long rowCount;
+
+ @Getter
+ private Long rowsetCount;
+
+ @Getter
+ private Long segmentCount;
+
+ public Statistics() {
+ this.dbName = null;
+ this.tableName = null;
+
+ this.dataSize = 0L;
+ this.totalReplicaDataSize = 0L;
+
+ this.remoteDataSize = 0L;
+
+ this.replicaCount = 0L;
+
+ this.rowCount = 0L;
+ this.rowsetCount = 0L;
+ this.segmentCount = 0L;
+
+ }
+
+ public Statistics(String dbName, String tableName,
+ Long dataSize, Long totalReplicaDataSize,
+ Long remoteDataSize, Long replicaCount, Long rowCount,
+ Long rowsetCount, Long segmentCount) {
+
+ this.dbName = dbName;
+ this.tableName = tableName;
+
+ this.dataSize = dataSize;
+ this.totalReplicaDataSize = totalReplicaDataSize;
+
+ this.remoteDataSize = remoteDataSize;
+
+ this.replicaCount = replicaCount;
+
+ this.rowCount = rowCount;
+ this.rowsetCount = rowsetCount;
+ this.segmentCount = segmentCount;
+ }
+ }
+
+ public long getDataSize() {
+ return getDataSize(false);
+ }
+
+ public long getDataSize(boolean singleReplica) {
+ if (singleReplica) {
+ statistics.getDataSize();
+ }
+
+ return statistics.getTotalReplicaDataSize();
+ }
+
+ public long getRemoteDataSize() {
+ return statistics.getRemoteDataSize();
+ }
+
+ public long getReplicaCount() {
+ return statistics.getReplicaCount();
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletStatMgr.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletStatMgr.java
index 896ecac6f8e..c610993474f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletStatMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletStatMgr.java
@@ -104,6 +104,16 @@ public class TabletStatMgr extends MasterDaemon {
continue;
}
OlapTable olapTable = (OlapTable) table;
+
+ Long tableDataSize = 0L;
+ Long tableTotalReplicaDataSize = 0L;
+
+ Long tableRemoteDataSize = 0L;
+
+ Long tableReplicaCount = 0L;
+
+ Long tableRowCount = 0L;
+
// Use try write lock to avoid such cases
// Time1: Thread1 hold read lock for 5min
// Time2: Thread2 want to add write lock, then it will be
the first element in lock queue
@@ -119,18 +129,43 @@ public class TabletStatMgr extends MasterDaemon {
for (MaterializedIndex index :
partition.getMaterializedIndices(IndexExtState.VISIBLE)) {
long indexRowCount = 0L;
for (Tablet tablet : index.getTablets()) {
- long tabletRowCount = 0L;
+
+ Long tabletDataSize = 0L;
+ Long tabletRemoteDataSize = 0L;
+
+ Long tabletRowCount = 0L;
+
for (Replica replica : tablet.getReplicas()) {
if (replica.checkVersionCatchUp(version,
false)
&& replica.getRowCount() >
tabletRowCount) {
tabletRowCount = replica.getRowCount();
}
+
+ if (replica.getDataSize() >
tabletDataSize) {
+ tabletDataSize = replica.getDataSize();
+ tableTotalReplicaDataSize +=
replica.getDataSize();
+ }
+
+ if (replica.getRemoteDataSize() >
tabletRemoteDataSize) {
+ tabletRemoteDataSize =
replica.getRemoteDataSize();
+ }
+ tableReplicaCount++;
}
+
+ tableDataSize += tabletDataSize;
+ tableRemoteDataSize += tabletRemoteDataSize;
+
+ tableRowCount += tabletRowCount;
indexRowCount += tabletRowCount;
} // end for tablets
index.setRowCount(indexRowCount);
} // end for indices
} // end for partitions
+
+ olapTable.setStatistics(new
OlapTable.Statistics(db.getName(), table.getName(),
+ tableDataSize, tableTotalReplicaDataSize,
+ tableRemoteDataSize, tableReplicaCount,
tableRowCount, 0L, 0L));
+
if (LOG.isDebugEnabled()) {
LOG.debug("finished to set row num for table: {} in
database: {}",
table.getName(), db.getFullName());
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/common/proc/DbsProcDir.java
b/fe/fe-core/src/main/java/org/apache/doris/common/proc/DbsProcDir.java
index 2fcd2f49b48..5a4fb3460ae 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/DbsProcDir.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/DbsProcDir.java
@@ -113,7 +113,7 @@ public class DbsProcDir implements ProcDirInterface {
String readableQuota = DebugUtil.printByteWithUnit(dataQuota);
String lastCheckTime = (db instanceof Database) ?
TimeUtils.longToTimeString(
((Database) db).getLastCheckTime()) :
FeConstants.null_string;
- long replicaCount = (db instanceof Database) ? ((Database)
db).getReplicaCountWithLock() : 0;
+ long replicaCount = (db instanceof Database) ? ((Database)
db).getReplicaCount() : 0;
long replicaQuota = (db instanceof Database) ? ((Database)
db).getReplicaQuota() : 0;
long transactionNum = (db instanceof Database) ?
env.getGlobalTransactionMgr()
.getRunningTxnNums(db.getId()) : 0;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/metric/PrometheusMetricVisitor.java
b/fe/fe-core/src/main/java/org/apache/doris/metric/PrometheusMetricVisitor.java
index 91df0410be4..c29bd958128 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/metric/PrometheusMetricVisitor.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/metric/PrometheusMetricVisitor.java
@@ -19,6 +19,7 @@ package org.apache.doris.metric;
import org.apache.doris.catalog.CloudTabletStatMgr;
import org.apache.doris.catalog.Env;
+import org.apache.doris.catalog.OlapTable;
import org.apache.doris.common.Config;
import org.apache.doris.common.Pair;
import org.apache.doris.monitor.jvm.JvmStats;
@@ -244,15 +245,15 @@ public class PrometheusMetricVisitor extends
MetricVisitor {
StringBuilder tableRowCountBuilder = new StringBuilder();
long totalTableSize = 0;
- for (CloudTabletStatMgr.CloudTableStats stats :
tabletStatMgr.getCloudTableStatsMap().values()) {
- totalTableSize += stats.getTableDataSize();
+ for (OlapTable.Statistics stats :
tabletStatMgr.getCloudTableStatsMap().values()) {
+ totalTableSize += stats.getDataSize();
dataSizeBuilder.append("doris_fe_table_data_size{db_name=\"");
dataSizeBuilder.append(stats.getDbName());
dataSizeBuilder.append("\", table_name=\"");
dataSizeBuilder.append(stats.getTableName());
dataSizeBuilder.append("\"} ");
- dataSizeBuilder.append(stats.getTableDataSize());
+ dataSizeBuilder.append(stats.getDataSize());
dataSizeBuilder.append("\n");
rowsetCountBuilder.append("doris_fe_table_rowset_count{db_name=\"");
@@ -260,7 +261,7 @@ public class PrometheusMetricVisitor extends MetricVisitor {
rowsetCountBuilder.append("\", table_name=\"");
rowsetCountBuilder.append(stats.getTableName());
rowsetCountBuilder.append("\"} ");
- rowsetCountBuilder.append(stats.getTableRowsetCount());
+ rowsetCountBuilder.append(stats.getRowsetCount());
rowsetCountBuilder.append("\n");
segmentCountBuilder.append("doris_fe_table_segment_count{db_name=\"");
@@ -268,7 +269,7 @@ public class PrometheusMetricVisitor extends MetricVisitor {
segmentCountBuilder.append("\", table_name=\"");
segmentCountBuilder.append(stats.getTableName());
segmentCountBuilder.append("\"} ");
- segmentCountBuilder.append(stats.getTableSegmentCount());
+ segmentCountBuilder.append(stats.getSegmentCount());
segmentCountBuilder.append("\n");
tableRowCountBuilder.append("doris_fe_table_row_count{db_name=\"");
@@ -276,7 +277,7 @@ public class PrometheusMetricVisitor extends MetricVisitor {
tableRowCountBuilder.append("\", table_name=\"");
tableRowCountBuilder.append(stats.getTableName());
tableRowCountBuilder.append("\"} ");
- tableRowCountBuilder.append(stats.getTableRowCount());
+ tableRowCountBuilder.append(stats.getRowCount());
tableRowCountBuilder.append("\n");
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
