This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 4f2a67251a8 [opt](fe) Optimize fe show table statistics (#35457)
(#43858)
4f2a67251a8 is described below
commit 4f2a67251a81f27d3124cabb472536b592936d10
Author: Lei Zhang <[email protected]>
AuthorDate: Thu Nov 14 12:41:29 2024 +0800
[opt](fe) Optimize fe show table statistics (#35457) (#43858)
---
.../org/apache/doris/analysis/ShowDataStmt.java | 13 +--
.../java/org/apache/doris/catalog/Database.java | 21 ++--
.../java/org/apache/doris/catalog/OlapTable.java | 119 ++++++++++++++++-----
.../org/apache/doris/catalog/TabletStatMgr.java | 36 ++++++-
.../org/apache/doris/common/proc/DbsProcDir.java | 2 +-
5 files changed, 138 insertions(+), 53 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowDataStmt.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowDataStmt.java
index 799fa68bcf7..58818667a92 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowDataStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowDataStmt.java
@@ -178,14 +178,11 @@ public class ShowDataStmt extends ShowStmt {
long tableSize = 0;
long replicaCount = 0;
long remoteSize = 0;
- olapTable.readLock();
- try {
- tableSize = olapTable.getDataSize();
- replicaCount = olapTable.getReplicaCount();
- remoteSize = olapTable.getRemoteDataSize();
- } finally {
- olapTable.readUnlock();
- }
+
+ tableSize = olapTable.getDataSize();
+ replicaCount = olapTable.getReplicaCount();
+ remoteSize = olapTable.getRemoteDataSize();
+
//|TableName|Size|ReplicaCount|RemoteSize
List<Object> row = Arrays.asList(table.getName(),
tableSize, replicaCount, remoteSize);
totalRowsObject.add(row);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java
index a4d8e5f0302..ba051a5c0a5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java
@@ -299,18 +299,14 @@ public class Database extends MetaObject implements
Writable, DatabaseIf<Table>
}
OlapTable olapTable = (OlapTable) table;
- olapTable.readLock();
- try {
- usedDataSize = usedDataSize + olapTable.getDataSize();
- usedRemoteDataSize = usedRemoteDataSize +
olapTable.getRemoteDataSize();
- } finally {
- olapTable.readUnlock();
- }
+ usedDataSize = usedDataSize + olapTable.getDataSize();
+ usedRemoteDataSize = usedRemoteDataSize +
olapTable.getRemoteDataSize();
+
}
return Pair.of(usedDataSize, usedRemoteDataSize);
}
- public long getReplicaCountWithLock() {
+ public long getReplicaCount() {
readLock();
try {
long usedReplicaCount = 0;
@@ -320,12 +316,7 @@ public class Database extends MetaObject implements
Writable, DatabaseIf<Table>
}
OlapTable olapTable = (OlapTable) table;
- olapTable.readLock();
- try {
- usedReplicaCount = usedReplicaCount +
olapTable.getReplicaCount();
- } finally {
- olapTable.readUnlock();
- }
+ usedReplicaCount = usedReplicaCount +
olapTable.getReplicaCount();
}
return usedReplicaCount;
} finally {
@@ -334,7 +325,7 @@ public class Database extends MetaObject implements
Writable, DatabaseIf<Table>
}
public long getReplicaQuotaLeftWithLock() {
- long leftReplicaQuota = replicaQuotaSize - getReplicaCountWithLock();
+ long leftReplicaQuota = replicaQuotaSize - getReplicaCount();
return Math.max(leftReplicaQuota, 0L);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
index 7ddc51224b7..8c7e8b59646 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
@@ -87,6 +87,7 @@ import com.google.common.collect.Maps;
import com.google.common.collect.Range;
import com.google.common.collect.Sets;
import com.google.gson.annotations.SerializedName;
+import lombok.Getter;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -186,6 +187,8 @@ public class OlapTable extends Table implements
MTMVRelatedTableIf {
private AutoIncrementGenerator autoIncrementGenerator;
+ private volatile Statistics statistics = new Statistics();
+
public OlapTable() {
// for persist
super(TableType.OLAP);
@@ -1981,34 +1984,6 @@ public class OlapTable extends Table implements
MTMVRelatedTableIf {
return oldPartition;
}
- public long getDataSize(boolean singleReplica) {
- long dataSize = 0;
- for (Partition partition : getAllPartitions()) {
- dataSize += partition.getDataSize(singleReplica);
- }
- return dataSize;
- }
-
- public long getDataSize() {
- return getDataSize(false);
- }
-
- public long getRemoteDataSize() {
- long remoteDataSize = 0;
- for (Partition partition : getAllPartitions()) {
- remoteDataSize += partition.getRemoteDataSize();
- }
- return remoteDataSize;
- }
-
- public long getReplicaCount() {
- long replicaCount = 0;
- for (Partition partition : getAllPartitions()) {
- replicaCount += partition.getReplicaCount();
- }
- return replicaCount;
- }
-
public void checkNormalStateForAlter() throws DdlException {
if (state != OlapTableState.NORMAL) {
throw new DdlException("Table[" + name + "]'s state(" +
state.toString()
@@ -3087,4 +3062,92 @@ public class OlapTable extends Table implements
MTMVRelatedTableIf {
protected void addIndexNameToIdForUnitTest(String name, long id) {
indexNameToId.put(name, id);
}
+
+ public void setStatistics(Statistics statistics) {
+ this.statistics = statistics;
+ }
+
+ public static class Statistics {
+ @Getter
+ private String dbName;
+ @Getter
+ private String tableName;
+
+ @Getter
+ private Long dataSize; // single replica data size
+ @Getter
+ private Long totalReplicaDataSize;
+
+ @Getter
+ private Long remoteDataSize; // single replica remote data size
+
+ @Getter
+ private Long replicaCount;
+
+ @Getter
+ private Long rowCount;
+
+ @Getter
+ private Long rowsetCount;
+
+ @Getter
+ private Long segmentCount;
+
+ public Statistics() {
+ this.dbName = null;
+ this.tableName = null;
+
+ this.dataSize = 0L;
+ this.totalReplicaDataSize = 0L;
+
+ this.remoteDataSize = 0L;
+
+ this.replicaCount = 0L;
+
+ this.rowCount = 0L;
+ this.rowsetCount = 0L;
+ this.segmentCount = 0L;
+
+ }
+
+ public Statistics(String dbName, String tableName,
+ Long dataSize, Long totalReplicaDataSize,
+ Long remoteDataSize, Long replicaCount, Long rowCount,
+ Long rowsetCount, Long segmentCount) {
+
+ this.dbName = dbName;
+ this.tableName = tableName;
+
+ this.dataSize = dataSize;
+ this.totalReplicaDataSize = totalReplicaDataSize;
+
+ this.remoteDataSize = remoteDataSize;
+
+ this.replicaCount = replicaCount;
+
+ this.rowCount = rowCount;
+ this.rowsetCount = rowsetCount;
+ this.segmentCount = segmentCount;
+ }
+ }
+
+ public long getDataSize() {
+ return getDataSize(false);
+ }
+
+ public long getDataSize(boolean singleReplica) {
+ if (singleReplica) {
+ statistics.getDataSize();
+ }
+
+ return statistics.getTotalReplicaDataSize();
+ }
+
+ public long getRemoteDataSize() {
+ return statistics.getRemoteDataSize();
+ }
+
+ public long getReplicaCount() {
+ return statistics.getReplicaCount();
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletStatMgr.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletStatMgr.java
index 435c7fb45d1..b9093dcd838 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletStatMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletStatMgr.java
@@ -104,6 +104,16 @@ public class TabletStatMgr extends MasterDaemon {
continue;
}
OlapTable olapTable = (OlapTable) table;
+
+ Long tableDataSize = 0L;
+ Long tableTotalReplicaDataSize = 0L;
+
+ Long tableRemoteDataSize = 0L;
+
+ Long tableReplicaCount = 0L;
+
+ Long tableRowCount = 0L;
+
// Use try write lock to avoid such cases
// Time1: Thread1 hold read lock for 5min
// Time2: Thread2 want to add write lock, then it will be
the first element in lock queue
@@ -120,7 +130,12 @@ public class TabletStatMgr extends MasterDaemon {
long indexRowCount = 0L;
boolean indexReported = true;
for (Tablet tablet : index.getTablets()) {
- long tabletRowCount = Long.MAX_VALUE;
+
+ Long tabletDataSize = 0L;
+ Long tabletRemoteDataSize = 0L;
+
+ Long tabletRowCount = Long.MAX_VALUE;
+
boolean tabletReported = false;
for (Replica replica : tablet.getReplicas()) {
LOG.debug("Table {} replica {} current
version {}, report version {}",
@@ -146,12 +161,26 @@ public class TabletStatMgr extends MasterDaemon {
}
tabletRowCount = replica.getRowCount();
}
+
+ if (replica.getDataSize() >
tabletDataSize) {
+ tabletDataSize = replica.getDataSize();
+ }
+ tableTotalReplicaDataSize +=
replica.getDataSize();
+
+ if (replica.getRemoteDataSize() >
tabletRemoteDataSize) {
+ tabletRemoteDataSize =
replica.getRemoteDataSize();
+ }
+ tableReplicaCount++;
}
+ tableDataSize += tabletDataSize;
+ tableRemoteDataSize += tabletRemoteDataSize;
+
// When all BEs are down, avoid set
Long.MAX_VALUE to index and table row count. Use 0.
if (tabletRowCount == Long.MAX_VALUE) {
tabletRowCount = 0L;
}
+ tableRowCount += tabletRowCount;
indexRowCount += tabletRowCount;
// Only when all tablets of this index are
reported, we set indexReported to true.
indexReported = indexReported &&
tabletReported;
@@ -163,6 +192,11 @@ public class TabletStatMgr extends MasterDaemon {
indexReported, indexRowCount);
} // end for indices
} // end for partitions
+
+ olapTable.setStatistics(new
OlapTable.Statistics(db.getName(), table.getName(),
+ tableDataSize, tableTotalReplicaDataSize,
+ tableRemoteDataSize, tableReplicaCount,
tableRowCount, 0L, 0L));
+
if (LOG.isDebugEnabled()) {
LOG.debug("finished to set row num for table: {} in
database: {}",
table.getName(), db.getFullName());
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/common/proc/DbsProcDir.java
b/fe/fe-core/src/main/java/org/apache/doris/common/proc/DbsProcDir.java
index a3818099d8e..aa65599cb9f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/DbsProcDir.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/DbsProcDir.java
@@ -113,7 +113,7 @@ public class DbsProcDir implements ProcDirInterface {
String readableQuota = DebugUtil.printByteWithUnit(dataQuota);
String lastCheckTime = (db instanceof Database) ?
TimeUtils.longToTimeString(
((Database) db).getLastCheckTime()) :
FeConstants.null_string;
- long replicaCount = (db instanceof Database) ? ((Database)
db).getReplicaCountWithLock() : 0;
+ long replicaCount = (db instanceof Database) ? ((Database)
db).getReplicaCount() : 0;
long replicaQuota = (db instanceof Database) ? ((Database)
db).getReplicaQuota() : 0;
long transactionQuota = (db instanceof Database) ? ((Database)
db).getTransactionQuotaSize() : 0;
dbInfo.add(readableUsedQuota);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
