This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.0 by this push:
new 47e68dea47c branch-4.0: pick #59776, #61318 (#61930)
47e68dea47c is described below
commit 47e68dea47cd0fbff9ef47f37b235c3dcc8502b1
Author: meiyi <[email protected]>
AuthorDate: Tue Mar 31 18:06:02 2026 +0800
branch-4.0: pick #59776, #61318 (#61930)
pick:
1. modify CloudTabletStatMgr to reduce memory
(https://github.com/apache/doris/pull/59776)
2. modify CloudTabletRebalancer and CloudTabletStatMgr to reduce memory
(https://github.com/apache/doris/pull/61318)
---
.../apache/doris/catalog/CloudTabletStatMgr.java | 171 ++++++++++++---------
.../java/org/apache/doris/catalog/OlapTable.java | 32 ++--
.../org/apache/doris/catalog/TabletStatMgr.java | 34 ++--
.../doris/cloud/catalog/CloudTabletRebalancer.java | 12 +-
.../doris/metric/PrometheusMetricVisitor.java | 28 +---
5 files changed, 141 insertions(+), 136 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/CloudTabletStatMgr.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/CloudTabletStatMgr.java
index 8b519c59867..fd500fac1a0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/CloudTabletStatMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/CloudTabletStatMgr.java
@@ -35,9 +35,9 @@ import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.ArrayList;
-import java.util.HashMap;
+import java.util.Comparator;
import java.util.List;
-import java.util.Map;
+import java.util.PriorityQueue;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@@ -50,8 +50,9 @@ import java.util.concurrent.Future;
public class CloudTabletStatMgr extends MasterDaemon {
private static final Logger LOG =
LogManager.getLogger(CloudTabletStatMgr.class);
- // <(dbId, tableId) -> OlapTable.Statistics>
- private volatile Map<Pair<Long, Long>, OlapTable.Statistics>
cloudTableStatsMap = new HashMap<>();
+ private volatile long totalTableSize = 0;
+ // keep Config.prom_output_table_metrics_limit tables with the largest
data size, used for prometheus output
+ private volatile List<OlapTable.Statistics> cloudTableStatsList = new
ArrayList<>();
private static final ExecutorService GET_TABLET_STATS_THREAD_POOL =
Executors.newFixedThreadPool(
Config.max_get_tablet_stat_task_threads_num);
@@ -63,9 +64,13 @@ public class CloudTabletStatMgr extends MasterDaemon {
@Override
protected void runAfterCatalogReady() {
LOG.info("cloud tablet stat begin");
- long start = System.currentTimeMillis();
+ List<Long> dbIds = getAllTabletStats();
+ updateStatInfo(dbIds);
+ }
- List<GetTabletStatsRequest> reqList = new
ArrayList<GetTabletStatsRequest>();
+ private List<Long> getAllTabletStats() {
+ long start = System.currentTimeMillis();
+ List<Future<Void>> futures = new ArrayList<>();
GetTabletStatsRequest.Builder builder =
GetTabletStatsRequest.newBuilder().setRequestIp(FrontendOptions.getLocalHostAddressCached());
List<Long> dbIds = Env.getCurrentInternalCatalog().getDbIds();
@@ -87,17 +92,16 @@ public class CloudTabletStatMgr extends MasterDaemon {
for (Partition partition : tbl.getAllPartitions()) {
for (MaterializedIndex index :
partition.getMaterializedIndices(IndexExtState.VISIBLE)) {
for (Long tabletId : index.getTabletIdsInOrder()) {
- Tablet tablet = index.getTablet(tabletId);
TabletIndexPB.Builder tabletBuilder =
TabletIndexPB.newBuilder();
tabletBuilder.setDbId(dbId);
tabletBuilder.setTableId(table.getId());
tabletBuilder.setIndexId(index.getId());
tabletBuilder.setPartitionId(partition.getId());
- tabletBuilder.setTabletId(tablet.getId());
+ tabletBuilder.setTabletId(tabletId);
builder.addTabletIdx(tabletBuilder);
if (builder.getTabletIdxCount() >=
Config.get_tablet_stat_batch_size) {
- reqList.add(builder.build());
+
futures.add(submitGetTabletStatsTask(builder.build()));
builder =
GetTabletStatsRequest.newBuilder()
.setRequestIp(FrontendOptions.getLocalHostAddressCached());
}
@@ -111,32 +115,7 @@ public class CloudTabletStatMgr extends MasterDaemon {
} // end for dbs
if (builder.getTabletIdxCount() > 0) {
- reqList.add(builder.build());
- }
-
- List<Future<Void>> futures = new ArrayList<>();
- for (GetTabletStatsRequest req : reqList) {
- futures.add(GET_TABLET_STATS_THREAD_POOL.submit(() -> {
- GetTabletStatsResponse resp =
GetTabletStatsResponse.newBuilder().build();
- try {
- resp = getTabletStats(req);
- } catch (RpcException e) {
- LOG.warn("get tablet stats exception:", e);
- }
- if (resp.getStatus().getCode() != MetaServiceCode.OK) {
- LOG.warn("get tablet stats return failed.");
- }
- if (LOG.isDebugEnabled()) {
- int i = 0;
- for (TabletIndexPB idx : req.getTabletIdxList()) {
- LOG.debug("db_id: {} table_id: {} index_id: {}
tablet_id: {} size: {}",
- idx.getDbId(), idx.getTableId(),
idx.getIndexId(),
- idx.getTabletId(),
resp.getTabletStats(i++).getDataSize());
- }
- }
- updateTabletStat(resp);
- return null;
- }));
+ futures.add(submitGetTabletStatsTask(builder.build()));
}
try {
@@ -149,20 +128,49 @@ public class CloudTabletStatMgr extends MasterDaemon {
LOG.info("finished to get tablet stat of all backends. cost: {} ms",
(System.currentTimeMillis() - start));
+ return dbIds;
+ }
+ private Future<Void> submitGetTabletStatsTask(GetTabletStatsRequest req) {
+ return GET_TABLET_STATS_THREAD_POOL.submit(() -> {
+ GetTabletStatsResponse resp;
+ try {
+ resp = getTabletStats(req);
+ } catch (RpcException e) {
+ LOG.warn("get tablet stats exception:", e);
+ return null;
+ }
+ if (resp.getStatus().getCode() != MetaServiceCode.OK) {
+ LOG.warn("get tablet stats return failed.");
+ return null;
+ }
+ if (LOG.isDebugEnabled()) {
+ int i = 0;
+ for (TabletIndexPB idx : req.getTabletIdxList()) {
+ LOG.debug("db_id: {} table_id: {} index_id: {} tablet_id:
{} size: {}",
+ idx.getDbId(), idx.getTableId(), idx.getIndexId(),
+ idx.getTabletId(),
resp.getTabletStats(i++).getDataSize());
+ }
+ }
+ updateTabletStat(resp);
+ return null;
+ });
+ }
+
+ private void updateStatInfo(List<Long> dbIds) {
// after update replica in all backends, update index row num
- start = System.currentTimeMillis();
+ long start = System.currentTimeMillis();
Pair<String, Long> maxTabletSize = Pair.of(/* tablet id= */null, /*
byte size= */0L);
Pair<String, Long> maxPartitionSize = Pair.of(/* partition id= */null,
/* byte size= */0L);
Pair<String, Long> maxTableSize = Pair.of(/* table id= */null, /* byte
size= */0L);
Pair<String, Long> minTabletSize = Pair.of(/* tablet id= */null, /*
byte size= */Long.MAX_VALUE);
Pair<String, Long> minPartitionSize = Pair.of(/* partition id= */null,
/* byte size= */Long.MAX_VALUE);
Pair<String, Long> minTableSize = Pair.of(/* tablet id= */null, /*
byte size= */Long.MAX_VALUE);
- Long totalTableSize = 0L;
- Long tabletCount = 0L;
- Long partitionCount = 0L;
- Long tableCount = 0L;
- Map<Pair<Long, Long>, OlapTable.Statistics> newCloudTableStatsMap =
new HashMap<>();
+ long totalTableSize = 0L;
+ long tabletCount = 0L;
+ long partitionCount = 0L;
+ long tableCount = 0L;
+ List<OlapTable.Statistics> newCloudTableStatsList = new ArrayList<>();
for (Long dbId : dbIds) {
Database db = Env.getCurrentInternalCatalog().getDbNullable(dbId);
if (db == null) {
@@ -176,25 +184,26 @@ public class CloudTabletStatMgr extends MasterDaemon {
tableCount++;
OlapTable olapTable = (OlapTable) table;
- Long tableDataSize = 0L;
- Long tableTotalReplicaDataSize = 0L;
- Long tableTotalLocalIndexSize = 0L;
- Long tableTotalLocalSegmentSize = 0L;
+ long tableDataSize = 0L;
+ long tableTotalReplicaDataSize = 0L;
+ long tableTotalLocalIndexSize = 0L;
+ long tableTotalLocalSegmentSize = 0L;
- Long tableReplicaCount = 0L;
+ long tableReplicaCount = 0L;
- Long tableRowCount = 0L;
- Long tableRowsetCount = 0L;
- Long tableSegmentCount = 0L;
+ long tableRowCount = 0L;
+ long tableRowsetCount = 0L;
+ long tableSegmentCount = 0L;
if (!table.readLockIfExist()) {
continue;
}
+ OlapTable.Statistics tableStats;
try {
List<Partition> allPartitions =
olapTable.getAllPartitions();
partitionCount += allPartitions.size();
for (Partition partition : allPartitions) {
- Long partitionDataSize = 0L;
+ long partitionDataSize = 0L;
for (MaterializedIndex index :
partition.getMaterializedIndices(IndexExtState.VISIBLE)) {
long indexRowCount = 0L;
List<Tablet> tablets = index.getTablets();
@@ -271,22 +280,22 @@ public class CloudTabletStatMgr extends MasterDaemon {
}
// this is only one thread to update table statistics,
readLock is enough
- olapTable.setStatistics(new
OlapTable.Statistics(db.getName(),
+ tableStats = new OlapTable.Statistics(db.getName(),
table.getName(), tableDataSize,
tableTotalReplicaDataSize, 0L,
tableReplicaCount, tableRowCount,
tableRowsetCount, tableSegmentCount,
- tableTotalLocalIndexSize,
tableTotalLocalSegmentSize, 0L, 0L));
+ tableTotalLocalIndexSize,
tableTotalLocalSegmentSize, 0L, 0L);
+ olapTable.setStatistics(tableStats);
LOG.debug("finished to set row num for table: {} in
database: {}",
table.getName(), db.getFullName());
} finally {
table.readUnlock();
}
totalTableSize += tableDataSize;
- newCloudTableStatsMap.put(Pair.of(dbId, table.getId()), new
OlapTable.Statistics(db.getName(),
- table.getName(), tableDataSize,
tableTotalReplicaDataSize, 0L,
- tableReplicaCount, tableRowCount, tableRowsetCount,
tableSegmentCount, 0L, 0L, 0L, 0L));
+ newCloudTableStatsList.add(tableStats);
}
}
- this.cloudTableStatsMap = newCloudTableStatsMap;
+ filterTopTableStatsByDataSize(newCloudTableStatsList);
+ this.totalTableSize = totalTableSize;
if (MetricRepo.isInit) {
MetricRepo.GAUGE_MAX_TABLE_SIZE_BYTES.setValue(maxTableSize.second);
@@ -332,19 +341,17 @@ public class CloudTabletStatMgr extends MasterDaemon {
private void updateTabletStat(GetTabletStatsResponse response) {
TabletInvertedIndex invertedIndex = Env.getCurrentInvertedIndex();
for (TabletStatsPB stat : response.getTabletStatsList()) {
- if (invertedIndex.getTabletMeta(stat.getIdx().getTabletId()) !=
null) {
- List<Replica> replicas =
invertedIndex.getReplicasByTabletId(stat.getIdx().getTabletId());
- if (replicas == null || replicas.isEmpty() || replicas.get(0)
== null) {
- continue;
- }
- Replica replica = replicas.get(0);
- replica.setDataSize(stat.getDataSize());
- replica.setRowsetCount(stat.getNumRowsets());
- replica.setSegmentCount(stat.getNumSegments());
- replica.setRowCount(stat.getNumRows());
- replica.setLocalInvertedIndexSize(stat.getIndexSize());
- replica.setLocalSegmentSize(stat.getSegmentSize());
+ List<Replica> replicas =
invertedIndex.getReplicasByTabletId(stat.getIdx().getTabletId());
+ if (replicas == null || replicas.isEmpty() || replicas.get(0) ==
null) {
+ continue;
}
+ Replica replica = replicas.get(0);
+ replica.setDataSize(stat.getDataSize());
+ replica.setRowsetCount(stat.getNumRowsets());
+ replica.setSegmentCount(stat.getNumSegments());
+ replica.setRowCount(stat.getNumRows());
+ replica.setLocalInvertedIndexSize(stat.getIndexSize());
+ replica.setLocalSegmentSize(stat.getSegmentSize());
}
}
@@ -360,7 +367,31 @@ public class CloudTabletStatMgr extends MasterDaemon {
return response;
}
- public Map<Pair<Long, Long>, OlapTable.Statistics> getCloudTableStatsMap()
{
- return this.cloudTableStatsMap;
+ public List<OlapTable.Statistics> getCloudTableStats() {
+ return this.cloudTableStatsList;
+ }
+
+ public long getTotalTableSize() {
+ return this.totalTableSize;
+ }
+
+ private void filterTopTableStatsByDataSize(List<OlapTable.Statistics>
newCloudTableStatsList) {
+ int limit = Config.prom_output_table_metrics_limit;
+ if (limit <= 0 || newCloudTableStatsList.size() <= limit) {
+ this.cloudTableStatsList = newCloudTableStatsList;
+ return;
+ }
+ // only copy elements if number of tables >
prom_output_table_metrics_limit
+ PriorityQueue<OlapTable.Statistics> topStats = new
PriorityQueue<>(limit,
+ Comparator.comparingLong(OlapTable.Statistics::getDataSize));
+ for (OlapTable.Statistics stats : newCloudTableStatsList) {
+ if (topStats.size() < limit) {
+ topStats.offer(stats);
+ } else if (!topStats.isEmpty() && stats.getDataSize() >
topStats.peek().getDataSize()) {
+ topStats.poll();
+ topStats.offer(stats);
+ }
+ }
+ this.cloudTableStatsList = new ArrayList<>(topStats);
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
index fd6ec6e7e9f..020173cd44d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
@@ -3647,36 +3647,36 @@ public class OlapTable extends Table implements
MTMVRelatedTableIf, GsonPostProc
private String tableName;
@Getter
- private Long dataSize; // single replica data size
+ private long dataSize; // single replica data size
@Getter
- private Long totalReplicaDataSize;
+ private long totalReplicaDataSize;
@Getter
- private Long remoteDataSize; // single replica remote data size
+ private long remoteDataSize; // single replica remote data size
@Getter
- private Long replicaCount;
+ private long replicaCount;
@Getter
- private Long rowCount;
+ private long rowCount;
@Getter
- private Long rowsetCount;
+ private long rowsetCount;
@Getter
- private Long segmentCount;
+ private long segmentCount;
@Getter
- private Long localInvertedIndexSize; // multi replicas
+ private long localInvertedIndexSize; // multi replicas
@Getter
- private Long localSegmentSize; // multi replicas
+ private long localSegmentSize; // multi replicas
@Getter
- private Long remoteInvertedIndexSize; // single replica
+ private long remoteInvertedIndexSize; // single replica
@Getter
- private Long remoteSegmentSize; // single replica
+ private long remoteSegmentSize; // single replica
public Statistics() {
this.dbName = null;
@@ -3699,11 +3699,11 @@ public class OlapTable extends Table implements
MTMVRelatedTableIf, GsonPostProc
}
public Statistics(String dbName, String tableName,
- Long dataSize, Long totalReplicaDataSize,
- Long remoteDataSize, Long replicaCount, Long rowCount,
- Long rowsetCount, Long segmentCount,
- Long localInvertedIndexSize, Long localSegmentSize,
- Long remoteInvertedIndexSize, Long remoteSegmentSize) {
+ long dataSize, long totalReplicaDataSize,
+ long remoteDataSize, long replicaCount, long rowCount,
+ long rowsetCount, long segmentCount,
+ long localInvertedIndexSize, long localSegmentSize,
+ long remoteInvertedIndexSize, long remoteSegmentSize) {
this.dbName = dbName;
this.tableName = tableName;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletStatMgr.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletStatMgr.java
index e56d9b7a661..37b198652be 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletStatMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletStatMgr.java
@@ -124,10 +124,10 @@ public class TabletStatMgr extends MasterDaemon {
Pair<String, Long> minTabletSize = Pair.of(/* tablet id= */null, /*
byte size= */Long.MAX_VALUE);
Pair<String, Long> minPartitionSize = Pair.of(/* partition id= */null,
/* byte size= */Long.MAX_VALUE);
Pair<String, Long> minTableSize = Pair.of(/* tablet id= */null, /*
byte size= */Long.MAX_VALUE);
- Long totalTableSize = 0L;
- Long tabletCount = 0L;
- Long partitionCount = 0L;
- Long tableCount = 0L;
+ long totalTableSize = 0L;
+ long tabletCount = 0L;
+ long partitionCount = 0L;
+ long tableCount = 0L;
List<Long> dbIds = Env.getCurrentInternalCatalog().getDbIds();
for (Long dbId : dbIds) {
Database db = Env.getCurrentInternalCatalog().getDbNullable(dbId);
@@ -143,19 +143,19 @@ public class TabletStatMgr extends MasterDaemon {
tableCount++;
OlapTable olapTable = (OlapTable) table;
- Long tableDataSize = 0L;
- Long tableTotalReplicaDataSize = 0L;
+ long tableDataSize = 0L;
+ long tableTotalReplicaDataSize = 0L;
- Long tableTotalLocalIndexSize = 0L;
- Long tableTotalLocalSegmentSize = 0L;
- Long tableTotalRemoteIndexSize = 0L;
- Long tableTotalRemoteSegmentSize = 0L;
+ long tableTotalLocalIndexSize = 0L;
+ long tableTotalLocalSegmentSize = 0L;
+ long tableTotalRemoteIndexSize = 0L;
+ long tableTotalRemoteSegmentSize = 0L;
- Long tableRemoteDataSize = 0L;
+ long tableRemoteDataSize = 0L;
- Long tableReplicaCount = 0L;
+ long tableReplicaCount = 0L;
- Long tableRowCount = 0L;
+ long tableRowCount = 0L;
if (!table.readLockIfExist()) {
continue;
@@ -164,7 +164,7 @@ public class TabletStatMgr extends MasterDaemon {
List<Partition> allPartitions =
olapTable.getAllPartitions();
partitionCount += allPartitions.size();
for (Partition partition : allPartitions) {
- Long partitionDataSize = 0L;
+ long partitionDataSize = 0L;
long version = partition.getVisibleVersion();
for (MaterializedIndex index :
partition.getMaterializedIndices(IndexExtState.VISIBLE)) {
long indexRowCount = 0L;
@@ -173,10 +173,10 @@ public class TabletStatMgr extends MasterDaemon {
tabletCount += tablets.size();
for (Tablet tablet : tablets) {
- Long tabletDataSize = 0L;
- Long tabletRemoteDataSize = 0L;
+ long tabletDataSize = 0L;
+ long tabletRemoteDataSize = 0L;
- Long tabletRowCount = Long.MAX_VALUE;
+ long tabletRowCount = Long.MAX_VALUE;
boolean tabletReported = false;
for (Replica replica : tablet.getReplicas()) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudTabletRebalancer.java
b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudTabletRebalancer.java
index 939fd37ad48..aea27e43023 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudTabletRebalancer.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudTabletRebalancer.java
@@ -240,10 +240,10 @@ public class CloudTabletRebalancer extends MasterDaemon {
@Getter
private class InfightTablet {
- private final Long tabletId;
+ private final long tabletId;
private final String clusterId;
- public InfightTablet(Long tabletId, String clusterId) {
+ public InfightTablet(long tabletId, String clusterId) {
this.tabletId = tabletId;
this.clusterId = clusterId;
}
@@ -257,7 +257,7 @@ public class CloudTabletRebalancer extends MasterDaemon {
return false;
}
InfightTablet that = (InfightTablet) o;
- return tabletId.equals(that.tabletId) &&
clusterId.equals(that.clusterId);
+ return tabletId == that.tabletId &&
clusterId.equals(that.clusterId);
}
@Override
@@ -270,7 +270,6 @@ public class CloudTabletRebalancer extends MasterDaemon {
public Tablet pickedTablet;
public long srcBe;
public long destBe;
- public Map<Long, Set<Tablet>> beToTablets;
public long startTimestamp;
BalanceType balanceType;
}
@@ -1511,7 +1510,7 @@ public class CloudTabletRebalancer extends MasterDaemon {
futurePartitionToTablets, futureBeToTabletsInTable)) {
continue;
}
- preheatAndUpdateTablet(pickedTablet, srcBe, destBe, clusterId,
balanceType, beToTablets);
+ preheatAndUpdateTablet(pickedTablet, srcBe, destBe, clusterId,
balanceType);
}
}
}
@@ -1552,7 +1551,7 @@ public class CloudTabletRebalancer extends MasterDaemon {
}
private void preheatAndUpdateTablet(Tablet pickedTablet, long srcBe, long
destBe, String clusterId,
- BalanceType balanceType, Map<Long,
Set<Tablet>> beToTablets) {
+ BalanceType balanceType) {
Backend srcBackend = cloudSystemInfoService.getBackend(srcBe);
Backend destBackend = cloudSystemInfoService.getBackend(destBe);
if (srcBackend == null || destBackend == null) {
@@ -1566,7 +1565,6 @@ public class CloudTabletRebalancer extends MasterDaemon {
task.srcBe = srcBe;
task.destBe = destBe;
task.balanceType = balanceType;
- task.beToTablets = beToTablets;
task.startTimestamp = System.currentTimeMillis() / 1000;
InfightTablet key = new InfightTablet(pickedTablet.getId(), clusterId);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/metric/PrometheusMetricVisitor.java
b/fe/fe-core/src/main/java/org/apache/doris/metric/PrometheusMetricVisitor.java
index f25c63339d3..2103dcb6c21 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/metric/PrometheusMetricVisitor.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/metric/PrometheusMetricVisitor.java
@@ -35,12 +35,10 @@ import org.apache.logging.log4j.Logger;
import java.util.ArrayList;
import java.util.Collection;
-import java.util.Comparator;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
-import java.util.PriorityQueue;
import java.util.Set;
import java.util.stream.Collectors;
@@ -251,31 +249,9 @@ public class PrometheusMetricVisitor extends MetricVisitor
{
StringBuilder segmentCountBuilder = new StringBuilder();
StringBuilder tableRowCountBuilder = new StringBuilder();
- Collection<OlapTable.Statistics> values =
tabletStatMgr.getCloudTableStatsMap().values();
- // calc totalTableSize
- long totalTableSize = 0;
+ Collection<OlapTable.Statistics> values =
tabletStatMgr.getCloudTableStats();
+ long totalTableSize = tabletStatMgr.getTotalTableSize();
for (OlapTable.Statistics stats : values) {
- totalTableSize += stats.getDataSize();
- }
- // output top N metrics
- if (values.size() > Config.prom_output_table_metrics_limit) {
- // only copy elements if number of tables >
prom_output_table_metrics_limit
- PriorityQueue<OlapTable.Statistics> topStats = new PriorityQueue<>(
- Config.prom_output_table_metrics_limit,
-
Comparator.comparingLong(OlapTable.Statistics::getDataSize));
- for (OlapTable.Statistics stats : values) {
- if (topStats.size() < Config.prom_output_table_metrics_limit) {
- topStats.offer(stats);
- } else if (!topStats.isEmpty()
- && stats.getDataSize() >
topStats.peek().getDataSize()) {
- topStats.poll();
- topStats.offer(stats);
- }
- }
- values = topStats;
- }
- for (OlapTable.Statistics stats : values) {
-
dataSizeBuilder.append("doris_fe_table_data_size{db_name=\"");
dataSizeBuilder.append(stats.getDbName());
dataSizeBuilder.append("\", table_name=\"");
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
