This is an automated email from the ASF dual-hosted git repository.
hezhangjian pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new 606db747ea [improve] add metrics: total_entry_log_space_bytes (#4507)
606db747ea is described below
commit 606db747eae9856fed0aeb3f16ef01e7c9254e26
Author: YingQun Zhong <[email protected]>
AuthorDate: Tue Feb 18 09:41:17 2025 +0800
[improve] add metrics: total_entry_log_space_bytes (#4507)
### Motivation
add metric `TOTAL_ENTRY_LOG_SPACE_BYTES` As a supplement to metric
`ACTIVE_ENTRY_LOG_SPACE_BYTES`
This allows us to easily analyze the proportion of valid data in the
cluster entry log.
### Changes
1. add metric `TOTAL_ENTRY_LOG_SPACE_BYTES` in GarbageCollectorStats.
2. Standardize totalEntryLogSize and activeEntryLogSize naming.
Signed-off-by: Zhangjian He <[email protected]>
Co-authored-by: Zhangjian He <[email protected]>
---
.../bookkeeper/bookie/BookKeeperServerStats.java | 1 +
.../bookkeeper/bookie/GarbageCollectorThread.java | 9 +++++++--
.../bookie/stats/GarbageCollectorStats.java | 19 +++++++++++++++++++
3 files changed, 27 insertions(+), 2 deletions(-)
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookKeeperServerStats.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookKeeperServerStats.java
index d9505b44a8..cc1d3a4ad6 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookKeeperServerStats.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookKeeperServerStats.java
@@ -146,6 +146,7 @@ public interface BookKeeperServerStats {
// Compaction/Garbage Collection Related Counters
String ACTIVE_ENTRY_LOG_COUNT = "ACTIVE_ENTRY_LOG_TOTAL";
String ACTIVE_ENTRY_LOG_SPACE_BYTES = "ACTIVE_ENTRY_LOG_SPACE_BYTES";
+ String ENTRY_LOG_SPACE_BYTES = "ENTRY_LOG_SPACE_BYTES";
String RECLAIMED_COMPACTION_SPACE_BYTES =
"RECLAIMED_COMPACTION_SPACE_BYTES";
String RECLAIMED_DELETION_SPACE_BYTES = "RECLAIMED_DELETION_SPACE_BYTES";
String RECLAIM_FAILED_TO_DELETE = "RECLAIM_FAILED_TO_DELETE";
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java
index 25c42e8e4e..f9bdef9d56 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java
@@ -97,6 +97,7 @@ public class GarbageCollectorThread implements Runnable {
// Stats loggers for garbage collection operations
private final GarbageCollectorStats gcStats;
+ private volatile long activeEntryLogSize;
private volatile long totalEntryLogSize;
private volatile int numActiveEntryLogs;
private volatile double entryLogCompactRatio;
@@ -175,6 +176,7 @@ public class GarbageCollectorThread implements Runnable {
this.gcWaitTime = conf.getGcWaitTime();
this.numActiveEntryLogs = 0;
+ this.activeEntryLogSize = 0L;
this.totalEntryLogSize = 0L;
this.entryLogCompactRatio = 0.0;
this.currentEntryLogUsageBuckets = new
int[ENTRY_LOG_USAGE_SEGMENT_COUNT];
@@ -182,6 +184,7 @@ public class GarbageCollectorThread implements Runnable {
this.gcStats = new GarbageCollectorStats(
statsLogger,
() -> numActiveEntryLogs,
+ () -> activeEntryLogSize,
() -> totalEntryLogSize,
() -> garbageCollector.getNumActiveLedgers(),
() -> entryLogCompactRatio,
@@ -524,6 +527,7 @@ public class GarbageCollectorThread implements Runnable {
*/
private void doGcEntryLogs() throws EntryLogMetadataMapException {
// Get a cumulative count, don't update until complete
+ AtomicLong activeEntryLogSizeAcc = new AtomicLong(0L);
AtomicLong totalEntryLogSizeAcc = new AtomicLong(0L);
// Loop through all of the entry logs and remove the non-active
ledgers.
@@ -550,9 +554,10 @@ public class GarbageCollectorThread implements Runnable {
// schedule task
LOG.warn("Failed to remove ledger from entry-log metadata {}",
entryLogId, e);
}
- totalEntryLogSizeAcc.getAndAdd(meta.getRemainingSize());
+ activeEntryLogSizeAcc.getAndAdd(meta.getRemainingSize());
+ totalEntryLogSizeAcc.getAndAdd(meta.getTotalSize());
});
-
+ this.activeEntryLogSize = activeEntryLogSizeAcc.get();
this.totalEntryLogSize = totalEntryLogSizeAcc.get();
this.numActiveEntryLogs = entryLogMetaMap.size();
}
diff --git
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/stats/GarbageCollectorStats.java
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/stats/GarbageCollectorStats.java
index 0b88a5effe..f579036df0 100644
---
a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/stats/GarbageCollectorStats.java
+++
b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/stats/GarbageCollectorStats.java
@@ -27,6 +27,7 @@ import static
org.apache.bookkeeper.bookie.BookKeeperServerStats.CATEGORY_SERVER
import static
org.apache.bookkeeper.bookie.BookKeeperServerStats.COMPACT_RUNTIME;
import static
org.apache.bookkeeper.bookie.BookKeeperServerStats.DELETED_LEDGER_COUNT;
import static
org.apache.bookkeeper.bookie.BookKeeperServerStats.ENTRY_LOG_COMPACT_RATIO;
+import static
org.apache.bookkeeper.bookie.BookKeeperServerStats.ENTRY_LOG_SPACE_BYTES;
import static
org.apache.bookkeeper.bookie.BookKeeperServerStats.EXTRACT_META_RUNTIME;
import static
org.apache.bookkeeper.bookie.BookKeeperServerStats.GC_LEDGER_RUNTIME;
import static
org.apache.bookkeeper.bookie.BookKeeperServerStats.MAJOR_COMPACTION_COUNT;
@@ -101,6 +102,11 @@ public class GarbageCollectorStats {
help = "Current number of active entry log space bytes"
)
private final Gauge<Long> activeEntryLogSpaceBytesGauge;
+ @StatsDoc(
+ name = ENTRY_LOG_SPACE_BYTES,
+ help = "Current number of total entry log space bytes"
+ )
+ private final Gauge<Long> entryLogSpaceBytesGauge;
@StatsDoc(
name = ACTIVE_LEDGER_COUNT,
help = "Current number of active ledgers"
@@ -133,6 +139,7 @@ public class GarbageCollectorStats {
public GarbageCollectorStats(StatsLogger statsLogger,
Supplier<Integer> activeEntryLogCountSupplier,
Supplier<Long>
activeEntryLogSpaceBytesSupplier,
+ Supplier<Long> entryLogSpaceBytesSupplier,
Supplier<Integer> activeLedgerCountSupplier,
Supplier<Double> entryLogCompactRatioSupplier,
Supplier<int[]> usageBuckets) {
@@ -174,6 +181,18 @@ public class GarbageCollectorStats {
}
};
statsLogger.registerGauge(ACTIVE_ENTRY_LOG_SPACE_BYTES,
activeEntryLogSpaceBytesGauge);
+ this.entryLogSpaceBytesGauge = new Gauge<Long>() {
+ @Override
+ public Long getDefaultValue() {
+ return 0L;
+ }
+
+ @Override
+ public Long getSample() {
+ return entryLogSpaceBytesSupplier.get();
+ }
+ };
+ statsLogger.registerGauge(ENTRY_LOG_SPACE_BYTES,
entryLogSpaceBytesGauge);
this.activeLedgerCountGauge = new Gauge<Integer>() {
@Override
public Integer getDefaultValue() {
