This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 030ce663b7 HDDS-8729. Add metric for count of blocks pending deletion
on datanode (#4800)
030ce663b7 is described below
commit 030ce663b7c2e2dd9c3d4cfde3dda82aef2dd005
Author: XiChen <[email protected]>
AuthorDate: Thu Jun 8 16:50:18 2023 +0800
HDDS-8729. Add metric for count of blocks pending deletion on datanode
(#4800)
---
.../common/helpers/BlockDeletingServiceMetrics.java | 16 +++++++++++++++-
.../statemachine/background/BlockDeletingService.java | 7 +++++++
.../ozone/container/common/TestBlockDeletingService.java | 11 ++++++++++-
3 files changed, 32 insertions(+), 2 deletions(-)
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/BlockDeletingServiceMetrics.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/BlockDeletingServiceMetrics.java
index 15ce6015b4..da42e66a9e 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/BlockDeletingServiceMetrics.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/BlockDeletingServiceMetrics.java
@@ -23,6 +23,7 @@ import org.apache.hadoop.metrics2.annotation.Metric;
import org.apache.hadoop.metrics2.annotation.Metrics;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.metrics2.lib.MutableCounterLong;
+import org.apache.hadoop.metrics2.lib.MutableGaugeLong;
import
org.apache.hadoop.ozone.container.keyvalue.statemachine.background.BlockDeletingService;
/**
@@ -48,6 +49,9 @@ public final class BlockDeletingServiceMetrics {
@Metric(about = "The number of out of order delete block transaction.")
private MutableCounterLong outOfOrderDeleteBlockTransactionCount;
+ @Metric(about = "The total number of blocks pending for processing.")
+ private MutableGaugeLong totalPendingBlockCount;
+
private BlockDeletingServiceMetrics() {
}
@@ -82,6 +86,10 @@ public final class BlockDeletingServiceMetrics {
this.failureCount.incr();
}
+ public void setTotalPendingBlockCount(long count) {
+ this.totalPendingBlockCount.set(count);
+ }
+
public long getSuccessCount() {
return successCount.value();
}
@@ -102,6 +110,10 @@ public final class BlockDeletingServiceMetrics {
return outOfOrderDeleteBlockTransactionCount.value();
}
+ public long getTotalPendingBlockCount() {
+ return totalPendingBlockCount.value();
+ }
+
@Override
public String toString() {
StringBuffer buffer = new StringBuffer();
@@ -109,7 +121,9 @@ public final class BlockDeletingServiceMetrics {
.append("successBytes = " + successBytes.value()).append("\t")
.append("failureCount = " + failureCount.value()).append("\t")
.append("outOfOrderDeleteBlockTransactionCount = "
- + outOfOrderDeleteBlockTransactionCount.value()).append("\t");
+ + outOfOrderDeleteBlockTransactionCount.value()).append("\t")
+ .append("totalPendingBlockCount = "
+ + totalPendingBlockCount.value()).append("\t");
return buffer.toString();
}
}
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/statemachine/background/BlockDeletingService.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/statemachine/background/BlockDeletingService.java
index 6b2bd0f2df..deee5610b3 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/statemachine/background/BlockDeletingService.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/statemachine/background/BlockDeletingService.java
@@ -182,6 +182,13 @@ public class BlockDeletingService extends
BackgroundService {
.filter(e -> isDeletionAllowed(e.getValue().getContainerData(),
deletionPolicy)).collect(Collectors
.toMap(Map.Entry::getKey, e -> e.getValue().getContainerData()));
+
+ long totalPendingBlockCount =
+ containerDataMap.values().stream().mapToLong(
+ containerData -> ((KeyValueContainerData) containerData)
+ .getNumPendingDeletionBlocks())
+ .sum();
+ metrics.setTotalPendingBlockCount(totalPendingBlockCount);
return deletionPolicy
.chooseContainerForBlockDeletion(blockLimit, containerDataMap);
}
diff --git
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java
index f7ca3c6009..09984fef80 100644
---
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java
+++
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java
@@ -613,6 +613,11 @@ public class TestBlockDeletingService {
Assert.assertEquals(2,
deletingServiceMetrics.getSuccessCount()
- deleteSuccessCount);
+ // The value of the getTotalPendingBlockCount Metrics is obtained
+ // before the deletion is processing
+ // So the Pending Block count will be 3
+ Assert.assertEquals(3,
+ deletingServiceMetrics.getTotalPendingBlockCount());
deleteAndWait(svc, 2);
@@ -634,8 +639,12 @@ public class TestBlockDeletingService {
// check if blockData get deleted
assertBlockDataTableRecordCount(0, meta, filter, data.getContainerID());
+ // The value of the getTotalPendingBlockCount Metrics is obtained
+ // before the deletion is processing
+ // So the Pending Block count will be 1
+ Assert.assertEquals(1,
+ deletingServiceMetrics.getTotalPendingBlockCount());
}
-
svc.shutdown();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
