This is an automated email from the ASF dual-hosted git repository.
agupta pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new abfa3becfa HDDS-12442. Add latency metrics for OM deletion services
(#7986)
abfa3becfa is described below
commit abfa3becfa1bb64b2225eaca9d06f29559eb3c44
Author: Tejaskriya <[email protected]>
AuthorDate: Tue Mar 4 14:10:48 2025 +0530
HDDS-12442. Add latency metrics for OM deletion services (#7986)
---
.../hadoop/ozone/om/OMPerformanceMetrics.java | 22 ++++++++++++++++++++++
.../om/service/AbstractKeyDeletingService.java | 9 +++++++--
.../ozone/om/service/OpenKeyCleanupService.java | 4 +++-
3 files changed, 32 insertions(+), 3 deletions(-)
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMPerformanceMetrics.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMPerformanceMetrics.java
index 26d82443d6..78d10acd6d 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMPerformanceMetrics.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMPerformanceMetrics.java
@@ -21,6 +21,7 @@
import org.apache.hadoop.metrics2.annotation.Metric;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.metrics2.lib.MutableGaugeFloat;
+import org.apache.hadoop.metrics2.lib.MutableGaugeLong;
import org.apache.hadoop.metrics2.lib.MutableRate;
/**
@@ -151,6 +152,15 @@ public static void unregister() {
@Metric(about = "ACLs check in getObjectTagging")
private MutableRate getObjectTaggingAclCheckLatencyNs;
+ @Metric(about = "Latency of each iteration of DirectoryDeletingService in
ms")
+ private MutableGaugeLong directoryDeletingServiceLatencyMs;
+
+ @Metric(about = "Latency of each iteration of KeyDeletingService in ms")
+ private MutableGaugeLong keyDeletingServiceLatencyMs;
+
+ @Metric(about = "Latency of each iteration of OpenKeyCleanupService in ms")
+ private MutableGaugeLong openKeyCleanupServiceLatencyMs;
+
public void addLookupLatency(long latencyInNs) {
lookupLatencyNs.add(latencyInNs);
}
@@ -299,4 +309,16 @@ public MutableRate getGetObjectTaggingAclCheckLatencyNs() {
public void addGetObjectTaggingLatencyNs(long latencyInNs) {
getObjectTaggingAclCheckLatencyNs.add(latencyInNs);
}
+
+ public void setDirectoryDeletingServiceLatencyMs(long latencyInMs) {
+ directoryDeletingServiceLatencyMs.set(latencyInMs);
+ }
+
+ public void setKeyDeletingServiceLatencyMs(long latencyInMs) {
+ keyDeletingServiceLatencyMs.set(latencyInMs);
+ }
+
+ public void setOpenKeyCleanupServiceLatencyMs(long latencyInMs) {
+ openKeyCleanupServiceLatencyMs.set(latencyInMs);
+ }
}
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/AbstractKeyDeletingService.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/AbstractKeyDeletingService.java
index 7c938c9056..ccf77eb44e 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/AbstractKeyDeletingService.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/AbstractKeyDeletingService.java
@@ -47,6 +47,7 @@
import org.apache.hadoop.ozone.om.KeyManager;
import org.apache.hadoop.ozone.om.OMConfigKeys;
import org.apache.hadoop.ozone.om.OMMetadataManager;
+import org.apache.hadoop.ozone.om.OMPerformanceMetrics;
import org.apache.hadoop.ozone.om.OzoneManager;
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
@@ -73,6 +74,7 @@ public abstract class AbstractKeyDeletingService extends
BackgroundService
private final OzoneManager ozoneManager;
private final DeletingServiceMetrics metrics;
+ private final OMPerformanceMetrics perfMetrics;
private final ScmBlockLocationProtocol scmClient;
private final ClientId clientId = ClientId.randomId();
private final AtomicLong deletedDirsCount;
@@ -94,6 +96,7 @@ public AbstractKeyDeletingService(String serviceName, long
interval,
this.movedFilesCount = new AtomicLong(0);
this.runCount = new AtomicLong(0);
this.metrics = ozoneManager.getDeletionMetrics();
+ this.perfMetrics = ozoneManager.getPerfMetrics();
}
protected int processKeyDeletes(List<BlockGroup> keyBlocksList,
@@ -119,14 +122,15 @@ protected int processKeyDeletes(List<BlockGroup>
keyBlocksList,
LOG.info("{} BlockGroup deletion are acked by SCM in {} ms",
keyBlocksList.size(), Time.monotonicNow() - startTime);
if (blockDeletionResults != null) {
- startTime = Time.monotonicNow();
+ long purgeStartTime = Time.monotonicNow();
delCount = submitPurgeKeysRequest(blockDeletionResults,
keysToModify, snapTableKey, expectedPreviousSnapshotId);
int limit =
ozoneManager.getConfiguration().getInt(OMConfigKeys.OZONE_KEY_DELETING_LIMIT_PER_TASK,
OMConfigKeys.OZONE_KEY_DELETING_LIMIT_PER_TASK_DEFAULT);
LOG.info("Blocks for {} (out of {}) keys are deleted from DB in {} ms.
Limit per task is {}.",
- delCount, blockDeletionResults.size(), Time.monotonicNow() -
startTime, limit);
+ delCount, blockDeletionResults.size(), Time.monotonicNow() -
purgeStartTime, limit);
}
+ perfMetrics.setKeyDeletingServiceLatencyMs(Time.monotonicNow() -
startTime);
return delCount;
}
@@ -416,6 +420,7 @@ public void optimizeDirDeletesAndSubmitRequest(
dirNum, subdirDelNum, subFileNum, (subDirNum - subdirDelNum),
timeTakenInIteration, rnCnt);
metrics.incrementDirectoryDeletionTotalMetrics(dirNum + subdirDelNum,
subDirNum, subFileNum);
+ perfMetrics.setDirectoryDeletingServiceLatencyMs(timeTakenInIteration);
}
}
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/OpenKeyCleanupService.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/OpenKeyCleanupService.java
index 60873f0fde..e10ca3f9a5 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/OpenKeyCleanupService.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/OpenKeyCleanupService.java
@@ -237,9 +237,11 @@ public BackgroundTaskResult call() throws Exception {
});
}
+ long timeTaken = Time.monotonicNow() - startTime;
LOG.info("Number of expired open keys submitted for deletion: {},"
+ " for commit: {}, cleanupLimit: {}, elapsed time: {}ms",
- numOpenKeys, numHsyncKeys, cleanupLimitPerTask, Time.monotonicNow()
- startTime);
+ numOpenKeys, numHsyncKeys, cleanupLimitPerTask, timeTaken);
+
ozoneManager.getPerfMetrics().setOpenKeyCleanupServiceLatencyMs(timeTaken);
final int numKeys = numOpenKeys + numHsyncKeys;
submittedOpenKeyCount.addAndGet(numKeys);
return () -> numKeys;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
