This is an automated email from the ASF dual-hosted git repository.
szetszwo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 5ae810a7d8 HDDS-7855. Add hsync metrics in OM. (#4479)
5ae810a7d8 is described below
commit 5ae810a7d865140c7df7dc65a5674adebbf7a4a3
Author: Tsz-Wo Nicholas Sze <[email protected]>
AuthorDate: Wed Mar 29 07:53:58 2023 +0800
HDDS-7855. Add hsync metrics in OM. (#4479)
---
.../java/org/apache/hadoop/ozone/om/OMMetrics.java | 29 ++++++++++++++++++++++
.../om/request/key/OMKeyCommitRequestWithFSO.java | 4 ++-
.../ozone/om/service/OpenKeyCleanupService.java | 21 ++++++++++++----
.../om/service/TestOpenKeyCleanupService.java | 24 +++++++++++++++---
4 files changed, 69 insertions(+), 9 deletions(-)
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetrics.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetrics.java
index faeaae6885..a93db3d92e 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetrics.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetrics.java
@@ -63,6 +63,7 @@ public class OMMetrics implements OmMetadataReaderMetrics {
private @Metric MutableCounterLong numTrashKeyLists;
private @Metric MutableCounterLong numVolumeLists;
private @Metric MutableCounterLong numKeyCommits;
+ private @Metric MutableCounterLong numKeyHSyncs;
private @Metric MutableCounterLong numBlockAllocations;
private @Metric MutableCounterLong numGetServiceLists;
private @Metric MutableCounterLong numBucketS3Lists;
@@ -81,6 +82,8 @@ public class OMMetrics implements OmMetadataReaderMetrics {
private @Metric MutableCounterLong numOpenKeyDeleteRequests;
private @Metric MutableCounterLong numOpenKeysSubmittedForDeletion;
private @Metric MutableCounterLong numOpenKeysDeleted;
+ private @Metric MutableCounterLong numOpenKeysCleaned;
+ private @Metric MutableCounterLong numOpenKeysHSyncCleaned;
private @Metric MutableCounterLong numAddAcl;
private @Metric MutableCounterLong numSetAcl;
@@ -746,6 +749,11 @@ public class OMMetrics implements OmMetadataReaderMetrics {
numKeyCommits.incr();
}
+ public void incNumKeyHSyncs() {
+ numKeyOps.incr();
+ numKeyHSyncs.incr();
+ }
+
public void incNumKeyCommitFails() {
numKeyCommitFails.incr();
}
@@ -791,6 +799,14 @@ public class OMMetrics implements OmMetadataReaderMetrics {
numOpenKeysDeleted.incr();
}
+ public void incNumOpenKeysCleaned(int delta) {
+ numOpenKeysCleaned.incr(delta);
+ }
+
+ public void incNumOpenKeysHSyncCleaned() {
+ numOpenKeysHSyncCleaned.incr();
+ }
+
public void incNumOpenKeyDeleteRequestFails() {
numOpenKeyDeleteRequestFails.incr();
}
@@ -1028,6 +1044,11 @@ public class OMMetrics implements
OmMetadataReaderMetrics {
return numKeyCommits.value();
}
+ @VisibleForTesting
+ public long getNumKeyHSyncs() {
+ return numKeyHSyncs.value();
+ }
+
@VisibleForTesting
public long getNumKeyCommitFails() {
return numKeyCommitFails.value();
@@ -1091,6 +1112,14 @@ public class OMMetrics implements
OmMetadataReaderMetrics {
return numOpenKeysDeleted.value();
}
+ public long getNumOpenKeysCleaned() {
+ return numOpenKeysCleaned.value();
+ }
+
+ public long getNumOpenKeysHSyncCleaned() {
+ return numOpenKeysHSyncCleaned.value();
+ }
+
public long getNumOpenKeyDeleteRequestFails() {
return numOpenKeyDeleteRequestFails.value();
}
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestWithFSO.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestWithFSO.java
index 909104f99e..4ca4f71652 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestWithFSO.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestWithFSO.java
@@ -97,7 +97,9 @@ public class OMKeyCommitRequestWithFSO extends
OMKeyCommitRequest {
Result result;
boolean isHSync = commitKeyRequest.hasHsync() &&
commitKeyRequest.getHsync();
- if (!isHSync) {
+ if (isHSync) {
+ omMetrics.incNumKeyHSyncs();
+ } else {
omMetrics.incNumKeyCommits();
}
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/OpenKeyCleanupService.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/OpenKeyCleanupService.java
index f4d09875e1..960f49acbc 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/OpenKeyCleanupService.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/OpenKeyCleanupService.java
@@ -35,6 +35,7 @@ import
org.apache.hadoop.ozone.om.ratis.OzoneManagerRatisServer;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CommitKeyRequest;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DeleteOpenKeysRequest;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest;
+import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OpenKeyBucket;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type;
import org.apache.hadoop.util.Time;
@@ -196,7 +197,10 @@ public class OpenKeyCleanupService extends
BackgroundService {
// delete non-hsync'ed keys
final OMRequest omRequest = createDeleteOpenKeysRequest(
openKeyBuckets.stream());
- submitRequest(omRequest);
+ final OMResponse response = submitRequest(omRequest);
+ if (response != null && response.getSuccess()) {
+ ozoneManager.getMetrics().incNumOpenKeysCleaned(numOpenKeys);
+ }
}
final List<CommitKeyRequest.Builder> hsyncKeys
@@ -204,7 +208,12 @@ public class OpenKeyCleanupService extends
BackgroundService {
final int numHsyncKeys = hsyncKeys.size();
if (!hsyncKeys.isEmpty()) {
// commit hsync'ed keys
- hsyncKeys.forEach(b -> submitRequest(createCommitKeyRequest(b)));
+ hsyncKeys.forEach(b -> {
+ final OMResponse response = submitRequest(createCommitKeyRequest(b));
+ if (response != null && response.getSuccess()) {
+ ozoneManager.getMetrics().incNumOpenKeysHSyncCleaned();
+ }
+ });
}
if (LOG.isDebugEnabled()) {
@@ -242,7 +251,7 @@ public class OpenKeyCleanupService extends
BackgroundService {
return omRequest;
}
- private void submitRequest(OMRequest omRequest) {
+ private OMResponse submitRequest(OMRequest omRequest) {
try {
if (isRatisEnabled()) {
OzoneManagerRatisServer server = ozoneManager.getOmRatisServer();
@@ -257,14 +266,16 @@ public class OpenKeyCleanupService extends
BackgroundService {
.setType(RaftClientRequest.writeRequestType())
.build();
- server.submitRequest(omRequest, raftClientRequest);
+ return server.submitRequest(omRequest, raftClientRequest);
} else {
- ozoneManager.getOmServerProtocol().submitRequest(null, omRequest);
+ return ozoneManager.getOmServerProtocol().submitRequest(
+ null, omRequest);
}
} catch (ServiceException e) {
LOG.error("Open key " + omRequest.getCmdType()
+ " request failed. Will retry at next run.", e);
}
+ return null;
}
}
}
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestOpenKeyCleanupService.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestOpenKeyCleanupService.java
index 54939f956b..6140de8910 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestOpenKeyCleanupService.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestOpenKeyCleanupService.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.hdds.utils.db.DBConfigFromFile;
import org.apache.hadoop.ozone.om.ExpiredOpenKeys;
import org.apache.hadoop.ozone.om.KeyManager;
import org.apache.hadoop.ozone.om.OMMetadataManager;
+import org.apache.hadoop.ozone.om.OMMetrics;
import org.apache.hadoop.ozone.om.OmTestManagers;
import org.apache.hadoop.ozone.om.OzoneManager;
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
@@ -130,6 +131,8 @@ public class TestOpenKeyCleanupService {
@Timeout(300)
public void testCleanupExpiredOpenKeys(
int numDEFKeys, int numFSOKeys, boolean hsync) throws Exception {
+ LOG.info("numDEFKeys={}, numFSOKeys={}, hsync? {}",
+ numDEFKeys, numFSOKeys, hsync);
OpenKeyCleanupService openKeyCleanupService =
(OpenKeyCleanupService) keyManager.getOpenKeyCleanupService();
@@ -139,15 +142,21 @@ public class TestOpenKeyCleanupService {
Thread.sleep(SERVICE_INTERVAL.toMillis());
final long oldkeyCount = openKeyCleanupService.getSubmittedOpenKeyCount();
final long oldrunCount = openKeyCleanupService.getRunCount();
+ LOG.info("oldkeyCount={}, oldrunCount={}", oldkeyCount, oldrunCount);
+ assertEquals(0, oldkeyCount);
+ final OMMetrics metrics = om.getMetrics();
+ assertEquals(0, metrics.getNumKeyHSyncs());
+ assertEquals(0, metrics.getNumOpenKeysCleaned());
+ assertEquals(0, metrics.getNumOpenKeysHSyncCleaned());
final int keyCount = numDEFKeys + numFSOKeys;
- createOpenKeys(numDEFKeys, hsync, BucketLayout.DEFAULT);
+ createOpenKeys(numDEFKeys, false, BucketLayout.DEFAULT);
createOpenKeys(numFSOKeys, hsync, BucketLayout.FILE_SYSTEM_OPTIMIZED);
// wait for open keys to expire
Thread.sleep(EXPIRE_THRESHOLD.toMillis());
- assertExpiredOpenKeys(numDEFKeys == 0, hsync, BucketLayout.DEFAULT);
+ assertExpiredOpenKeys(numDEFKeys == 0, false, BucketLayout.DEFAULT);
assertExpiredOpenKeys(numFSOKeys == 0, hsync,
BucketLayout.FILE_SYSTEM_OPTIMIZED);
@@ -164,9 +173,18 @@ public class TestOpenKeyCleanupService {
assertTrue(openKeyCleanupService.getSubmittedOpenKeyCount() >=
oldkeyCount + keyCount);
- assertExpiredOpenKeys(true, hsync, BucketLayout.DEFAULT);
+ assertExpiredOpenKeys(true, false, BucketLayout.DEFAULT);
assertExpiredOpenKeys(true, hsync,
BucketLayout.FILE_SYSTEM_OPTIMIZED);
+ if (hsync) {
+ assertEquals(numDEFKeys, metrics.getNumOpenKeysCleaned());
+ assertTrue(metrics.getNumOpenKeysHSyncCleaned() >= numFSOKeys);
+ assertEquals(numFSOKeys, metrics.getNumKeyHSyncs());
+ } else {
+ assertEquals(keyCount, metrics.getNumOpenKeysCleaned());
+ assertEquals(0, metrics.getNumOpenKeysHSyncCleaned());
+ assertEquals(0, metrics.getNumKeyHSyncs());
+ }
}
void assertExpiredOpenKeys(boolean expectedToEmpty, boolean hsync,
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
