This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new b54ed1006e HDDS-8662. Improve S3G-related metrics (#4753)
b54ed1006e is described below
commit b54ed1006e10bde91abaf899836e0175cc34a393
Author: XiChen <[email protected]>
AuthorDate: Wed May 24 19:46:50 2023 +0800
HDDS-8662. Improve S3G-related metrics (#4753)
---
.../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 18 ++++++--
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 51 +++++++++++++++-------
.../hadoop/ozone/s3/metrics/S3GatewayMetrics.java | 49 +++++++++++++++++++++
.../ozone/s3/metrics/TestS3GatewayMetrics.java | 7 ++-
4 files changed, 103 insertions(+), 22 deletions(-)
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java
index 095b160c8e..9842ccaee0 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java
@@ -249,9 +249,11 @@ public class BucketEndpoint extends EndpointBase {
AUDIT.logReadSuccess(buildAuditMessageForSuccess(s3GAction,
getAuditParameters()));
+ int keyCount =
+ response.getCommonPrefixes().size() + response.getContents().size();
getMetrics().updateGetBucketSuccessStats(startNanos);
- response.setKeyCount(
- response.getCommonPrefixes().size() + response.getContents().size());
+ getMetrics().incListKeyCount(keyCount);
+ response.setKeyCount(keyCount);
return Response.ok(response).build();
}
@@ -418,25 +420,33 @@ public class BucketEndpoint extends EndpointBase {
MultiDeleteResponse result = new MultiDeleteResponse();
if (request.getObjects() != null) {
for (DeleteObject keyToDelete : request.getObjects()) {
+ long startNanos = Time.monotonicNowNanos();
try {
bucket.deleteKey(keyToDelete.getKey());
+ getMetrics().updateDeleteKeySuccessStats(startNanos);
if (!request.isQuiet()) {
result.addDeleted(new DeletedObject(keyToDelete.getKey()));
}
} catch (OMException ex) {
if (isAccessDenied(ex)) {
+ getMetrics().updateDeleteKeyFailureStats(startNanos);
result.addError(
new Error(keyToDelete.getKey(), "PermissionDenied",
ex.getMessage()));
} else if (ex.getResult() != ResultCodes.KEY_NOT_FOUND) {
+ getMetrics().updateDeleteKeyFailureStats(startNanos);
result.addError(
new Error(keyToDelete.getKey(), "InternalError",
ex.getMessage()));
- } else if (!request.isQuiet()) {
- result.addDeleted(new DeletedObject(keyToDelete.getKey()));
+ } else {
+ if (!request.isQuiet()) {
+ result.addDeleted(new DeletedObject(keyToDelete.getKey()));
+ }
+ getMetrics().updateDeleteKeySuccessStats(startNanos);
}
} catch (Exception ex) {
+ getMetrics().updateDeleteKeyFailureStats(startNanos);
result.addError(
new Error(keyToDelete.getKey(), "InternalError",
ex.getMessage()));
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
index d88c61b795..5c02ae2cf7 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
@@ -236,9 +236,9 @@ public class ObjectEndpoint extends EndpointBase {
output = getClientProtocol().createKey(volume.getName(), bucketName,
keyPath, length, replicationConfig, customMetadata);
- IOUtils.copy(body, output);
-
- getMetrics().updateCreateKeySuccessStats(startNanos);
+ getMetrics().updatePutKeyMetadataStats(startNanos);
+ long putLength = IOUtils.copyLarge(body, output);
+ getMetrics().incPutKeySuccessLength(putLength);
return Response.ok().status(HttpStatus.SC_OK)
.build();
} catch (OMException ex) {
@@ -275,12 +275,13 @@ public class ObjectEndpoint extends EndpointBase {
}
throw ex;
} finally {
+ if (output != null) {
+ output.close();
+ }
if (auditSuccess) {
AUDIT.logWriteSuccess(
buildAuditMessageForSuccess(s3GAction, getAuditParameters()));
- }
- if (output != null) {
- output.close();
+ getMetrics().updateCreateKeySuccessStats(startNanos);
}
}
}
@@ -340,8 +341,10 @@ public class ObjectEndpoint extends EndpointBase {
if (rangeHeaderVal == null || rangeHeader.isReadFull()) {
StreamingOutput output = dest -> {
try (OzoneInputStream key = keyDetails.getContent()) {
- IOUtils.copy(key, dest);
+ long readLength = IOUtils.copyLarge(key, dest);
+ getMetrics().incGetKeySuccessLength(readLength);
}
+ getMetrics().updateGetKeySuccessStats(startNanos);
};
responseBuilder = Response
.ok(output)
@@ -357,9 +360,11 @@ public class ObjectEndpoint extends EndpointBase {
StreamingOutput output = dest -> {
try (OzoneInputStream ozoneInputStream = keyDetails.getContent()) {
ozoneInputStream.seek(startOffset);
- IOUtils.copyLarge(ozoneInputStream, dest, 0,
+ long readLength = IOUtils.copyLarge(ozoneInputStream, dest, 0,
copyLength, new byte[bufferSize]);
+ getMetrics().incGetKeySuccessLength(readLength);
}
+ getMetrics().updateGetKeySuccessStats(startNanos);
};
responseBuilder = Response
.status(Status.PARTIAL_CONTENT)
@@ -399,7 +404,7 @@ public class ObjectEndpoint extends EndpointBase {
}
}
addLastModifiedDate(responseBuilder, keyDetails);
- getMetrics().updateGetKeySuccessStats(startNanos);
+ getMetrics().updateGetKeyMetadataStats(startNanos);
return responseBuilder.build();
} catch (OMException ex) {
auditSuccess = false;
@@ -747,8 +752,8 @@ public class ObjectEndpoint extends EndpointBase {
String uploadID, InputStream body)
throws IOException, OS3Exception {
long startNanos = Time.monotonicNowNanos();
+ String copyHeader = null;
try {
- String copyHeader;
OzoneOutputStream ozoneOutputStream = null;
if ("STREAMING-AWS4-HMAC-SHA256-PAYLOAD"
@@ -783,6 +788,7 @@ public class ObjectEndpoint extends EndpointBase {
String range =
headers.getHeaderString(COPY_SOURCE_HEADER_RANGE);
+ long copyLength;
if (range != null) {
RangeHeader rangeHeader =
RangeHeaderParserUtil.parseRangeHeader(range, 0);
@@ -793,15 +799,20 @@ public class ObjectEndpoint extends EndpointBase {
"Bytes to skip: "
+ rangeHeader.getStartOffset() + " actual: " +
skipped);
}
- IOUtils.copyLarge(sourceObject, ozoneOutputStream, 0,
+ getMetrics().updateCopyKeyMetadataStats(startNanos);
+ copyLength = IOUtils.copyLarge(sourceObject, ozoneOutputStream,
0,
rangeHeader.getEndOffset() - rangeHeader.getStartOffset()
+ 1);
} else {
- IOUtils.copy(sourceObject, ozoneOutputStream);
+ getMetrics().updateCopyKeyMetadataStats(startNanos);
+ copyLength = IOUtils.copyLarge(sourceObject, ozoneOutputStream);
}
+ getMetrics().incCopyObjectSuccessLength(copyLength);
}
} else {
- IOUtils.copy(body, ozoneOutputStream);
+ getMetrics().updatePutKeyMetadataStats(startNanos);
+ long putLength = IOUtils.copyLarge(body, ozoneOutputStream);
+ getMetrics().incPutKeySuccessLength(putLength);
}
} finally {
if (ozoneOutputStream != null) {
@@ -814,16 +825,21 @@ public class ObjectEndpoint extends EndpointBase {
ozoneOutputStream.getCommitUploadPartInfo();
String eTag = omMultipartCommitUploadPartInfo.getPartName();
- getMetrics().updateCreateMultipartKeySuccessStats(startNanos);
if (copyHeader != null) {
+ getMetrics().updateCopyObjectSuccessStats(startNanos);
return Response.ok(new CopyPartResult(eTag)).build();
} else {
+ getMetrics().updateCreateMultipartKeySuccessStats(startNanos);
return Response.ok().header("ETag",
eTag).build();
}
} catch (OMException ex) {
- getMetrics().updateCreateMultipartKeyFailureStats(startNanos);
+ if (copyHeader != null) {
+ getMetrics().updateCopyObjectFailureStats(startNanos);
+ } else {
+ getMetrics().updateCreateMultipartKeyFailureStats(startNanos);
+ }
if (ex.getResult() == ResultCodes.NO_SUCH_MULTIPART_UPLOAD_ERROR) {
throw newError(NO_SUCH_UPLOAD, uploadID, ex);
} else if (isAccessDenied(ex)) {
@@ -907,12 +923,14 @@ public class ObjectEndpoint extends EndpointBase {
String destKey, String destBucket,
ReplicationConfig replication,
Map<String, String> metadata) throws IOException {
+ long copyLength;
try (OzoneOutputStream dest =
getClientProtocol().createKey(
volume.getName(), destBucket, destKey, srcKeyLen,
replication, metadata)) {
- IOUtils.copy(src, dest);
+ copyLength = IOUtils.copyLarge(src, dest);
}
+ getMetrics().incCopyObjectSuccessLength(copyLength);
}
private CopyObjectResponse copyObject(OzoneVolume volume,
@@ -961,6 +979,7 @@ public class ObjectEndpoint extends EndpointBase {
try (OzoneInputStream src = getClientProtocol().getKey(volume.getName(),
sourceBucket, sourceKey)) {
+ getMetrics().updateCopyKeyMetadataStats(startNanos);
copy(volume, src, sourceKeyLen, destkey, destBucket, replicationConfig,
sourceKeyDetails.getMetadata());
}
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/metrics/S3GatewayMetrics.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/metrics/S3GatewayMetrics.java
index 4eb2ab2af1..b18b9f3354 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/metrics/S3GatewayMetrics.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/metrics/S3GatewayMetrics.java
@@ -58,6 +58,8 @@ public final class S3GatewayMetrics implements MetricsSource {
private @Metric MutableCounterLong putAclFailure;
private @Metric MutableCounterLong listMultipartUploadsSuccess;
private @Metric MutableCounterLong listMultipartUploadsFailure;
+ private @Metric MutableCounterLong listKeyCount;
+
// RootEndpoint
private @Metric MutableCounterLong listS3BucketsSuccess;
@@ -84,6 +86,9 @@ public final class S3GatewayMetrics implements MetricsSource {
private @Metric MutableCounterLong abortMultipartUploadFailure;
private @Metric MutableCounterLong deleteKeySuccess;
private @Metric MutableCounterLong deleteKeyFailure;
+ private @Metric MutableCounterLong copyObjectSuccessLength;
+ private @Metric MutableCounterLong putKeySuccessLength;
+ private @Metric MutableCounterLong getKeySuccessLength;
// S3 Gateway Latency Metrics
// BucketEndpoint
@@ -226,6 +231,15 @@ public final class S3GatewayMetrics implements
MetricsSource {
@Metric(about = "Latency for failing to delete an S3 object in nanoseconds")
private MutableRate deleteKeyFailureLatencyNs;
+ @Metric(about = "Latency for put metadata of an key in nanoseconds")
+ private MutableRate putKeyMetadataLatencyNs;
+
+ @Metric(about = "Latency for get metadata of an key in nanoseconds")
+ private MutableRate getKeyMetadataLatencyNs;
+
+ @Metric(about = "Latency for copy metadata of an key in nanoseconds")
+ private MutableRate copyKeyMetadataLatencyNs;
+
/**
* Private constructor.
*/
@@ -336,6 +350,13 @@ public final class S3GatewayMetrics implements
MetricsSource {
deleteKeySuccessLatencyNs.snapshot(recordBuilder, true);
deleteKeyFailure.snapshot(recordBuilder, true);
deleteKeyFailureLatencyNs.snapshot(recordBuilder, true);
+ putKeyMetadataLatencyNs.snapshot(recordBuilder, true);
+ getKeyMetadataLatencyNs.snapshot(recordBuilder, true);
+ copyKeyMetadataLatencyNs.snapshot(recordBuilder, true);
+ copyObjectSuccessLength.snapshot(recordBuilder, true);
+ putKeySuccessLength.snapshot(recordBuilder, true);
+ getKeySuccessLength.snapshot(recordBuilder, true);
+ listKeyCount.snapshot(recordBuilder, true);
}
// INC and UPDATE
@@ -396,6 +417,10 @@ public final class S3GatewayMetrics implements
MetricsSource {
putAclFailureLatencyNs.add(Time.monotonicNowNanos() - startNanos);
}
+ public void incListKeyCount(int count) {
+ listKeyCount.incr(count);
+ }
+
public void updateListMultipartUploadsSuccessStats(long startNanos) {
listMultipartUploadsSuccess.incr();
listMultipartUploadsSuccessLatencyNs.add(
@@ -530,6 +555,30 @@ public final class S3GatewayMetrics implements
MetricsSource {
deleteKeyFailureLatencyNs.add(Time.monotonicNowNanos() - startNanos);
}
+ public void updateGetKeyMetadataStats(long startNanos) {
+ getKeyMetadataLatencyNs.add(Time.monotonicNowNanos() - startNanos);
+ }
+
+ public void updateCopyKeyMetadataStats(long startNanos) {
+ copyKeyMetadataLatencyNs.add(Time.monotonicNowNanos() - startNanos);
+ }
+
+ public void updatePutKeyMetadataStats(long startNanos) {
+ putKeyMetadataLatencyNs.add(Time.monotonicNowNanos() - startNanos);
+ }
+
+ public void incCopyObjectSuccessLength(long bytes) {
+ copyObjectSuccessLength.incr(bytes);
+ }
+
+ public void incPutKeySuccessLength(long bytes) {
+ putKeySuccessLength.incr(bytes);
+ }
+
+ public void incGetKeySuccessLength(long bytes) {
+ getKeySuccessLength.incr(bytes);
+ }
+
// GET
public long getListS3BucketsSuccess() {
return listS3BucketsSuccess.value();
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/metrics/TestS3GatewayMetrics.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/metrics/TestS3GatewayMetrics.java
index 28a449a31f..bb1764ac2d 100644
---
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/metrics/TestS3GatewayMetrics.java
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/metrics/TestS3GatewayMetrics.java
@@ -43,8 +43,10 @@ import javax.ws.rs.container.ContainerRequestContext;
import javax.ws.rs.core.HttpHeaders;
import javax.ws.rs.core.MultivaluedHashMap;
import javax.ws.rs.core.Response;
+import javax.ws.rs.core.StreamingOutput;
import javax.ws.rs.core.UriInfo;
import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
@@ -376,8 +378,9 @@ public class TestS3GatewayMetrics {
keyEndpoint.put(bucketName, keyName, CONTENT
.length(), 1, null, body);
// GET the key from the bucket
- keyEndpoint.get(bucketName, keyName, null, 0,
- null);
+ Response response = keyEndpoint.get(bucketName, keyName, null, 0, null);
+ StreamingOutput stream = (StreamingOutput) response.getEntity();
+ stream.write(new ByteArrayOutputStream());
long curMetric = metrics.getGetKeySuccess();
assertEquals(1L, curMetric - oriMetric);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
