This is an automated email from the ASF dual-hosted git repository.
ivandika pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 5ab59c952b5 HDDS-13737. S3 ETag JSON should be quoted (#9248)
5ab59c952b5 is described below
commit 5ab59c952b58890e7310c59c426c6e54a2b505fe
Author: Eric C. Ho <[email protected]>
AuthorDate: Thu Nov 6 09:19:27 2025 +0800
HDDS-13737. S3 ETag JSON should be quoted (#9248)
---
.../dist/src/main/smoketest/s3/mpu_lib.robot | 1 +
.../ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java | 3 +-
.../ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java | 9 +++--
.../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 3 +-
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 45 +++++++++-------------
.../ozone/s3/endpoint/ObjectEndpointStreaming.java | 5 ++-
.../org/apache/hadoop/ozone/s3/util/S3Utils.java | 20 ++++++++++
7 files changed, 52 insertions(+), 34 deletions(-)
diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/mpu_lib.robot
b/hadoop-ozone/dist/src/main/smoketest/s3/mpu_lib.robot
index 0aaa0affec1..fed0c539a07 100644
--- a/hadoop-ozone/dist/src/main/smoketest/s3/mpu_lib.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/s3/mpu_lib.robot
@@ -42,6 +42,7 @@ Upload MPU part
IF '${expected_rc}' == '0'
Should contain ${result} ETag
${etag} = Execute echo '${result}' | jq -r '.ETag'
+ ${etag} = Replace String ${etag} \" ${EMPTY}
${md5sum} = Execute md5sum ${file} | awk '{print $1}'
Should Be Equal As Strings ${etag} ${md5sum}
RETURN ${etag}
diff --git
a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java
b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java
index 00b48262993..016ab60537f 100644
---
a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java
+++
b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java
@@ -21,6 +21,7 @@
import static org.apache.hadoop.ozone.s3.awssdk.S3SDKTestUtils.calculateDigest;
import static org.apache.hadoop.ozone.s3.awssdk.S3SDKTestUtils.createFile;
import static
org.apache.hadoop.ozone.s3.util.S3Consts.CUSTOM_METADATA_HEADER_PREFIX;
+import static org.apache.hadoop.ozone.s3.util.S3Utils.stripQuotes;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
@@ -1231,7 +1232,7 @@ private void completeMPU(String keyName, String uploadId,
List<PartETag> complet
for (PartETag part : completedParts) {
completionXml.append(" <Part>\n");
completionXml.append("
<PartNumber>").append(part.getPartNumber()).append("</PartNumber>\n");
- completionXml.append("
<ETag>").append(part.getETag()).append("</ETag>\n");
+ completionXml.append("
<ETag>").append(stripQuotes(part.getETag())).append("</ETag>\n");
completionXml.append(" </Part>\n");
}
completionXml.append("</CompleteMultipartUpload>");
diff --git
a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java
b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java
index 925e2e75df5..119849281ac 100644
---
a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java
+++
b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java
@@ -20,6 +20,7 @@
import static org.apache.hadoop.ozone.OzoneConsts.MB;
import static org.apache.hadoop.ozone.s3.awssdk.S3SDKTestUtils.calculateDigest;
import static org.apache.hadoop.ozone.s3.awssdk.S3SDKTestUtils.createFile;
+import static org.apache.hadoop.ozone.s3.util.S3Utils.stripQuotes;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -970,7 +971,7 @@ private String
buildCompleteMultipartUploadXml(List<CompletedPart> parts) {
for (CompletedPart part : parts) {
xml.append(" <Part>\n");
xml.append("
<PartNumber>").append(part.partNumber()).append("</PartNumber>\n");
- xml.append(" <ETag>").append(part.eTag()).append("</ETag>\n");
+ xml.append("
<ETag>").append(stripQuotes(part.eTag())).append("</ETag>\n");
xml.append(" </Part>\n");
}
xml.append("</CompleteMultipartUpload>");
@@ -1142,11 +1143,11 @@ private List<CompletedPart> uploadParts(String
bucketName, String key, String up
RequestBody.fromByteBuffer(bb));
assertEquals(DatatypeConverter.printHexBinary(
- calculateDigest(fileInputStream, 0, partSize)).toLowerCase(),
partResponse.eTag());
+ calculateDigest(fileInputStream, 0, partSize)).toLowerCase(),
stripQuotes(partResponse.eTag()));
CompletedPart part = CompletedPart.builder()
.partNumber(partNumber)
- .eTag(partResponse.eTag())
+ .eTag(stripQuotes(partResponse.eTag()))
.build();
completedParts.add(part);
@@ -1643,7 +1644,7 @@ public void testCompleteMultipartUpload() {
CompletedMultipartUpload completedUpload =
CompletedMultipartUpload.builder()
.parts(
-
CompletedPart.builder().partNumber(1).eTag(uploadPartResponse.eTag()).build()
+
CompletedPart.builder().partNumber(1).eTag(stripQuotes(uploadPartResponse.eTag())).build()
).build();
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java
index 066b31fb7d1..c808f0cce76 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java
@@ -28,6 +28,7 @@
import static
org.apache.hadoop.ozone.s3.exception.S3ErrorTable.NOT_IMPLEMENTED;
import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.newError;
import static org.apache.hadoop.ozone.s3.util.S3Consts.ENCODING_TYPE;
+import static org.apache.hadoop.ozone.s3.util.S3Utils.wrapInQuotes;
import com.google.common.annotations.VisibleForTesting;
import java.io.IOException;
@@ -763,7 +764,7 @@ private void addKey(ListObjectResponse response, OzoneKey
next) {
keyMetadata.setSize(next.getDataSize());
String eTag = next.getMetadata().get(ETAG);
if (eTag != null) {
- keyMetadata.setETag(ObjectEndpoint.wrapInQuotes(eTag));
+ keyMetadata.setETag(wrapInQuotes(eTag));
}
keyMetadata.setStorageClass(S3StorageType.fromReplicationConfig(
next.getReplicationConfig()).toString());
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
index 7b8f8e99b49..b495ea346dc 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
@@ -58,9 +58,11 @@
import static org.apache.hadoop.ozone.s3.util.S3Consts.TAG_DIRECTIVE_HEADER;
import static org.apache.hadoop.ozone.s3.util.S3Utils.hasMultiChunksPayload;
import static org.apache.hadoop.ozone.s3.util.S3Utils.hasUnsignedPayload;
+import static org.apache.hadoop.ozone.s3.util.S3Utils.stripQuotes;
import static org.apache.hadoop.ozone.s3.util.S3Utils.urlDecode;
import static
org.apache.hadoop.ozone.s3.util.S3Utils.validateMultiChunksUpload;
import static org.apache.hadoop.ozone.s3.util.S3Utils.validateSignatureHeader;
+import static org.apache.hadoop.ozone.s3.util.S3Utils.wrapInQuotes;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableMap;
@@ -905,7 +907,7 @@ public Response
completeMultipartUpload(@PathParam("bucket") String bucket,
S3Owner.verifyBucketOwnerCondition(headers, bucket,
ozoneBucket.getOwner());
for (CompleteMultipartUploadRequest.Part part : partList) {
- partsMap.put(part.getPartNumber(), part.getETag());
+ partsMap.put(part.getPartNumber(), stripQuotes(part.getETag()));
}
if (LOG.isDebugEnabled()) {
LOG.debug("Parts map {}", partsMap);
@@ -1044,29 +1046,21 @@ private Response createMultipartKey(OzoneVolume volume,
OzoneBucket ozoneBucket,
"Bytes to skip: "
+ rangeHeader.getStartOffset() + " actual: " + skipped);
}
- try (OzoneOutputStream ozoneOutputStream = getClientProtocol()
- .createMultipartKey(volume.getName(), bucketName, key, length,
- partNumber, uploadID)) {
- metadataLatencyNs =
- getMetrics().updateCopyKeyMetadataStats(startNanos);
- copyLength = IOUtils.copyLarge(
- sourceObject, ozoneOutputStream, 0, length, new
byte[getIOBufferSize(length)]);
- ozoneOutputStream.getMetadata()
- .putAll(sourceKeyDetails.getMetadata());
- outputStream = ozoneOutputStream;
- }
- } else {
- try (OzoneOutputStream ozoneOutputStream = getClientProtocol()
- .createMultipartKey(volume.getName(), bucketName, key, length,
- partNumber, uploadID)) {
- metadataLatencyNs =
- getMetrics().updateCopyKeyMetadataStats(startNanos);
- copyLength = IOUtils.copyLarge(sourceObject, ozoneOutputStream,
0, length,
- new byte[getIOBufferSize(length)]);
- ozoneOutputStream.getMetadata()
- .putAll(sourceKeyDetails.getMetadata());
- outputStream = ozoneOutputStream;
+ }
+ try (OzoneOutputStream ozoneOutputStream = getClientProtocol()
+ .createMultipartKey(volume.getName(), bucketName, key, length,
+ partNumber, uploadID)) {
+ metadataLatencyNs =
+ getMetrics().updateCopyKeyMetadataStats(startNanos);
+ copyLength = IOUtils.copyLarge(sourceObject, ozoneOutputStream, 0,
length,
+ new byte[getIOBufferSize(length)]);
+ ozoneOutputStream.getMetadata()
+ .putAll(sourceKeyDetails.getMetadata());
+ String raw = ozoneOutputStream.getMetadata().get(ETAG);
+ if (raw != null) {
+ ozoneOutputStream.getMetadata().put(ETAG, stripQuotes(raw));
}
+ outputStream = ozoneOutputStream;
}
getMetrics().incCopyObjectSuccessLength(copyLength);
perf.appendSizeBytes(copyLength);
@@ -1099,6 +1093,7 @@ private Response createMultipartKey(OzoneVolume volume,
OzoneBucket ozoneBucket,
if (StringUtils.isEmpty(eTag)) {
eTag = omMultipartCommitUploadPartInfo.getPartName();
}
+ eTag = wrapInQuotes(eTag);
if (copyHeader != null) {
getMetrics().updateCopyObjectSuccessStats(startNanos);
@@ -1518,10 +1513,6 @@ public boolean isDatastreamEnabled() {
return datastreamEnabled;
}
- static String wrapInQuotes(String value) {
- return "\"" + value + "\"";
- }
-
@VisibleForTesting
public MessageDigest getMessageDigestInstance() {
return E_TAG_PROVIDER.get();
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpointStreaming.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpointStreaming.java
index e9db0882acb..647aafe839c 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpointStreaming.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpointStreaming.java
@@ -20,6 +20,7 @@
import static
org.apache.hadoop.ozone.audit.AuditLogger.PerformanceStringBuilder;
import static
org.apache.hadoop.ozone.s3.exception.S3ErrorTable.INVALID_REQUEST;
import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.NO_SUCH_UPLOAD;
+import static org.apache.hadoop.ozone.s3.util.S3Utils.wrapInQuotes;
import java.io.IOException;
import java.io.InputStream;
@@ -189,6 +190,8 @@ public static Response createMultipartKey(OzoneBucket
ozoneBucket, String key,
}
throw ex;
}
- return Response.ok().header(OzoneConsts.ETAG, eTag).build();
+ return Response.ok()
+ .header(OzoneConsts.ETAG, wrapInQuotes(eTag))
+ .build();
}
}
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Utils.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Utils.java
index 2b698c50272..36c4445470d 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Utils.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Utils.java
@@ -202,4 +202,24 @@ public static String generateCanonicalUserId(String input)
{
return DigestUtils.sha256Hex(input);
}
+ /**
+ * Strips leading and trailing double quotes from the given string.
+ *
+ * @param value the input string
+ * @return the string without leading and trailing double quotes
+ */
+ public static String stripQuotes(String value) {
+ return StringUtils.strip(value, "\"");
+ }
+
+ /**
+ * Wraps the given string in double quotes.
+ *
+ * @param value the input string
+ * @return the string wrapped in double quotes
+ */
+ public static String wrapInQuotes(String value) {
+ return StringUtils.wrap(value, '\"');
+ }
+
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
