This is an automated email from the ASF dual-hosted git repository.
peterxcli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new b5bf1d26fea HDDS-16315. Skip MD5 recomputation in CopyObject when
source ETag can be reused (#11136)
b5bf1d26fea is described below
commit b5bf1d26feae86325b73e87d4bcf5fb0cb39a67d
Author: Peter Lee <[email protected]>
AuthorDate: Tue Sep 22 15:09:18 2026 +0800
HDDS-16315. Skip MD5 recomputation in CopyObject when source ETag can be
reused (#11136)
---
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 27 +++++--
.../ozone/s3/endpoint/ObjectEndpointStreaming.java | 12 +++-
.../hadoop/ozone/s3/endpoint/TestObjectPut.java | 84 ++++++++++++++++++++--
.../ozone/s3/endpoint/TestUploadWithStream.java | 50 +++++++++++++
4 files changed, 161 insertions(+), 12 deletions(-)
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
index a04a82e9ebd..cccbe387fe0 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
@@ -1127,8 +1127,14 @@ uploadID, getChunkSize(), multiDigestInputStream, perf,
getHeaders(),
}
}
+ /**
+ * Writes the source object content to the destination key. When {@code
reusedETag} is null the
+ * MD5 digest {@code src} computes during the copy becomes the destination
ETag; otherwise
+ * {@code reusedETag} is stored as-is and {@code src} has digesting switched
off, so the content
+ * is not re-hashed.
+ */
@SuppressWarnings("checkstyle:ParameterNumber")
- CopyResult copy(OzoneVolume volume, DigestInputStream src, long srcKeyLen,
+ CopyResult copy(OzoneVolume volume, DigestInputStream src, String
reusedETag, long srcKeyLen,
String destKey, String destBucket,
ReplicationConfig replication,
Map<String, String> metadata,
@@ -1145,7 +1151,7 @@ srcKeyLen > getDatastreamMinLength()) {
perf.appendStreamMode();
final CopyResult copyResult = ObjectEndpointStreaming
.copyKeyWithStream(volume.getBucket(destBucket), destKey, srcKeyLen,
- getChunkSize(), replication, metadata, src, perf, startNanos,
tags,
+ getChunkSize(), replication, metadata, src, reusedETag, perf,
startNanos, tags,
writeConditions);
eTag = copyResult.getETag();
copyLength = copyResult.getSize();
@@ -1160,7 +1166,8 @@ srcKeyLen > getDatastreamMinLength()) {
getMetrics().updateCopyKeyMetadataStats(startNanos);
perf.appendMetaLatencyNanos(metadataLatencyNs);
copyLength = dest.copyFrom(src, getIOBufferSize(expectedLength));
- eTag =
DatatypeConverter.printHexBinary(src.getMessageDigest().digest()).toLowerCase();
+ eTag = reusedETag != null ? reusedETag
+ :
DatatypeConverter.printHexBinary(src.getMessageDigest().digest()).toLowerCase();
dest.getMetadata().put(OzoneConsts.ETAG, eTag);
}
modificationTime = destStream.getModificationTime();
@@ -1269,12 +1276,24 @@ private CopyObjectResponse copyObject(OzoneVolume
volume,
throw ex;
}
+ // A whole-object copy writes byte-identical content, so a plain
(non-multipart) MD5 ETag
+ // stored on the source is also the correct content MD5 for the
destination and the data does
+ // not need to be re-hashed during the copy. An aggregate "-N" ETag of
an MPU-created source
+ // is not a content MD5, so a fresh digest is still computed for it.
+ final String sourceETag =
sourceKeyDetails.getMetadata().get(OzoneConsts.ETAG);
+ final String reusedETag = StringUtils.isNotEmpty(sourceETag) &&
!sourceETag.contains("-")
+ ? stripQuotes(sourceETag) : null;
+
try (OzoneInputStream src = runWithS3ActionString(
"GetObject", () -> getClientProtocol().getKey(volume.getName(),
sourceBucket, sourceKey));
DigestInputStream sourceDigestInputStream = new
DigestInputStream(src, md5Digest)) {
getMetrics().updateCopyKeyMetadataStats(startNanos);
+ if (reusedETag != null) {
+ // Nothing reads the digest in this case, so do not hash the copied
bytes at all.
+ sourceDigestInputStream.on(false);
+ }
final CopyResult copyResult = runWithS3ActionString("PutObject", () ->
- copy(volume, sourceDigestInputStream, sourceKeyLen, destkey,
destBucket,
+ copy(volume, sourceDigestInputStream, reusedETag, sourceKeyLen,
destkey, destBucket,
replicationConfig, customMetadata, perf, startNanos, tags,
writeConditions));
getMetrics().updateCopyObjectSuccessStats(startNanos);
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpointStreaming.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpointStreaming.java
index 74b2059986e..76838f27b6e 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpointStreaming.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpointStreaming.java
@@ -178,6 +178,12 @@ private static OzoneDataStreamOutput
openStreamKeyForPut(OzoneBucket bucket,
return bucket.createStreamKey(keyPath, length, replicationConfig,
keyMetadata, tags, derivedKeyPiggyBacking);
}
+ /**
+ * Writes the copy source stream to the destination key. When {@code
reusedETag} is null the MD5
+ * digest {@code body} computes during the copy becomes the destination
ETag; otherwise
+ * {@code reusedETag} is stored as-is and {@code body} has digesting
switched off, so the content
+ * is not re-hashed.
+ */
@SuppressWarnings("checkstyle:ParameterNumber")
public static CopyResult copyKeyWithStream(
OzoneBucket bucket,
@@ -186,7 +192,7 @@ public static CopyResult copyKeyWithStream(
int bufferSize,
ReplicationConfig replicationConfig,
Map<String, String> keyMetadata,
- DigestInputStream body, PerformanceStringBuilder perf, long startNanos,
+ DigestInputStream body, String reusedETag, PerformanceStringBuilder
perf, long startNanos,
Map<String, String> tags,
S3ConditionalRequest.WriteConditions writeConditions)
throws IOException {
@@ -200,8 +206,8 @@ public static CopyResult copyKeyWithStream(
long metadataLatencyNs =
METRICS.updateCopyKeyMetadataStats(startNanos);
writeLen = writeGuard.copyFrom(body, bufferSize);
- eTag = DatatypeConverter.printHexBinary(body.getMessageDigest().digest())
- .toLowerCase();
+ eTag = reusedETag != null ? reusedETag
+ :
DatatypeConverter.printHexBinary(body.getMessageDigest().digest()).toLowerCase();
perf.appendMetaLatencyNanos(metadataLatencyNs);
writeGuard.getMetadata().put(OzoneConsts.ETAG, eTag);
}
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java
index 9a51f4a3e06..86abd3bfa8d 100644
---
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java
@@ -68,10 +68,12 @@
import com.google.common.collect.ImmutableMap;
import java.io.ByteArrayInputStream;
import java.io.IOException;
+import java.io.OutputStream;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.Base64;
+import java.util.HashMap;
import java.util.Map;
import java.util.stream.Stream;
import javax.ws.rs.HttpMethod;
@@ -79,6 +81,7 @@
import javax.ws.rs.core.MultivaluedHashMap;
import javax.ws.rs.core.MultivaluedMap;
import javax.ws.rs.core.Response;
+import javax.xml.bind.DatatypeConverter;
import org.apache.commons.lang3.RandomStringUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.hdds.client.ECReplicationConfig;
@@ -98,6 +101,7 @@
import org.apache.hadoop.ozone.s3.exception.OS3Exception;
import org.apache.hadoop.ozone.s3.exception.S3ErrorTable;
import org.apache.hadoop.ozone.s3.util.S3Consts;
+import org.apache.http.HttpStatus;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
@@ -116,6 +120,7 @@ class TestObjectPut {
private static final String DEST_BUCKET_NAME = "b2";
private static final String DEST_KEY = "key=value/2";
private static final String NONEXISTENT_BUCKET = "nonexist";
+ private static final String MULTIPART_ETAG =
"9b2cf535f27731c974343645a3985328-2";
/** Well-formed but meaningless signature, for the path that only strips the
chunk framing. */
private static final String FAKE_SIGNATURE = StringUtils.repeat('0', 64);
@@ -657,11 +662,8 @@ void testStandardObjectHeadersCopiedAndReplaced() throws
Exception {
@Test
public void testCopyObjectMessageDigestResetDuringException() throws
Exception {
- assertSucceeds(() -> putObject(CONTENT));
-
- OzoneKeyDetails keyDetails = assertKeyContent(bucket, KEY_NAME, CONTENT);
- assertNotNull(keyDetails.getMetadata());
- assertThat(keyDetails.getMetadata().get(OzoneConsts.ETAG)).isNotEmpty();
+ // A multipart-style "-N" source ETag keeps the copy on the digesting path
+ createSourceKeyWithETag(KEY_NAME, MULTIPART_ETAG);
MessageDigest messageDigest = mock(MessageDigest.class);
try (MockedStatic<EndpointBase> endpoint =
@@ -685,6 +687,58 @@ public void
testCopyObjectMessageDigestResetDuringException() throws Exception {
}
}
+ @Test
+ void testCopyObjectReusesSourceETagWithoutRehashing() throws Exception {
+ assertSucceeds(() -> putObject(CONTENT));
+ String sourceETag =
bucket.getKey(KEY_NAME).getMetadata().get(OzoneConsts.ETAG);
+ assertThat(sourceETag).isNotEmpty();
+
+ when(headers.getHeaderString(COPY_SOURCE_HEADER)).thenReturn(
+ BUCKET_NAME + "/" + urlEncode(KEY_NAME));
+
+ MessageDigest messageDigest = mock(MessageDigest.class);
+ try (MockedStatic<EndpointBase> endpoint =
+ mockStatic(EndpointBase.class, CALLS_REAL_METHODS)) {
+ // The copy must reuse the source's plain MD5 ETag: fail loudly if it
re-hashes the content
+
endpoint.when(EndpointBase::getMD5DigestInstance).thenReturn(messageDigest);
+ doThrow(new RuntimeException("digest should not be used"))
+ .when(messageDigest).update(any(byte[].class), anyInt(), anyInt());
+
+ Response response = put(objectEndpoint, DEST_BUCKET_NAME, DEST_KEY,
CONTENT);
+ assertEquals(HttpStatus.SC_OK, response.getStatus());
+ CopyObjectResponse copyObjectResponse = (CopyObjectResponse)
response.getEntity();
+ assertEquals("\"" + sourceETag + "\"", copyObjectResponse.getETag());
+ }
+
+ OzoneKeyDetails destKeyDetails = assertKeyContent(destBucket, DEST_KEY,
CONTENT);
+ assertEquals(sourceETag,
destKeyDetails.getMetadata().get(OzoneConsts.ETAG));
+ }
+
+ @Test
+ void testCopyObjectRecomputesETagForMultipartSource() throws Exception {
+ createSourceKeyWithETag(KEY_NAME, MULTIPART_ETAG);
+
+ when(headers.getHeaderString(COPY_SOURCE_HEADER)).thenReturn(
+ BUCKET_NAME + "/" + urlEncode(KEY_NAME));
+ assertSucceeds(() -> put(objectEndpoint, DEST_BUCKET_NAME, DEST_KEY,
CONTENT));
+
+ // The aggregate "-N" ETag is not a content MD5, so the destination gets a
fresh digest
+ OzoneKeyDetails destKeyDetails = assertKeyContent(destBucket, DEST_KEY,
CONTENT);
+ assertEquals(contentMd5Hex(),
destKeyDetails.getMetadata().get(OzoneConsts.ETAG));
+ }
+
+ @Test
+ void testCopyObjectComputesETagWhenSourceHasNoETag() throws Exception {
+ createSourceKeyWithETag(KEY_NAME, null);
+
+ when(headers.getHeaderString(COPY_SOURCE_HEADER)).thenReturn(
+ BUCKET_NAME + "/" + urlEncode(KEY_NAME));
+ assertSucceeds(() -> put(objectEndpoint, DEST_BUCKET_NAME, DEST_KEY,
CONTENT));
+
+ OzoneKeyDetails destKeyDetails = assertKeyContent(destBucket, DEST_KEY,
CONTENT);
+ assertEquals(contentMd5Hex(),
destKeyDetails.getMetadata().get(OzoneConsts.ETAG));
+ }
+
@Test
public void testCopyObjectWithTags() throws Exception {
// Put object in to source bucket
@@ -1107,6 +1161,26 @@ void testParseETag() {
assertEquals(null, parseETag(null));
}
+ /**
+ * Create {@code keyName} with pre-defined {@link #CONTENT} directly through
the client stub,
+ * with the given ETag stored in the key metadata ({@code null} for no ETag).
+ */
+ private void createSourceKeyWithETag(String keyName, String eTag) throws
IOException {
+ Map<String, String> metadata = new HashMap<>();
+ if (eTag != null) {
+ metadata.put(OzoneConsts.ETAG, eTag);
+ }
+ try (OutputStream out = bucket.createKey(keyName, CONTENT.length(),
+
RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.THREE),
metadata)) {
+ out.write(CONTENT.getBytes(StandardCharsets.UTF_8));
+ }
+ }
+
+ private static String contentMd5Hex() throws NoSuchAlgorithmException {
+ return DatatypeConverter.printHexBinary(
+
MessageDigest.getInstance("MD5").digest(CONTENT.getBytes(StandardCharsets.UTF_8))).toLowerCase();
+ }
+
/** Put object at {@code bucketName}/{@code keyName} with pre-defined {@link
#CONTENT}. */
private Response putObject(String bucketName, String keyName) throws
IOException, OS3Exception {
return put(objectEndpoint, bucketName, keyName, CONTENT);
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestUploadWithStream.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestUploadWithStream.java
index 27826401b7c..fdfb1ec527c 100644
---
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestUploadWithStream.java
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestUploadWithStream.java
@@ -46,6 +46,7 @@
import java.util.HashMap;
import java.util.Map;
import javax.ws.rs.core.HttpHeaders;
+import javax.xml.bind.DatatypeConverter;
import org.apache.hadoop.hdds.client.ReplicationConfig;
import org.apache.hadoop.hdds.client.ReplicationFactor;
import org.apache.hadoop.hdds.client.ReplicationType;
@@ -190,6 +191,55 @@ public void testUploadWithCopy() throws Exception {
assertEquals(dataSize, newDataSize);
}
+ @Test
+ public void testUploadWithCopyReusesSourceETag() throws Exception {
+ // A deliberately fake plain (non-multipart) ETag: if the copy re-hashed
the
+ // content, the destination would get the real MD5 instead of this value
+ String sourceETag = "00000000000000000000000000000000";
+ createCopySourceKey(sourceETag);
+ setCopyHeaders();
+
+ assertSucceeds(() -> put(rest, S3BUCKET, S3KEY, null));
+
+ OzoneBucket bucket = client.getObjectStore().getS3Bucket(S3BUCKET);
+ assertEquals(sourceETag,
bucket.getKey(S3KEY).getMetadata().get(OzoneConsts.ETAG));
+ }
+
+ @Test
+ public void testUploadWithCopyRecomputesETagForMultipartSource() throws
Exception {
+ // An aggregate "-N" ETag of an MPU-created source is not a content MD5, so
+ // the copy computes a fresh digest for the destination
+ createCopySourceKey("9b2cf535f27731c974343645a3985328-2");
+ setCopyHeaders();
+
+ assertSucceeds(() -> put(rest, S3BUCKET, S3KEY, null));
+
+ String expectedETag = DatatypeConverter.printHexBinary(
+
MessageDigest.getInstance("MD5").digest(S3_COPY_EXISTING_KEY_CONTENT.getBytes(UTF_8))).toLowerCase();
+ OzoneBucket bucket = client.getObjectStore().getS3Bucket(S3BUCKET);
+ assertEquals(expectedETag,
bucket.getKey(S3KEY).getMetadata().get(OzoneConsts.ETAG));
+ }
+
+ private void createCopySourceKey(String eTag) throws IOException {
+ OzoneBucket bucket = client.getObjectStore().getS3Bucket(S3BUCKET);
+ byte[] keyContent = S3_COPY_EXISTING_KEY_CONTENT.getBytes(UTF_8);
+ Map<String, String> metadata = new HashMap<>();
+ metadata.put(OzoneConsts.ETAG, eTag);
+ try (OutputStream stream = bucket
+ .createStreamKey(S3_COPY_EXISTING_KEY, keyContent.length,
+ ReplicationConfig.fromTypeAndFactor(ReplicationType.RATIS,
+ ReplicationFactor.THREE), metadata)) {
+ stream.write(keyContent);
+ }
+ }
+
+ private void setCopyHeaders() {
+ HttpHeaders copyHeaders = mock(HttpHeaders.class);
+
when(copyHeaders.getHeaderString(STORAGE_CLASS_HEADER)).thenReturn("STANDARD");
+ when(copyHeaders.getHeaderString(COPY_SOURCE_HEADER)).thenReturn(S3BUCKET
+ "/" + S3_COPY_EXISTING_KEY);
+ rest.setHeaders(copyHeaders);
+ }
+
private OzoneBucket configureSignedChunks(int decodedLength) throws
IOException {
OzoneBucket bucket = client.getObjectStore().getS3Bucket(S3BUCKET);
((OzoneBucketStub) bucket).setDerivedKey(signingKey());
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
