This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new ac9a57ddc1b HDDS-16658. Reuse source key details when opening input
stream in S3 CopyObject (#11396)
ac9a57ddc1b is described below
commit ac9a57ddc1b3908276d6c5d3b02af9b9994475d4
Author: KUAN-HAO HUANG <[email protected]>
AuthorDate: Sun Oct 4 15:01:59 2026 +0800
HDDS-16658. Reuse source key details when opening input stream in S3
CopyObject (#11396)
---
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 2 +-
.../hadoop/ozone/client/OzoneBucketStub.java | 2 +-
.../hadoop/ozone/s3/endpoint/TestObjectPut.java | 25 ++++++++++++++++++++++
3 files changed, 27 insertions(+), 2 deletions(-)
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
index e911b07eee1..4b09a72dede 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
@@ -1287,7 +1287,7 @@ private CopyObjectResponse copyObject(OzoneVolume volume,
? stripQuotes(sourceETag) : null;
try (OzoneInputStream src = runWithS3ActionString(
- "GetObject", () -> getClientProtocol().getKey(volume.getName(),
sourceBucket, sourceKey));
+ "GetObject", sourceKeyDetails::getContent);
DigestInputStream sourceDigestInputStream = new
DigestInputStream(src, md5Digest)) {
getMetrics().updateCopyKeyMetadataStats(startNanos);
if (reusedETag != null) {
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java
index 0aef8607c87..841076030a1 100644
---
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java
@@ -316,7 +316,7 @@ public void close() throws IOException {
mtime,
mtime,
new ArrayList<>(), rConfig, objectMetadata, null,
- null, false,
+ () -> readKey(key), false,
UserGroupInformation.getCurrentUser().getShortUserName(),
tags
));
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java
index 8e48c6ca1f9..9e5bf45a067 100644
---
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java
@@ -63,6 +63,7 @@
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.mockStatic;
+import static org.mockito.Mockito.never;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
@@ -99,6 +100,7 @@
import org.apache.hadoop.ozone.client.OzoneClient;
import org.apache.hadoop.ozone.client.OzoneKeyDetails;
import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.ozone.s3.HeaderPreprocessor;
import org.apache.hadoop.ozone.s3.exception.OS3Exception;
@@ -109,6 +111,7 @@
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.CsvSource;
import org.junit.jupiter.params.provider.EmptySource;
import org.junit.jupiter.params.provider.MethodSource;
import org.junit.jupiter.params.provider.NullAndEmptySource;
@@ -738,6 +741,28 @@ public void
testCopyObjectMessageDigestResetDuringException() throws Exception {
}
}
+ @ParameterizedTest
+ @CsvSource({"false, false", "false, true", "true, false", "true, true"})
+ void testCopyObjectReusesSourceKeyDetails(boolean streamingUpload, boolean
streamingCopy) throws Exception {
+ objectEndpoint.init();
+ doReturn(streamingUpload).when(objectEndpoint).isDatastreamEnabled();
+ doReturn(0L).when(objectEndpoint).getDatastreamMinLength();
+ assertSucceeds(() -> putObject(CONTENT));
+
+ doReturn(streamingCopy).when(objectEndpoint).isDatastreamEnabled();
+ ClientProtocol protocol = spy(objectEndpoint.getClientProtocol());
+ doReturn(protocol).when(objectEndpoint).getClientProtocol();
+ when(headers.getHeaderString(COPY_SOURCE_HEADER)).thenReturn(BUCKET_NAME +
"/" + urlEncode(KEY_NAME));
+
+ try (Response response = put(objectEndpoint, DEST_BUCKET_NAME, DEST_KEY,
CONTENT)) {
+ assertThat(response.getStatus()).isEqualTo(HttpStatus.SC_OK);
+ assertThat(((CopyObjectResponse)
response.getEntity()).getETag()).isEqualTo("\"" + contentMd5Hex() + "\"");
+ }
+ assertKeyContent(destBucket, DEST_KEY, CONTENT);
+ verify(protocol).getKeyDetails(bucket.getVolumeName(), BUCKET_NAME,
KEY_NAME);
+ verify(protocol, never()).getKey(bucket.getVolumeName(), BUCKET_NAME,
KEY_NAME);
+ }
+
@Test
void testStreamingCopyReusesDestinationBucket() throws Exception {
assertSucceeds(() -> putObject(CONTENT));
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
