This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new babf85c762 HDDS-10403. CopyObject should set ETag based on the key
content (#6251)
babf85c762 is described below
commit babf85c762ec159b6086145299f9a69b0a27f2ae
Author: Ivan Andika <[email protected]>
AuthorDate: Fri Feb 23 20:35:43 2024 +0800
HDDS-10403. CopyObject should set ETag based on the key content (#6251)
---
.../dist/src/main/smoketest/s3/objectcopy.robot | 14 +++++++++++
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 17 ++++++++------
.../ozone/s3/endpoint/ObjectEndpointStreaming.java | 9 +++++---
.../hadoop/ozone/client/OzoneBucketStub.java | 2 +-
.../hadoop/ozone/s3/endpoint/TestObjectPut.java | 27 ++++++++++++++++++++++
5 files changed, 58 insertions(+), 11 deletions(-)
diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/objectcopy.robot
b/hadoop-ozone/dist/src/main/smoketest/s3/objectcopy.robot
index 21764d65c4..af7571d35b 100644
--- a/hadoop-ozone/dist/src/main/smoketest/s3/objectcopy.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/s3/objectcopy.robot
@@ -37,15 +37,26 @@ Create Dest Bucket
Copy Object Happy Scenario
Run Keyword if '${DESTBUCKET}' == 'generated1' Create Dest Bucket
Execute date > /tmp/copyfile
+ ${file_checksum} = Execute md5sum /tmp/copyfile | awk
'{print $1}'
+
${result} = Execute AWSS3ApiCli put-object --bucket
${BUCKET} --key ${PREFIX}/copyobject/key=value/f1 --body /tmp/copyfile
+ ${eTag} = Execute and checkrc echo '${result}' | jq -r
'.ETag' 0
+ Should Be Equal ${eTag}
\"${file_checksum}\"
+
${result} = Execute AWSS3ApiCli list-objects --bucket
${BUCKET} --prefix ${PREFIX}/copyobject/key=value/
Should contain ${result} f1
${result} = Execute AWSS3ApiCli copy-object --bucket
${DESTBUCKET} --key ${PREFIX}/copyobject/key=value/f1 --copy-source
${BUCKET}/${PREFIX}/copyobject/key=value/f1
+ ${eTag} = Execute and checkrc echo '${result}' | jq -r
'.CopyObjectResult.ETag' 0
+ Should Be Equal ${eTag}
\"${file_checksum}\"
+
${result} = Execute AWSS3ApiCli list-objects --bucket
${DESTBUCKET} --prefix ${PREFIX}/copyobject/key=value/
Should contain ${result} f1
#copying again will not throw error
${result} = Execute AWSS3ApiCli copy-object --bucket
${DESTBUCKET} --key ${PREFIX}/copyobject/key=value/f1 --copy-source
${BUCKET}/${PREFIX}/copyobject/key=value/f1
+ ${eTag} = Execute and checkrc echo '${result}' | jq -r
'.CopyObjectResult.ETag' 0
+ Should Be Equal ${eTag}
\"${file_checksum}\"
+
${result} = Execute AWSS3ApiCli list-objects --bucket
${DESTBUCKET} --prefix ${PREFIX}/copyobject/key=value/
Should contain ${result} f1
@@ -56,8 +67,11 @@ Copy Object Where Bucket is not available
Should contain ${result}
NoSuchBucket
Copy Object Where both source and dest are same with change to storageclass
+ ${file_checksum} = Execute md5sum /tmp/copyfile | awk
'{print $1}'
${result} = Execute AWSS3APICli copy-object
--storage-class REDUCED_REDUNDANCY --bucket ${DESTBUCKET} --key
${PREFIX}/copyobject/key=value/f1 --copy-source
${DESTBUCKET}/${PREFIX}/copyobject/key=value/f1
Should contain ${result} ETag
+ ${eTag} = Execute and checkrc echo '${result}' | jq -r
'.CopyObjectResult.ETag' 0
+ Should Be Equal ${eTag}
\"${file_checksum}\"
Copy Object Where Key not available
${result} = Execute AWSS3APICli and checkrc copy-object
--bucket ${DESTBUCKET} --key ${PREFIX}/copyobject/key=value/f1 --copy-source
${BUCKET}/nonnonexistentkey 255
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
index 24115abe8e..0514125abd 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
@@ -1118,13 +1118,14 @@ public class ObjectEndpoint extends EndpointBase {
PerformanceStringBuilder perf, long startNanos)
throws IOException {
long copyLength;
+ src = new DigestInputStream(src, E_TAG_PROVIDER.get());
if (datastreamEnabled && !(replication != null &&
replication.getReplicationType() == EC) &&
srcKeyLen > datastreamMinLength) {
perf.appendStreamMode();
copyLength = ObjectEndpointStreaming
.copyKeyWithStream(volume.getBucket(destBucket), destKey, srcKeyLen,
- chunkSize, replication, metadata, src, perf, startNanos);
+ chunkSize, replication, metadata, (DigestInputStream) src, perf,
startNanos);
} else {
try (OzoneOutputStream dest = getClientProtocol()
.createKey(volume.getName(), destBucket, destKey, srcKeyLen,
@@ -1133,6 +1134,10 @@ public class ObjectEndpoint extends EndpointBase {
getMetrics().updateCopyKeyMetadataStats(startNanos);
perf.appendMetaLatencyNanos(metadataLatencyNs);
copyLength = IOUtils.copyLarge(src, dest);
+ String eTag = DatatypeConverter.printHexBinary(
+ ((DigestInputStream) src).getMessageDigest().digest())
+ .toLowerCase();
+ dest.getMetadata().put(ETAG, eTag);
}
}
getMetrics().incCopyObjectSuccessLength(copyLength);
@@ -1151,8 +1156,9 @@ public class ObjectEndpoint extends EndpointBase {
String sourceBucket = result.getLeft();
String sourceKey = result.getRight();
try {
+ OzoneKeyDetails sourceKeyDetails = getClientProtocol().getKeyDetails(
+ volume.getName(), sourceBucket, sourceKey);
// Checking whether we trying to copying to it self.
-
if (sourceBucket.equals(destBucket) && sourceKey
.equals(destkey)) {
// When copying to same storage type when storage type is provided,
@@ -1171,15 +1177,12 @@ public class ObjectEndpoint extends EndpointBase {
// still does not support this just returning dummy response
// for now
CopyObjectResponse copyObjectResponse = new CopyObjectResponse();
- copyObjectResponse.setETag(OzoneUtils.getRequestID());
+
copyObjectResponse.setETag(wrapInQuotes(sourceKeyDetails.getMetadata().get(ETAG)));
copyObjectResponse.setLastModified(Instant.ofEpochMilli(
Time.now()));
return copyObjectResponse;
}
}
-
- OzoneKeyDetails sourceKeyDetails = getClientProtocol().getKeyDetails(
- volume.getName(), sourceBucket, sourceKey);
long sourceKeyLen = sourceKeyDetails.getDataSize();
try (OzoneInputStream src = getClientProtocol().getKey(volume.getName(),
@@ -1194,7 +1197,7 @@ public class ObjectEndpoint extends EndpointBase {
getMetrics().updateCopyObjectSuccessStats(startNanos);
CopyObjectResponse copyObjectResponse = new CopyObjectResponse();
- copyObjectResponse.setETag(OzoneUtils.getRequestID());
+
copyObjectResponse.setETag(wrapInQuotes(destKeyDetails.getMetadata().get(ETAG)));
copyObjectResponse.setLastModified(destKeyDetails.getModificationTime());
return copyObjectResponse;
} catch (OMException ex) {
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpointStreaming.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpointStreaming.java
index bbb743ee35..b916fc111d 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpointStreaming.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpointStreaming.java
@@ -122,15 +122,18 @@ final class ObjectEndpointStreaming {
int bufferSize,
ReplicationConfig replicationConfig,
Map<String, String> keyMetadata,
- InputStream body, PerformanceStringBuilder perf, long startNanos)
+ DigestInputStream body, PerformanceStringBuilder perf, long startNanos)
throws IOException {
- long writeLen = 0;
+ long writeLen;
try (OzoneDataStreamOutput streamOutput = bucket.createStreamKey(keyPath,
length, replicationConfig, keyMetadata)) {
long metadataLatencyNs =
METRICS.updateCopyKeyMetadataStats(startNanos);
- perf.appendMetaLatencyNanos(metadataLatencyNs);
writeLen = writeToStreamOutput(streamOutput, body, bufferSize, length);
+ String eTag =
DatatypeConverter.printHexBinary(body.getMessageDigest().digest())
+ .toLowerCase();
+ perf.appendMetaLatencyNanos(metadataLatencyNs);
+ ((KeyMetadataAware)streamOutput).getMetadata().put(OzoneConsts.ETAG,
eTag);
}
return writeLen;
}
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java
index 39ae9cc4af..0cbe0781c4 100644
---
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java
@@ -209,7 +209,7 @@ public class OzoneBucketStub extends OzoneBucket {
Map<String, String> keyMetadata)
throws IOException {
ByteBufferStreamOutput byteBufferStreamOutput =
- new ByteBufferStreamOutput() {
+ new KeyMetadataAwareByteBufferStreamOutput(keyMetadata) {
private final ByteBuffer buffer = ByteBuffer.allocate((int) size);
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java
index ae8279f258..0daa666ae4 100644
---
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java
@@ -26,9 +26,11 @@ import java.io.InputStream;
import javax.ws.rs.core.HttpHeaders;
import javax.ws.rs.core.Response;
import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.hdds.client.ECReplicationConfig;
import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.client.ObjectStore;
import org.apache.hadoop.ozone.client.OzoneBucket;
import org.apache.hadoop.ozone.client.OzoneClient;
@@ -52,7 +54,9 @@ import static
org.apache.hadoop.ozone.s3.util.S3Consts.STORAGE_CLASS_HEADER;
import static org.apache.hadoop.ozone.s3.util.S3Utils.urlEncode;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.any;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.eq;
@@ -108,9 +112,12 @@ public class TestObjectPut {
.readKey(keyName);
String keyContent =
IOUtils.toString(ozoneInputStream, UTF_8);
+ OzoneKeyDetails keyDetails =
clientStub.getObjectStore().getS3Bucket(bucketName).getKey(keyName);
assertEquals(200, response.getStatus());
assertEquals(CONTENT, keyContent);
+ assertNotNull(keyDetails.getMetadata());
+
assertTrue(StringUtils.isNotEmpty(keyDetails.getMetadata().get(OzoneConsts.ETAG)));
}
@Test
@@ -136,9 +143,12 @@ public class TestObjectPut {
.readKey(keyName);
String keyContent =
IOUtils.toString(ozoneInputStream, UTF_8);
+ OzoneKeyDetails keyDetails =
clientStub.getObjectStore().getS3Bucket(bucketName).getKey(keyName);
assertEquals(200, response.getStatus());
assertEquals(CONTENT, keyContent);
+ assertNotNull(keyDetails.getMetadata());
+
assertTrue(StringUtils.isNotEmpty(keyDetails.getMetadata().get(OzoneConsts.ETAG)));
}
@Test
@@ -208,9 +218,12 @@ public class TestObjectPut {
clientStub.getObjectStore().getS3Bucket(bucketName)
.readKey(keyName);
String keyContent = IOUtils.toString(ozoneInputStream, UTF_8);
+ OzoneKeyDetails keyDetails =
clientStub.getObjectStore().getS3Bucket(bucketName).getKey(keyName);
assertEquals(200, response.getStatus());
assertEquals("1234567890abcde", keyContent);
+ assertNotNull(keyDetails.getMetadata());
+
assertTrue(StringUtils.isNotEmpty(keyDetails.getMetadata().get(OzoneConsts.ETAG)));
}
@Test
@@ -230,10 +243,14 @@ public class TestObjectPut {
.readKey(keyName);
String keyContent = IOUtils.toString(ozoneInputStream, UTF_8);
+ OzoneKeyDetails keyDetails =
clientStub.getObjectStore().getS3Bucket(bucketName).getKey(keyName);
assertEquals(200, response.getStatus());
assertEquals(CONTENT, keyContent);
+ assertNotNull(keyDetails.getMetadata());
+
assertTrue(StringUtils.isNotEmpty(keyDetails.getMetadata().get(OzoneConsts.ETAG)));
+ String sourceETag = keyDetails.getMetadata().get(OzoneConsts.ETAG);
// Add copy header, and then call put
when(headers.getHeaderString(COPY_SOURCE_HEADER)).thenReturn(
@@ -247,9 +264,19 @@ public class TestObjectPut {
.readKey(destkey);
keyContent = IOUtils.toString(ozoneInputStream, UTF_8);
+ OzoneKeyDetails sourceKeyDetails = clientStub.getObjectStore()
+ .getS3Bucket(bucketName).getKey(keyName);
+ OzoneKeyDetails destKeyDetails = clientStub.getObjectStore()
+ .getS3Bucket(destBucket).getKey(destkey);
assertEquals(200, response.getStatus());
assertEquals(CONTENT, keyContent);
+ assertNotNull(keyDetails.getMetadata());
+
assertTrue(StringUtils.isNotEmpty(keyDetails.getMetadata().get(OzoneConsts.ETAG)));
+ // Source key eTag should remain unchanged and the dest key should have
+ // the same Etag since the key content is the same
+ assertEquals(sourceETag,
sourceKeyDetails.getMetadata().get(OzoneConsts.ETAG));
+ assertEquals(sourceETag,
destKeyDetails.getMetadata().get(OzoneConsts.ETAG));
// source and dest same
OS3Exception e = assertThrows(OS3Exception.class, () -> objectEndpoint.put(
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
