This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new e81f5c09d2 HDDS-10645. Support x-amz-metadata-directive in CopyObject
(#6490)
e81f5c09d2 is described below
commit e81f5c09d216ca0d401977f69a2b6c0728eb3f55
Author: Ivan Andika <[email protected]>
AuthorDate: Wed Apr 10 13:54:52 2024 +0800
HDDS-10645. Support x-amz-metadata-directive in CopyObject (#6490)
---
.../dist/src/main/smoketest/s3/objectcopy.robot | 32 +++++++++--
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 22 +++++++-
.../org/apache/hadoop/ozone/s3/util/S3Consts.java | 10 ++++
.../hadoop/ozone/s3/endpoint/TestObjectPut.java | 63 +++++++++++++++++++++-
4 files changed, 122 insertions(+), 5 deletions(-)
diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/objectcopy.robot
b/hadoop-ozone/dist/src/main/smoketest/s3/objectcopy.robot
index af7571d35b..e2bca772bc 100644
--- a/hadoop-ozone/dist/src/main/smoketest/s3/objectcopy.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/s3/objectcopy.robot
@@ -39,26 +39,42 @@ Copy Object Happy Scenario
Execute date > /tmp/copyfile
${file_checksum} = Execute md5sum /tmp/copyfile | awk
'{print $1}'
- ${result} = Execute AWSS3ApiCli put-object --bucket
${BUCKET} --key ${PREFIX}/copyobject/key=value/f1 --body /tmp/copyfile
+ ${result} = Execute AWSS3ApiCli put-object --bucket
${BUCKET} --key ${PREFIX}/copyobject/key=value/f1 --body /tmp/copyfile
--metadata="custom-key1=custom-value1,custom-key2=custom-value2,gdprEnabled=true"
${eTag} = Execute and checkrc echo '${result}' | jq -r
'.ETag' 0
Should Be Equal ${eTag}
\"${file_checksum}\"
${result} = Execute AWSS3ApiCli list-objects --bucket
${BUCKET} --prefix ${PREFIX}/copyobject/key=value/
Should contain ${result} f1
- ${result} = Execute AWSS3ApiCli copy-object --bucket
${DESTBUCKET} --key ${PREFIX}/copyobject/key=value/f1 --copy-source
${BUCKET}/${PREFIX}/copyobject/key=value/f1
+ ${result} = Execute AWSS3ApiCli copy-object --bucket
${DESTBUCKET} --key ${PREFIX}/copyobject/key=value/f1 --copy-source
${BUCKET}/${PREFIX}/copyobject/key=value/f1
--metadata="custom-key3=custom-value3,custom-key4=custom-value4"
${eTag} = Execute and checkrc echo '${result}' | jq -r
'.CopyObjectResult.ETag' 0
Should Be Equal ${eTag}
\"${file_checksum}\"
${result} = Execute AWSS3ApiCli list-objects --bucket
${DESTBUCKET} --prefix ${PREFIX}/copyobject/key=value/
Should contain ${result} f1
+
+ #check that the custom metadata of the source key has been copied to the
destination key (default copy directive is COPY)
+ ${result} = Execute AWSS3ApiCli head-object --bucket
${BUCKET} --key ${PREFIX}/copyobject/key=value/f1
+ Should contain ${result}
\"custom-key1\": \"custom-value1\"
+ Should contain ${result}
\"custom-key2\": \"custom-value2\"
+ # COPY directive ignores any metadata specified in the
copy object request
+ Should Not contain ${result}
\"custom-key3\": \"custom-value3\"
+ Should Not contain ${result}
\"custom-key4\": \"custom-value4\"
+
#copying again will not throw error
- ${result} = Execute AWSS3ApiCli copy-object --bucket
${DESTBUCKET} --key ${PREFIX}/copyobject/key=value/f1 --copy-source
${BUCKET}/${PREFIX}/copyobject/key=value/f1
+ #also uses the REPLACE copy directive
+ ${result} = Execute AWSS3ApiCli copy-object --bucket
${DESTBUCKET} --key ${PREFIX}/copyobject/key=value/f1 --copy-source
${BUCKET}/${PREFIX}/copyobject/key=value/f1
--metadata="custom-key3=custom-value3,custom-key4=custom-value4"
--metadata-directive REPLACE
${eTag} = Execute and checkrc echo '${result}' | jq -r
'.CopyObjectResult.ETag' 0
Should Be Equal ${eTag}
\"${file_checksum}\"
${result} = Execute AWSS3ApiCli list-objects --bucket
${DESTBUCKET} --prefix ${PREFIX}/copyobject/key=value/
Should contain ${result} f1
+ ${result} = Execute AWSS3ApiCli head-object --bucket
${DESTBUCKET} --key ${PREFIX}/copyobject/key=value/f1
+ Should contain ${result}
\"custom-key3\": \"custom-value3\"
+ Should contain ${result}
\"custom-key4\": \"custom-value4\"
+ # REPLACE directive uses the custom metadata specified
in the request instead of the source key's custom metadata
+ Should Not contain ${result}
\"custom-key1\": \"custom-value1\"
+ Should Not contain ${result}
\"custom-key2\": \"custom-value2\"
Copy Object Where Bucket is not available
${result} = Execute AWSS3APICli and checkrc copy-object
--bucket dfdfdfdfdfnonexistent --key ${PREFIX}/copyobject/key=value/f1
--copy-source ${BUCKET}/${PREFIX}/copyobject/key=value/f1 255
@@ -76,3 +92,13 @@ Copy Object Where both source and dest are same with change
to storageclass
Copy Object Where Key not available
${result} = Execute AWSS3APICli and checkrc copy-object
--bucket ${DESTBUCKET} --key ${PREFIX}/copyobject/key=value/f1 --copy-source
${BUCKET}/nonnonexistentkey 255
Should contain ${result} NoSuchKey
+
+Copy Object using an invalid copy directive
+ ${result} = Execute AWSS3ApiCli and checkrc copy-object
--bucket ${DESTBUCKET} --key ${PREFIX}/copyobject/key=value/f1 --copy-source
${BUCKET}/${PREFIX}/copyobject/key=value/f1 --metadata-directive INVALID
255
+ Should contain ${result}
InvalidArgument
+
+Copy Object with user defined metadata size larger than 2 KB
+ Execute echo "Randomtext" >
/tmp/testfile2
+ ${custom_metadata_value} = Execute printf 'v%.0s'
{1..3000}
+ ${result} = Execute AWSS3ApiCli and checkrc
copy-object --bucket ${DESTBUCKET} --key ${PREFIX}/copyobject/key=value/f1
--copy-source ${BUCKET}/${PREFIX}/copyobject/key=value/f1
--metadata="custom-key1=${custom_metadata_value}" --metadata-directive REPLACE
255
+ Should contain
${result} MetadataTooLarge
\ No newline at end of file
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
index 8b7db9f061..26e51a6d66 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
@@ -126,6 +126,7 @@ import static
org.apache.hadoop.ozone.s3.exception.S3ErrorTable.NO_SUCH_UPLOAD;
import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.PRECOND_FAILED;
import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.newError;
import static org.apache.hadoop.ozone.s3.util.S3Consts.ACCEPT_RANGE_HEADER;
+import static
org.apache.hadoop.ozone.s3.util.S3Consts.CUSTOM_METADATA_COPY_DIRECTIVE_HEADER;
import static
org.apache.hadoop.ozone.s3.util.S3Consts.DECODED_CONTENT_LENGTH_HEADER;
import static org.apache.hadoop.ozone.s3.util.S3Consts.CONTENT_RANGE_HEADER;
import static org.apache.hadoop.ozone.s3.util.S3Consts.COPY_SOURCE_HEADER;
@@ -135,6 +136,7 @@ import static
org.apache.hadoop.ozone.s3.util.S3Consts.COPY_SOURCE_IF_UNMODIFIED
import static org.apache.hadoop.ozone.s3.util.S3Consts.RANGE_HEADER;
import static
org.apache.hadoop.ozone.s3.util.S3Consts.RANGE_HEADER_SUPPORTED_UNIT;
import static org.apache.hadoop.ozone.s3.util.S3Consts.STORAGE_CLASS_HEADER;
+import static org.apache.hadoop.ozone.s3.util.S3Consts.CopyDirective;
import static org.apache.hadoop.ozone.s3.util.S3Utils.urlDecode;
/**
@@ -1208,12 +1210,30 @@ public class ObjectEndpoint extends EndpointBase {
}
}
long sourceKeyLen = sourceKeyDetails.getDataSize();
+
+ // Custom metadata in copyObject with metadata directive
+ Map<String, String> customMetadata;
+ String metadataCopyDirective =
headers.getHeaderString(CUSTOM_METADATA_COPY_DIRECTIVE_HEADER);
+ if (StringUtils.isEmpty(metadataCopyDirective) ||
metadataCopyDirective.equals(CopyDirective.COPY.name())) {
+ // The custom metadata will be copied from the source key
+ customMetadata = sourceKeyDetails.getMetadata();
+ } else if (metadataCopyDirective.equals(CopyDirective.REPLACE.name())) {
+ // Replace the metadata with the metadata form the request headers
+ customMetadata =
getCustomMetadataFromHeaders(headers.getRequestHeaders());
+ } else {
+ OS3Exception ex = newError(INVALID_ARGUMENT, metadataCopyDirective);
+ ex.setErrorMessage("An error occurred (InvalidArgument) " +
+ "when calling the CopyObject operation: " +
+ "The metadata directive specified is invalid. Valid values are
COPY or REPLACE.");
+ throw ex;
+ }
+
try (OzoneInputStream src = getClientProtocol().getKey(volume.getName(),
sourceBucket, sourceKey)) {
getMetrics().updateCopyKeyMetadataStats(startNanos);
sourceDigestInputStream = new DigestInputStream(src,
getMessageDigestInstance());
copy(volume, sourceDigestInputStream, sourceKeyLen, destkey,
destBucket, replicationConfig,
- sourceKeyDetails.getMetadata(), perf, startNanos);
+ customMetadata, perf, startNanos);
}
final OzoneKeyDetails destKeyDetails = getClientProtocol().getKeyDetails(
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java
index df3d01936b..3b38ff03c4 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java
@@ -62,10 +62,20 @@ public final class S3Consts {
public static final String S3_XML_NAMESPACE = "http://s3.amazonaws"; +
".com/doc/2006-03-01/";
+ // Constants related to custom metadata
public static final String CUSTOM_METADATA_HEADER_PREFIX = "x-amz-meta-";
+ public static final String CUSTOM_METADATA_COPY_DIRECTIVE_HEADER =
"x-amz-metadata-directive";
public static final String DECODED_CONTENT_LENGTH_HEADER =
"x-amz-decoded-content-length";
+ /**
+ * Copy directive for metadata and tags.
+ */
+ public enum CopyDirective {
+ COPY, // Default directive
+ REPLACE
+ }
+
}
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java
index 17c3cba304..abae489b41 100644
---
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java
@@ -27,6 +27,8 @@ import java.util.stream.Stream;
import java.io.OutputStream;
import java.security.MessageDigest;
import javax.ws.rs.core.HttpHeaders;
+import javax.ws.rs.core.MultivaluedHashMap;
+import javax.ws.rs.core.MultivaluedMap;
import javax.ws.rs.core.Response;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.RandomStringUtils;
@@ -56,6 +58,8 @@ import org.junit.jupiter.params.provider.MethodSource;
import org.mockito.MockedStatic;
import static java.nio.charset.StandardCharsets.UTF_8;
+import static
org.apache.hadoop.ozone.s3.util.S3Consts.CUSTOM_METADATA_COPY_DIRECTIVE_HEADER;
+import static
org.apache.hadoop.ozone.s3.util.S3Consts.CUSTOM_METADATA_HEADER_PREFIX;
import static
org.apache.hadoop.ozone.s3.util.S3Consts.DECODED_CONTENT_LENGTH_HEADER;
import static org.apache.hadoop.ozone.s3.util.S3Consts.COPY_SOURCE_HEADER;
import static org.apache.hadoop.ozone.s3.util.S3Consts.STORAGE_CLASS_HEADER;
@@ -254,6 +258,14 @@ class TestObjectPut {
ByteArrayInputStream body =
new ByteArrayInputStream(CONTENT.getBytes(UTF_8));
+ // Add some custom metadata
+ MultivaluedMap<String, String> metadataHeaders = new
MultivaluedHashMap<>();
+ metadataHeaders.putSingle(CUSTOM_METADATA_HEADER_PREFIX + "custom-key-1",
"custom-value-1");
+ metadataHeaders.putSingle(CUSTOM_METADATA_HEADER_PREFIX + "custom-key-2",
"custom-value-2");
+ when(headers.getRequestHeaders()).thenReturn(metadataHeaders);
+ // Add COPY metadata directive (default)
+
when(headers.getHeaderString(CUSTOM_METADATA_COPY_DIRECTIVE_HEADER)).thenReturn("COPY");
+
Response response = objectEndpoint.put(BUCKET_NAME, KEY_NAME,
CONTENT.length(), 1, null, body);
@@ -268,9 +280,14 @@ class TestObjectPut {
assertEquals(CONTENT, keyContent);
assertNotNull(keyDetails.getMetadata());
assertThat(keyDetails.getMetadata().get(OzoneConsts.ETAG)).isNotEmpty();
+
assertThat(keyDetails.getMetadata().get("custom-key-1")).isEqualTo("custom-value-1");
+
assertThat(keyDetails.getMetadata().get("custom-key-2")).isEqualTo("custom-value-2");
String sourceETag = keyDetails.getMetadata().get(OzoneConsts.ETAG);
+ // This will be ignored since the copy directive is COPY
+ metadataHeaders.putSingle(CUSTOM_METADATA_HEADER_PREFIX + "custom-key-3",
"custom-value-3");
+
// Add copy header, and then call put
when(headers.getHeaderString(COPY_SOURCE_HEADER)).thenReturn(
BUCKET_NAME + "/" + urlEncode(KEY_NAME));
@@ -296,9 +313,53 @@ class TestObjectPut {
// the same Etag since the key content is the same
assertEquals(sourceETag,
sourceKeyDetails.getMetadata().get(OzoneConsts.ETAG));
assertEquals(sourceETag,
destKeyDetails.getMetadata().get(OzoneConsts.ETAG));
+
assertThat(destKeyDetails.getMetadata().get("custom-key-1")).isEqualTo("custom-value-1");
+
assertThat(destKeyDetails.getMetadata().get("custom-key-2")).isEqualTo("custom-value-2");
+
assertThat(destKeyDetails.getMetadata().containsKey("custom-key-3")).isFalse();
- // source and dest same
+ // Now use REPLACE metadata directive (default) and remove some custom
metadata used in the source key
+
when(headers.getHeaderString(CUSTOM_METADATA_COPY_DIRECTIVE_HEADER)).thenReturn("REPLACE");
+ metadataHeaders.remove(CUSTOM_METADATA_HEADER_PREFIX + "custom-key-1");
+ metadataHeaders.remove(CUSTOM_METADATA_HEADER_PREFIX + "custom-key-2");
+
+ response = objectEndpoint.put(DEST_BUCKET_NAME, DEST_KEY,
CONTENT.length(), 1,
+ null, body);
+
+ ozoneInputStream =
clientStub.getObjectStore().getS3Bucket(DEST_BUCKET_NAME)
+ .readKey(DEST_KEY);
+
+ keyContent = IOUtils.toString(ozoneInputStream, UTF_8);
+ sourceKeyDetails = clientStub.getObjectStore()
+ .getS3Bucket(BUCKET_NAME).getKey(KEY_NAME);
+ destKeyDetails = clientStub.getObjectStore()
+ .getS3Bucket(DEST_BUCKET_NAME).getKey(DEST_KEY);
+
+ assertEquals(200, response.getStatus());
+ assertEquals(CONTENT, keyContent);
+ assertNotNull(keyDetails.getMetadata());
+ assertThat(keyDetails.getMetadata().get(OzoneConsts.ETAG)).isNotEmpty();
+ // Source key eTag should remain unchanged and the dest key should have
+ // the same Etag since the key content is the same
+ assertEquals(sourceETag,
sourceKeyDetails.getMetadata().get(OzoneConsts.ETAG));
+ assertEquals(sourceETag,
destKeyDetails.getMetadata().get(OzoneConsts.ETAG));
+
assertThat(destKeyDetails.getMetadata().containsKey("custom-key-1")).isFalse();
+
assertThat(destKeyDetails.getMetadata().containsKey("custom-key-2")).isFalse();
+
assertThat(destKeyDetails.getMetadata().get("custom-key-3")).isEqualTo("custom-value-3");
+
+
+ // wrong copy metadata directive
+
when(headers.getHeaderString(CUSTOM_METADATA_COPY_DIRECTIVE_HEADER)).thenReturn("INVALID");
OS3Exception e = assertThrows(OS3Exception.class, () -> objectEndpoint.put(
+ DEST_BUCKET_NAME, DEST_KEY, CONTENT.length(), 1, null, body),
+ "test copy object failed");
+ assertThat(e.getHttpCode()).isEqualTo(400);
+ assertThat(e.getCode()).isEqualTo("InvalidArgument");
+ assertThat(e.getErrorMessage()).contains("The metadata directive specified
is invalid");
+
+
when(headers.getHeaderString(CUSTOM_METADATA_COPY_DIRECTIVE_HEADER)).thenReturn("COPY");
+
+ // source and dest same
+ e = assertThrows(OS3Exception.class, () -> objectEndpoint.put(
BUCKET_NAME, KEY_NAME, CONTENT.length(), 1, null, body),
"test copy object failed");
assertThat(e.getErrorMessage()).contains("This copy request is illegal");
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
