This is an automated email from the ASF dual-hosted git repository.
ritesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 0a5fc405e5 HDDS-6440. Handle custom metadata (x-amz-meta) during
put-object through S3 API. (#3728)
0a5fc405e5 is described below
commit 0a5fc405e52e91f0f1911b5f94d3f912dac8dce7
Author: DaveTeng0 <[email protected]>
AuthorDate: Tue Oct 11 21:26:38 2022 -0700
HDDS-6440. Handle custom metadata (x-amz-meta) during put-object through S3
API. (#3728)
---
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 2 +
.../org/apache/hadoop/ozone/client/OzoneKey.java | 32 +++++--
.../hadoop/ozone/client/OzoneKeyDetails.java | 11 +--
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 3 +-
.../dist/src/main/smoketest/s3/commonawslib.robot | 5 +
.../dist/src/main/smoketest/s3/objectputget.robot | 26 +++++
.../hadoop/ozone/s3/endpoint/EndpointBase.java | 72 +++++++++++++-
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 21 ++--
.../org/apache/hadoop/ozone/s3/util/S3Consts.java | 2 +
.../hadoop/ozone/s3/endpoint/TestEndpointBase.java | 106 +++++++++++++++++++++
10 files changed, 253 insertions(+), 27 deletions(-)
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
index dd01be2a2f..063157fe17 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
@@ -392,6 +392,8 @@ public final class OzoneConsts {
public static final int S3_SECRET_KEY_MIN_LENGTH = 8;
+ public static final int S3_REQUEST_HEADER_METADATA_SIZE_LIMIT_KB = 2;
+
//GDPR
public static final String GDPR_FLAG = "gdprEnabled";
public static final String GDPR_ALGORITHM_NAME = "AES";
diff --git
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java
index f0302c8e08..72dfb26f21 100644
---
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java
+++
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java
@@ -24,6 +24,8 @@ import org.apache.hadoop.hdds.client.ReplicationFactor;
import org.apache.hadoop.hdds.client.ReplicationType;
import java.time.Instant;
+import java.util.Map;
+import java.util.HashMap;
/**
* A class that encapsulates OzoneKey.
@@ -57,6 +59,7 @@ public class OzoneKey {
private ReplicationConfig replicationConfig;
+ private Map<String, String> metadata = new HashMap<>();
/**
* Constructs OzoneKey from OmKeyInfo.
*
@@ -67,14 +70,9 @@ public class OzoneKey {
String keyName, long size, long creationTime,
long modificationTime, ReplicationType type,
int replicationFactor) {
- this.volumeName = volumeName;
- this.bucketName = bucketName;
- this.name = keyName;
- this.dataSize = size;
- this.creationTime = Instant.ofEpochMilli(creationTime);
- this.modificationTime = Instant.ofEpochMilli(modificationTime);
- this.replicationConfig = ReplicationConfig.fromTypeAndFactor(type,
- ReplicationFactor.valueOf(replicationFactor));
+ this(volumeName, bucketName, keyName, size, creationTime, modificationTime,
+ ReplicationConfig.fromTypeAndFactor(type,
+ ReplicationFactor.valueOf(replicationFactor)));
}
/**
@@ -94,6 +92,16 @@ public class OzoneKey {
this.replicationConfig = replicationConfig;
}
+ @SuppressWarnings("parameternumber")
+ public OzoneKey(String volumeName, String bucketName,
+ String keyName, long size, long creationTime,
+ long modificationTime, ReplicationConfig replicationConfig,
+ Map<String, String> metadata) {
+ this(volumeName, bucketName, keyName, size, creationTime,
+ modificationTime, replicationConfig);
+ this.metadata.putAll(metadata);
+ }
+
/**
* Returns Volume Name associated with the Key.
*
@@ -154,6 +162,14 @@ public class OzoneKey {
* @return replicationType
*/
+ public Map<String, String> getMetadata() {
+ return metadata;
+ }
+
+ public void setMetadata(Map<String, String> metadata) {
+ this.metadata.putAll(metadata);
+ }
+
@Deprecated
@JsonIgnore
public ReplicationType getReplicationType() {
diff --git
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKeyDetails.java
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKeyDetails.java
index 016c34683c..757431bc6b 100644
---
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKeyDetails.java
+++
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKeyDetails.java
@@ -39,8 +39,6 @@ public class OzoneKeyDetails extends OzoneKey {
*/
private List<OzoneKeyLocation> ozoneKeyLocations;
- private Map<String, String> metadata;
-
private FileEncryptionInfo feInfo;
private SupplierWithIOException<OzoneInputStream> contentSupplier;
@@ -58,8 +56,8 @@ public class OzoneKeyDetails extends OzoneKey {
super(volumeName, bucketName, keyName, size, creationTime,
modificationTime, type, replicationFactor);
this.ozoneKeyLocations = ozoneKeyLocations;
- this.metadata = metadata;
this.feInfo = feInfo;
+ this.setMetadata(metadata);
}
@@ -75,9 +73,8 @@ public class OzoneKeyDetails extends OzoneKey {
FileEncryptionInfo feInfo,
SupplierWithIOException<OzoneInputStream> contentSupplier) {
super(volumeName, bucketName, keyName, size, creationTime,
- modificationTime, replicationConfig);
+ modificationTime, replicationConfig, metadata);
this.ozoneKeyLocations = ozoneKeyLocations;
- this.metadata = metadata;
this.feInfo = feInfo;
this.contentSupplier = contentSupplier;
}
@@ -89,10 +86,6 @@ public class OzoneKeyDetails extends OzoneKey {
return ozoneKeyLocations;
}
- public Map<String, String> getMetadata() {
- return metadata;
- }
-
public FileEncryptionInfo getFileEncryptionInfo() {
return feInfo;
}
diff --git
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
index 876ed2ade1..973f98ff3b 100644
---
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
+++
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
@@ -2002,7 +2002,8 @@ public class RpcClient implements ClientProtocol {
return new OzoneKey(keyInfo.getVolumeName(), keyInfo.getBucketName(),
keyInfo.getKeyName(), keyInfo.getDataSize(), keyInfo.getCreationTime(),
- keyInfo.getModificationTime(), keyInfo.getReplicationConfig());
+ keyInfo.getModificationTime(), keyInfo.getReplicationConfig(),
+ keyInfo.getMetadata());
}
diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/commonawslib.robot
b/hadoop-ozone/dist/src/main/smoketest/s3/commonawslib.robot
index 74a3c95a86..375c33d280 100644
--- a/hadoop-ozone/dist/src/main/smoketest/s3/commonawslib.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/s3/commonawslib.robot
@@ -36,6 +36,11 @@ Execute AWSS3APICli and checkrc
${output} = Execute and checkrc aws s3api --endpoint-url
${ENDPOINT_URL} ${command} ${expected_error_code}
[return] ${output}
+Execute AWSS3APICli and ignore error
+ [Arguments] ${command}
+ ${output} = Execute And Ignore Error aws s3api --endpoint-url
${ENDPOINT_URL} ${command}
+ [return] ${output}
+
Execute AWSS3Cli
[Arguments] ${command}
${output} = Execute aws s3 --endpoint-url
${ENDPOINT_URL} ${command}
diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot
b/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot
index 46608e7357..2f74decb61 100644
--- a/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot
@@ -159,3 +159,29 @@ Zero byte file
${result} = Execute AWSS3APICli and checkrc
get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/key=value/zerobyte
--range bytes=0-10000 /tmp/testfile2.result 255
Should contain ${result}
InvalidRange
+
+Create file with user defined metadata
+ Execute echo "Randomtext" >
/tmp/testfile2
+ Execute AWSS3ApiCli put-object --bucket
${BUCKET} --key ${PREFIX}/putobject/custom-metadata/key1 --body /tmp/testfile2
--metadata="custom-key1=custom-value1,custom-key2=custom-value2"
+
+ ${result} = Execute AWSS3APICli head-object --bucket
${BUCKET} --key ${PREFIX}/putobject/custom-metadata/key1
+ Should contain ${result}
\"custom-key1\": \"custom-value1\"
+ Should contain ${result}
\"custom-key2\": \"custom-value2\"
+
+ ${result} = Execute ozone sh key info
/s3v/${BUCKET}/${PREFIX}/putobject/custom-metadata/key1
+ Should contain ${result}
\"custom-key1\" : \"custom-value1\"
+ Should contain ${result}
\"custom-key2\" : \"custom-value2\"
+
+Create file with user defined metadata with gdpr enabled value in request
+ Execute echo "Randomtext" >
/tmp/testfile2
+ Execute AWSS3ApiCli put-object --bucket
${BUCKET} --key ${PREFIX}/putobject/custom-metadata/key2 --body /tmp/testfile2
--metadata="gdprEnabled=true,custom-key2=custom-value2"
+ ${result} = Execute AWSS3ApiCli head-object
--bucket ${BUCKET} --key ${PREFIX}/putobject/custom-metadata/key2
+ Should contain ${result}
\"custom-key2\": \"custom-value2\"
+ Should not contain ${result}
\"gdprEnabled\": \"true\"
+
+
+Create file with user defined metadata size larger than 2 KB
+ Execute echo "Randomtext" >
/tmp/testfile2
+ ${custom_metadata_value} = Execute printf 'v%.0s'
{1..3000}
+ ${result} = Execute AWSS3APICli and ignore error
put-object --bucket ${BUCKET} --key ${PREFIX}/putobject/custom-metadata/key2
--body /tmp/testfile2 --metadata="custom-key1=${custom_metadata_value}"
+ Should not contain
${result} custom-key1: ${custom_metadata_value}
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java
index cc76f267b1..56a187fd00 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java
@@ -19,14 +19,24 @@ package org.apache.hadoop.ozone.s3.endpoint;
import javax.annotation.PostConstruct;
import javax.inject.Inject;
+import javax.ws.rs.core.MultivaluedMap;
+import javax.ws.rs.core.Response;
import javax.ws.rs.container.ContainerRequestContext;
import javax.ws.rs.core.Context;
import java.io.IOException;
-import java.util.Collections;
-import java.util.Iterator;
+import java.util.Set;
+import java.util.HashSet;
+import java.util.Arrays;
import java.util.Map;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Collections;
import java.util.function.Function;
+import java.util.stream.Collectors;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.audit.AuditAction;
import org.apache.hadoop.ozone.audit.AuditEventStatus;
import org.apache.hadoop.ozone.audit.AuditLogger;
@@ -35,6 +45,7 @@ import org.apache.hadoop.ozone.audit.AuditMessage;
import org.apache.hadoop.ozone.audit.Auditor;
import org.apache.hadoop.ozone.client.OzoneBucket;
import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneKey;
import org.apache.hadoop.ozone.client.OzoneVolume;
import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
import org.apache.hadoop.ozone.om.exceptions.OMException;
@@ -44,12 +55,16 @@ import org.apache.hadoop.ozone.s3.exception.OS3Exception;
import org.apache.hadoop.ozone.s3.exception.S3ErrorTable;
import com.google.common.annotations.VisibleForTesting;
+
import org.apache.hadoop.ozone.s3.metrics.S3GatewayMetrics;
import org.apache.hadoop.ozone.s3.util.AuditUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.hadoop.ozone.OzoneConsts.KB;
import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.newError;
+import static
org.apache.hadoop.ozone.s3.util.S3Consts.CUSTOM_METADATA_HEADER_PREFIX;
/**
* Basic helpers for all the REST endpoints.
@@ -63,6 +78,8 @@ public abstract class EndpointBase implements Auditor {
@Context
private ContainerRequestContext context;
+ private Set<String> excludeMetadataFields =
+ new HashSet<>(Arrays.asList(OzoneConsts.GDPR_FLAG));
private static final Logger LOG =
LoggerFactory.getLogger(EndpointBase.class);
@@ -240,6 +257,57 @@ public abstract class EndpointBase implements Auditor {
}
}
+ protected Map<String, String> getCustomMetadataFromHeaders(
+ MultivaluedMap<String, String> requestHeaders) throws OS3Exception {
+ Map<String, String> customMetadata = new HashMap<>();
+ if (requestHeaders == null || requestHeaders.isEmpty()) {
+ return customMetadata;
+ }
+
+ Set<String> customMetadataKeys = requestHeaders.keySet().stream()
+ .filter(k -> {
+ if (k.startsWith(CUSTOM_METADATA_HEADER_PREFIX) &&
+ !excludeMetadataFields.contains(
+ k.substring(
+ CUSTOM_METADATA_HEADER_PREFIX.length()))) {
+ return true;
+ }
+ return false;
+ })
+ .collect(Collectors.toSet());
+
+ long sizeInBytes = 0;
+ if (!customMetadataKeys.isEmpty()) {
+ for (String key : customMetadataKeys) {
+ String mapKey =
+ key.substring(CUSTOM_METADATA_HEADER_PREFIX.length());
+ List<String> values = requestHeaders.get(key);
+ String value = StringUtils.join(values, ",");
+ sizeInBytes += mapKey.getBytes(UTF_8).length;
+ sizeInBytes += value.getBytes(UTF_8).length;
+
+ if (sizeInBytes >
+ OzoneConsts.S3_REQUEST_HEADER_METADATA_SIZE_LIMIT_KB * KB) {
+ throw new IllegalArgumentException("Illegal user defined metadata." +
+ " Combined size cannot exceed 2KB.");
+ }
+ customMetadata.put(mapKey, value);
+ }
+ }
+ return customMetadata;
+ }
+
+ protected void addCustomMetadataHeaders(
+ Response.ResponseBuilder responseBuilder, OzoneKey key) {
+
+ Map<String, String> metadata = key.getMetadata();
+ for (Map.Entry<String, String> entry : metadata.entrySet()) {
+ responseBuilder
+ .header(CUSTOM_METADATA_HEADER_PREFIX + entry.getKey(),
+ entry.getValue());
+ }
+ }
+
private AuditMessage.Builder auditMessageBaseBuilder(AuditAction op,
Map<String, String> auditMap) {
AuditMessage.Builder builder = new AuditMessage.Builder()
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
index 52df945940..488dc9ce49 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
@@ -48,10 +48,9 @@ import java.text.ParseException;
import java.time.Instant;
import java.time.ZoneId;
import java.time.ZonedDateTime;
-import java.util.HashMap;
+import java.util.Map;
import java.util.LinkedHashMap;
import java.util.List;
-import java.util.Map;
import java.util.OptionalLong;
import org.apache.commons.lang3.StringUtils;
@@ -210,13 +209,17 @@ public class ObjectEndpoint extends EndpointBase {
"Connection", "close").build();
}
+ // Normal put object
+ Map<String, String> customMetadata =
+ getCustomMetadataFromHeaders(headers.getRequestHeaders());
+
if ("STREAMING-AWS4-HMAC-SHA256-PAYLOAD"
.equals(headers.getHeaderString("x-amz-content-sha256"))) {
body = new SignedChunksInputStream(body);
}
output = getClientProtocol().createKey(volume.getName(), bucketName,
- keyPath, length, replicationConfig, new HashMap<>());
+ keyPath, length, replicationConfig, customMetadata);
IOUtils.copy(body, output);
getMetrics().incCreateKeySuccess();
@@ -471,6 +474,7 @@ public class ObjectEndpoint extends EndpointBase {
.header("Content-Length", key.getDataSize())
.header("Content-Type", "binary/octet-stream");
addLastModifiedDate(response, key);
+ addCustomMetadataHeaders(response, key);
getMetrics().incHeadKeySuccess();
AUDIT.logReadSuccess(buildAuditMessageForSuccess(s3GAction,
getAuditParameters()));
@@ -883,10 +887,12 @@ public class ObjectEndpoint extends EndpointBase {
void copy(OzoneVolume volume, InputStream src, long srcKeyLen,
String destKey, String destBucket,
- ReplicationConfig replication) throws IOException {
- try (OzoneOutputStream dest = getClientProtocol().createKey(
+ ReplicationConfig replication,
+ Map<String, String> metadata) throws IOException {
+ try (OzoneOutputStream dest =
+ getClientProtocol().createKey(
volume.getName(), destBucket, destKey, srcKeyLen,
- replication, new HashMap<>())) {
+ replication, metadata)) {
IOUtils.copy(src, dest);
}
}
@@ -937,7 +943,8 @@ public class ObjectEndpoint extends EndpointBase {
try (OzoneInputStream src = getClientProtocol().getKey(volume.getName(),
sourceBucket, sourceKey)) {
- copy(volume, src, sourceKeyLen, destkey, destBucket,
replicationConfig);
+ copy(volume, src, sourceKeyLen, destkey, destBucket, replicationConfig,
+ sourceKeyDetails.getMetadata());
}
final OzoneKeyDetails destKeyDetails = getClientProtocol().getKeyDetails(
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java
index f891e13d77..f620fd624c 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java
@@ -62,4 +62,6 @@ public final class S3Consts {
public static final String S3_XML_NAMESPACE = "http://s3.amazonaws"; +
".com/doc/2006-03-01/";
+ public static final String CUSTOM_METADATA_HEADER_PREFIX = "x-amz-meta-";
+
}
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestEndpointBase.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestEndpointBase.java
new file mode 100644
index 0000000000..9c1f254b79
--- /dev/null
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestEndpointBase.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+/**
+ * Tests the s3 EndpointBase class methods.
+ */
+package org.apache.hadoop.ozone.s3.endpoint;
+
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.s3.exception.OS3Exception;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.jupiter.api.Assertions;
+
+import javax.ws.rs.core.MultivaluedHashMap;
+import javax.ws.rs.core.MultivaluedMap;
+
+import java.nio.charset.StandardCharsets;
+import java.util.Map;
+
+import static
org.apache.hadoop.ozone.s3.util.S3Consts.CUSTOM_METADATA_HEADER_PREFIX;
+
+/**
+ * Test methods of the EndpointBase.
+ */
+public class TestEndpointBase {
+
+ /**
+ * Verify s3 metadata key "gdprEnabled" can't be set up directly
+ * from the normal client's request,
+ * it should be decided on the server side.
+ */
+ @Test
+ public void testFilterGDPRFromCustomMetadataHeaders()
+ throws OS3Exception {
+ MultivaluedMap<String, String> s3requestHeaders
+ = new MultivaluedHashMap<>();
+ s3requestHeaders.add(
+ CUSTOM_METADATA_HEADER_PREFIX + "custom-key1", "custom-value1");
+ s3requestHeaders.add(
+ CUSTOM_METADATA_HEADER_PREFIX + "custom-key2", "custom-value2");
+ s3requestHeaders.add(
+ CUSTOM_METADATA_HEADER_PREFIX + OzoneConsts.GDPR_FLAG, "true");
+
+ EndpointBase endpointBase = new EndpointBase() {
+ @Override
+ public void init() { }
+ };
+
+ Map<String, String> filteredCustomMetadata =
+ endpointBase.getCustomMetadataFromHeaders(s3requestHeaders);
+ Assert.assertTrue(filteredCustomMetadata.containsKey("custom-key1"));
+ Assert.assertEquals(
+ "custom-value1", filteredCustomMetadata.get("custom-key1"));
+ Assert.assertTrue(filteredCustomMetadata.containsKey("custom-key2"));
+ Assert.assertEquals(
+ "custom-value2", filteredCustomMetadata.get("custom-key2"));
+ Assert.assertFalse(
+ filteredCustomMetadata.containsKey(OzoneConsts.GDPR_FLAG));
+ }
+
+ /**
+ * Verify s3 request metadata size should be smaller than 2 KB.
+ */
+ @Test
+ public void testCustomMetadataHeadersSizeOverbig() {
+ MultivaluedMap<String, String> s3requestHeaders
+ = new MultivaluedHashMap<>();
+ s3requestHeaders.add(
+ CUSTOM_METADATA_HEADER_PREFIX + "custom-key1", "custom-value1");
+ s3requestHeaders.add(
+ CUSTOM_METADATA_HEADER_PREFIX + "custom-key2", "custom-value2");
+ s3requestHeaders.add(
+ CUSTOM_METADATA_HEADER_PREFIX + "custom-key3",
+ new String(new byte[3000], StandardCharsets.UTF_8));
+
+ EndpointBase endpointBase = new EndpointBase() {
+ @Override
+ public void init() { }
+ };
+
+ Exception exception = Assertions.assertThrows(
+ IllegalArgumentException.class,
+ () -> endpointBase.getCustomMetadataFromHeaders(s3requestHeaders));
+ Assert.assertEquals(
+ "Illegal user defined metadata. Combined size cannot exceed 2KB.",
+ exception.getMessage());
+ }
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
