This is an automated email from the ASF dual-hosted git repository.
ivandika3 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 0024c945da5 HDDS-15512. Implement S3 PUT/GET/DELETE ?tagging on
buckets (#10644)
0024c945da5 is described below
commit 0024c945da5d7574cb6bbc05f946b4721e5e998f
Author: Gargi Jaiswal <[email protected]>
AuthorDate: Fri Jul 3 17:43:53 2026 +0530
HDDS-15512. Implement S3 PUT/GET/DELETE ?tagging on buckets (#10644)
---
.../dist/src/main/smoketest/s3/buckettagging.robot | 89 +++++++++
.../dist/src/main/smoketest/s3/commonawslib.robot | 21 ++-
.../src/main/smoketest/s3/s3_compatbility_check.sh | 1 +
.../ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java | 33 ++++
.../ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java | 38 ++++
.../org/apache/hadoop/ozone/audit/S3GAction.java | 3 +
.../ozone/s3/endpoint/BucketCrudHandler.java | 5 +-
.../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 1 +
.../ozone/s3/endpoint/BucketTaggingHandler.java | 152 +++++++++++++++
.../hadoop/ozone/s3/endpoint/EndpointBase.java | 18 +-
.../hadoop/ozone/s3/metrics/S3GatewayMetrics.java | 90 +++++++++
.../org/apache/hadoop/ozone/s3/util/S3Consts.java | 4 +
.../ozone/s3/endpoint/EndpointTestUtils.java | 43 ++++-
.../ozone/s3/endpoint/TestBucketTaggingDelete.java | 92 +++++++++
.../ozone/s3/endpoint/TestBucketTaggingGet.java | 126 +++++++++++++
.../ozone/s3/endpoint/TestBucketTaggingPut.java | 206 +++++++++++++++++++++
.../ozone/s3/metrics/TestS3GatewayMetrics.java | 69 +++++++
17 files changed, 979 insertions(+), 12 deletions(-)
diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/buckettagging.robot
b/hadoop-ozone/dist/src/main/smoketest/s3/buckettagging.robot
new file mode 100644
index 00000000000..700d50e2945
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/smoketest/s3/buckettagging.robot
@@ -0,0 +1,89 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+*** Settings ***
+Documentation S3 gateway bucket tagging tests with aws cli
+Library OperatingSystem
+Library String
+Resource ../commonlib.robot
+Resource commonawslib.robot
+Test Timeout 5 minutes
+Suite Setup Setup bucket tagging tests
+
+*** Variables ***
+${ENDPOINT_URL} http://s3g:9878
+${OZONE_TEST} true
+${BUCKET} generated
+${LINK_BUCKET} link-bucket-tagging
+
+*** Keywords ***
+Setup bucket tagging tests
+ Setup s3 tests
+ Setup link bucket for tagging ${LINK_BUCKET}
+
+*** Test Cases ***
+
+Get bucket tagging without tags
+ ${result} = Execute AWSS3APICli and checkrc get-bucket-tagging
--bucket ${BUCKET} 255
+ Should contain ${result}
NoSuchTagSet
+
+Put bucket tagging
+ Execute AWSS3ApiCli put-bucket-tagging
--bucket ${BUCKET} --tagging '{"TagSet": [{ "Key": "tag-key1", "Value":
"tag-value1" }]}'
+
+Get bucket tagging
+ ${result} = Execute AWSS3ApiCli get-bucket-tagging
--bucket ${BUCKET}
+ Should contain ${result}
TagSet
+ ${tagCount} = Execute and checkrc echo '${result}' | jq
'.TagSet | length' 0
+ Should Be Equal ${tagCount}
1
+
+Put bucket tagging overwrites existing tags
+ Execute AWSS3ApiCli put-bucket-tagging
--bucket ${BUCKET} --tagging '{"TagSet": [{ "Key": "tag-key2", "Value":
"tag-value2" },{ "Key": "tag-key3", "Value": "tag-value3" }]}'
+
+Get bucket tagging after overwrite
+ ${result} = Execute AWSS3ApiCli get-bucket-tagging
--bucket ${BUCKET}
+ Should contain ${result}
TagSet
+ ${tagCount} = Execute and checkrc echo '${result}' | jq
'.TagSet | length' 0
+ Should Be Equal ${tagCount}
2
+
+Put bucket tagging on nonexistent bucket
+ ${result} = Execute AWSS3APICli and checkrc put-bucket-tagging
--bucket ${PREFIX}-missing-bucket-tagging --tagging '{"TagSet": [{ "Key":
"tag-key1", "Value": "tag-value1" }]}' 255
+ Should contain ${result}
NoSuchBucket
+
+Delete bucket tagging
+ Execute AWSS3ApiCli delete-bucket-tagging --bucket
${BUCKET}
+
+Get bucket tagging after delete returns NoSuchTagSet
+ ${result} = Execute AWSS3APICli and checkrc get-bucket-tagging
--bucket ${BUCKET} 255
+ Should contain ${result}
NoSuchTagSet
+
+Get bucket tagging on link bucket without tags
+ ${result} = Execute AWSS3APICli and checkrc get-bucket-tagging
--bucket ${LINK_BUCKET} 255
+ Should contain ${result}
NoSuchTagSet
+
+Put bucket tagging on link bucket
+ Execute AWSS3ApiCli put-bucket-tagging
--bucket ${LINK_BUCKET} --tagging '{"TagSet": [{ "Key": "tag-key1", "Value":
"tag-value1" }]}'
+
+Get bucket tagging on link bucket
+ ${result} = Execute AWSS3ApiCli get-bucket-tagging
--bucket ${LINK_BUCKET}
+ Should contain ${result}
TagSet
+ ${tagCount} = Execute and checkrc echo '${result}' | jq
'.TagSet | length' 0
+ Should Be Equal ${tagCount}
1
+
+Delete bucket tagging on link bucket
+ Execute AWSS3ApiCli delete-bucket-tagging
--bucket ${LINK_BUCKET}
+
+Get bucket tagging on link bucket after delete
+ ${result} = Execute AWSS3APICli and checkrc get-bucket-tagging
--bucket ${LINK_BUCKET} 255
+ Should contain ${result}
NoSuchTagSet
diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/commonawslib.robot
b/hadoop-ozone/dist/src/main/smoketest/s3/commonawslib.robot
index 09f86e6b537..16bf579f4df 100644
--- a/hadoop-ozone/dist/src/main/smoketest/s3/commonawslib.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/s3/commonawslib.robot
@@ -143,12 +143,25 @@ Setup s3 tests
Set Global Variable ${OZONE_S3_TESTS_SET_UP} ${TRUE}
Setup links for S3 tests
- ${exists} = Bucket Exists o3://${OM_SERVICE_ID}/s3v/link
+ ${exists} = Bucket Exists s3v/link
Return From Keyword If ${exists}
- Execute ozone sh volume create o3://${OM_SERVICE_ID}/legacy
- Execute ozone sh bucket create --layout ${BUCKET_LAYOUT}
o3://${OM_SERVICE_ID}/legacy/source-bucket
+ Ensure legacy source bucket
Create link link
+Ensure legacy source bucket
+ ${source_exists} = Bucket Exists legacy/source-bucket
+ Return From Keyword If ${source_exists}
+ ${rc} ${output} = Run And Return Rc And Output ozone sh volume
create legacy
+ Run Keyword If ${rc} != 0 Should Contain ${output}
VOLUME_ALREADY_EXISTS
+ Execute ozone sh bucket create --layout ${BUCKET_LAYOUT}
legacy/source-bucket
+
+Setup link bucket for tagging
+ [Arguments] ${link_bucket_name}=link-bucket-tagging
+ ${exists} = Bucket Exists s3v/${link_bucket_name}
+ Return From Keyword If ${exists}
+ Ensure legacy source bucket
+ Create link ${link_bucket_name}
+
Create generated bucket
[Arguments] ${layout}=OBJECT_STORE
${BUCKET} = Create bucket with layout s3v ${layout}
@@ -162,7 +175,7 @@ Create encrypted bucket
Create link
[arguments] ${bucket}
- Execute ozone sh bucket link
o3://${OM_SERVICE_ID}/legacy/source-bucket o3://${OM_SERVICE_ID}/s3v/${bucket}
+ Execute ozone sh bucket link legacy/source-bucket s3v/${bucket}
[return] ${bucket}
Create EC bucket
diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/s3_compatbility_check.sh
b/hadoop-ozone/dist/src/main/smoketest/s3/s3_compatbility_check.sh
index 8b6a4f5a55a..475075fdab6 100755
--- a/hadoop-ozone/dist/src/main/smoketest/s3/s3_compatbility_check.sh
+++ b/hadoop-ozone/dist/src/main/smoketest/s3/s3_compatbility_check.sh
@@ -85,6 +85,7 @@ run_robot_test objectmultidelete
run_robot_test objecthead
run_robot_test MultipartUpload
run_robot_test objecttagging
+run_robot_test buckettagging
run_robot_test objectlist
rebot --outputdir results/ results/*.xml
diff --git
a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java
b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java
index e810a8786b9..c31e3154c32 100644
---
a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java
+++
b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java
@@ -39,13 +39,16 @@
import com.amazonaws.services.s3.model.AbortMultipartUploadRequest;
import com.amazonaws.services.s3.model.AccessControlList;
import com.amazonaws.services.s3.model.Bucket;
+import com.amazonaws.services.s3.model.BucketTaggingConfiguration;
import com.amazonaws.services.s3.model.CanonicalGrantee;
import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest;
import com.amazonaws.services.s3.model.CompleteMultipartUploadResult;
import com.amazonaws.services.s3.model.CopyObjectRequest;
import com.amazonaws.services.s3.model.CopyObjectResult;
import com.amazonaws.services.s3.model.CreateBucketRequest;
+import com.amazonaws.services.s3.model.DeleteBucketTaggingConfigurationRequest;
import com.amazonaws.services.s3.model.GeneratePresignedUrlRequest;
+import com.amazonaws.services.s3.model.GetBucketTaggingConfigurationRequest;
import com.amazonaws.services.s3.model.GetObjectRequest;
import com.amazonaws.services.s3.model.GetObjectTaggingRequest;
import com.amazonaws.services.s3.model.GetObjectTaggingResult;
@@ -72,9 +75,11 @@
import com.amazonaws.services.s3.model.S3Object;
import com.amazonaws.services.s3.model.S3ObjectInputStream;
import com.amazonaws.services.s3.model.S3ObjectSummary;
+import com.amazonaws.services.s3.model.SetBucketTaggingConfigurationRequest;
import com.amazonaws.services.s3.model.SetObjectAclRequest;
import com.amazonaws.services.s3.model.SetObjectTaggingRequest;
import com.amazonaws.services.s3.model.Tag;
+import com.amazonaws.services.s3.model.TagSet;
import com.amazonaws.services.s3.model.UploadPartRequest;
import com.amazonaws.services.s3.model.UploadPartResult;
import com.amazonaws.services.s3.transfer.TransferManager;
@@ -1265,6 +1270,34 @@ public void testGetObjectTaggingReturnsTagsSortedByKey()
{
assertEquals("val2", tagSet.get(1).getValue());
}
+ @Test
+ public void testBucketTaggingPutGetDelete() {
+ final String bucketName = getBucketName();
+ s3Client.createBucket(bucketName);
+
+ // AWS SDK v1 returns null when no bucket tagging is configured.
+ assertNull(s3Client.getBucketTaggingConfiguration(
+ new GetBucketTaggingConfigurationRequest(bucketName)));
+
+ TagSet tagSet = new TagSet();
+ tagSet.setTag("tag-key1", "tag-value1");
+ tagSet.setTag("tag-key2", "tag-value2");
+ s3Client.setBucketTaggingConfiguration(new
SetBucketTaggingConfigurationRequest(bucketName,
+ new BucketTaggingConfiguration(Collections.singletonList(tagSet))));
+
+ BucketTaggingConfiguration taggingConfiguration =
+ s3Client.getBucketTaggingConfiguration(new
GetBucketTaggingConfigurationRequest(bucketName));
+ Map<String, String> actualTags =
taggingConfiguration.getTagSet().getAllTags();
+ assertEquals(2, actualTags.size());
+ assertEquals("tag-value1", actualTags.get("tag-key1"));
+ assertEquals("tag-value2", actualTags.get("tag-key2"));
+
+ s3Client.deleteBucketTaggingConfiguration(new
DeleteBucketTaggingConfigurationRequest(bucketName));
+
+ assertNull(s3Client.getBucketTaggingConfiguration(
+ new GetBucketTaggingConfigurationRequest(bucketName)));
+ }
+
@Test
public void testGetObjectWithoutETag() throws Exception {
// Object uploaded using other protocols (e.g. ofs / ozone cli) will not
diff --git
a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java
b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java
index d807be57483..90c850abff5 100644
---
a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java
+++
b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java
@@ -116,10 +116,13 @@
import software.amazon.awssdk.services.s3.model.CreateMultipartUploadResponse;
import software.amazon.awssdk.services.s3.model.Delete;
import software.amazon.awssdk.services.s3.model.DeleteBucketRequest;
+import software.amazon.awssdk.services.s3.model.DeleteBucketTaggingRequest;
import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
import software.amazon.awssdk.services.s3.model.DeleteObjectTaggingRequest;
import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest;
import software.amazon.awssdk.services.s3.model.GetBucketAclRequest;
+import software.amazon.awssdk.services.s3.model.GetBucketTaggingRequest;
+import software.amazon.awssdk.services.s3.model.GetBucketTaggingResponse;
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
import software.amazon.awssdk.services.s3.model.GetObjectResponse;
import software.amazon.awssdk.services.s3.model.GetObjectTaggingRequest;
@@ -139,6 +142,7 @@
import software.amazon.awssdk.services.s3.model.NoSuchKeyException;
import software.amazon.awssdk.services.s3.model.ObjectIdentifier;
import software.amazon.awssdk.services.s3.model.PutBucketAclRequest;
+import software.amazon.awssdk.services.s3.model.PutBucketTaggingRequest;
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
import software.amazon.awssdk.services.s3.model.PutObjectResponse;
import software.amazon.awssdk.services.s3.model.PutObjectTaggingRequest;
@@ -305,6 +309,40 @@ public void testGetObjectTaggingReturnsTagsSortedByKey() {
assertEquals("val2", tagSet.get(1).value());
}
+ @Test
+ public void testBucketTaggingPutGetDelete() {
+ final String bucketName = getBucketName();
+ s3Client.createBucket(b -> b.bucket(bucketName));
+
+ S3Exception noTags = assertThrows(S3Exception.class,
+ () ->
s3Client.getBucketTagging(GetBucketTaggingRequest.builder().bucket(bucketName).build()));
+ assertEquals(404, noTags.statusCode());
+ assertEquals("NoSuchTagSet", noTags.awsErrorDetails().errorCode());
+
+ List<Tag> tags = Arrays.asList(
+ Tag.builder().key("tag-key1").value("tag-value1").build(),
+ Tag.builder().key("tag-key2").value("tag-value2").build());
+ s3Client.putBucketTagging(PutBucketTaggingRequest.builder()
+ .bucket(bucketName)
+ .tagging(Tagging.builder().tagSet(tags).build())
+ .build());
+
+ GetBucketTaggingResponse taggingResult = s3Client.getBucketTagging(
+ GetBucketTaggingRequest.builder().bucket(bucketName).build());
+ Map<String, String> actualTags = taggingResult.tagSet().stream()
+ .collect(Collectors.toMap(Tag::key, Tag::value));
+ assertEquals(2, actualTags.size());
+ assertEquals("tag-value1", actualTags.get("tag-key1"));
+ assertEquals("tag-value2", actualTags.get("tag-key2"));
+
+
s3Client.deleteBucketTagging(DeleteBucketTaggingRequest.builder().bucket(bucketName).build());
+
+ S3Exception afterDelete = assertThrows(S3Exception.class,
+ () ->
s3Client.getBucketTagging(GetBucketTaggingRequest.builder().bucket(bucketName).build()));
+ assertEquals(404, afterDelete.statusCode());
+ assertEquals("NoSuchTagSet", afterDelete.awsErrorDetails().errorCode());
+ }
+
@Test
public void testPutObjectIfNoneMatch() {
final String bucketName = getBucketName();
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/audit/S3GAction.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/audit/S3GAction.java
index 969b2c04809..0ab5a1004ba 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/audit/S3GAction.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/audit/S3GAction.java
@@ -55,6 +55,9 @@ public enum S3GAction implements AuditAction {
GET_OBJECT_TAGGING,
PUT_OBJECT_TAGGING,
DELETE_OBJECT_TAGGING,
+ GET_BUCKET_TAGGING,
+ PUT_BUCKET_TAGGING,
+ DELETE_BUCKET_TAGGING,
PUT_OBJECT_ACL;
@Override
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketCrudHandler.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketCrudHandler.java
index a2698b3098e..41a19231a82 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketCrudHandler.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketCrudHandler.java
@@ -34,7 +34,7 @@
*
* This handler processes bucket-level requests that do not target
* specific subresources (such as {@code ?acl}, {@code ?uploads},
- * or {@code ?delete}), which are handled by dedicated handlers.
+ * {@code ?delete} or {@code ?tagging}), which are handled by dedicated
handlers.
*
* This handler extends EndpointBase to inherit all required functionality
* (configuration, headers, request context, audit logging, metrics, etc.).
@@ -47,7 +47,8 @@ public class BucketCrudHandler extends BucketOperationHandler
{
private boolean shouldHandle() {
return queryParams().get(QueryParams.ACL) == null
&& queryParams().get(QueryParams.UPLOADS) == null
- && queryParams().get(QueryParams.DELETE) == null;
+ && queryParams().get(QueryParams.DELETE) == null
+ && queryParams().get(QueryParams.TAGGING) == null;
}
/**
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java
index d35da257cd0..383dc753f19 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java
@@ -424,6 +424,7 @@ protected void init() {
.add(new BucketGetLocationHandler())
.add(new BucketAclHandler())
.add(new ListMultipartUploadsHandler())
+ .add(new BucketTaggingHandler())
.add(new BucketCrudHandler())
.add(this)
.build();
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketTaggingHandler.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketTaggingHandler.java
new file mode 100644
index 00000000000..10d41558ec9
--- /dev/null
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketTaggingHandler.java
@@ -0,0 +1,152 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.s3.endpoint;
+
+import static org.apache.hadoop.ozone.s3.util.S3Consts.TAG_BUCKET_NUM_LIMIT;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Map;
+import java.util.function.Supplier;
+import javax.ws.rs.HttpMethod;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import org.apache.hadoop.ozone.audit.S3GAction;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes;
+import org.apache.hadoop.ozone.s3.exception.OS3Exception;
+import org.apache.hadoop.ozone.s3.exception.S3ErrorTable;
+import org.apache.hadoop.ozone.s3.util.S3Consts;
+import org.apache.ratis.util.MemoizedSupplier;
+
+/**
+ * S3 bucket tagging (?tagging).
+ */
+public class BucketTaggingHandler extends BucketOperationHandler {
+
+ private static final Supplier<MessageUnmarshaller<S3Tagging>> UNMARSHALLER =
+ MemoizedSupplier.valueOf(() -> new
MessageUnmarshaller<>(S3Tagging.class));
+
+ @Override
+ public Response handlePutRequest(S3RequestContext context, String
bucketName, InputStream body)
+ throws IOException, OS3Exception {
+ S3GAction action = resolveAction();
+ if (action == null) {
+ return null;
+ }
+ context.setAction(action);
+
+ try {
+ S3Tagging tagging;
+ try {
+ tagging = UNMARSHALLER.get().readFrom(body);
+ tagging.validate();
+ } catch (Exception ex) {
+ OS3Exception exception =
S3ErrorTable.newError(S3ErrorTable.MALFORMED_XML, bucketName, ex);
+ exception.setErrorMessage(exception.getErrorMessage() + ". " +
ex.getMessage());
+ throw exception;
+ }
+
+ Map<String, String> tags = validateAndGetTagging(
+ tagging.getTagSet().getTags(),
+ S3Tagging.Tag::getKey,
+ S3Tagging.Tag::getValue,
+ TAG_BUCKET_NUM_LIMIT
+ );
+
+ context.getVolume().getBucket(bucketName).putBucketTagging(tags);
+
+ getMetrics().updatePutBucketTaggingSuccessStats(context.getStartNanos());
+
+ return Response.ok().build();
+ } catch (Exception e) {
+ getMetrics().updatePutBucketTaggingFailureStats(context.getStartNanos());
+ throw e;
+ }
+ }
+
+ @Override
+ public Response handleDeleteRequest(S3RequestContext context, String
bucketName)
+ throws IOException, OS3Exception {
+ S3GAction action = resolveAction();
+ if (action == null) {
+ return null;
+ }
+ context.setAction(action);
+
+ try {
+ context.getVolume().getBucket(bucketName).deleteBucketTagging();
+
getMetrics().updateDeleteBucketTaggingSuccessStats(context.getStartNanos());
+ return Response.noContent().build();
+ } catch (OMException ex) {
+
getMetrics().updateDeleteBucketTaggingFailureStats(context.getStartNanos());
+ if (ex.getResult() == ResultCodes.BUCKET_NOT_FOUND) {
+ throw S3ErrorTable.newError(S3ErrorTable.NO_SUCH_BUCKET, bucketName);
+ }
+ throw ex;
+ } catch (IOException | RuntimeException ex) {
+
getMetrics().updateDeleteBucketTaggingFailureStats(context.getStartNanos());
+ throw ex;
+ }
+ }
+
+ @Override
+ public Response handleGetRequest(S3RequestContext context, String bucketName)
+ throws IOException, OS3Exception {
+ S3GAction action = resolveAction();
+ if (action == null) {
+ return null;
+ }
+ context.setAction(action);
+
+ try {
+ Map<String, String> tagMap =
context.getVolume().getBucket(bucketName).getBucketTagging();
+ if (tagMap.isEmpty()) {
+ throw S3ErrorTable.newError(S3ErrorTable.NO_SUCH_TAG_SET, bucketName);
+ }
+ getMetrics().updateGetBucketTaggingSuccessStats(context.getStartNanos());
+ return Response.ok(S3Tagging.fromMap(tagMap),
MediaType.APPLICATION_XML_TYPE).build();
+ } catch (OMException ex) {
+ getMetrics().updateGetBucketTaggingFailureStats(context.getStartNanos());
+ if (ex.getResult() == ResultCodes.BUCKET_NOT_FOUND) {
+ throw S3ErrorTable.newError(S3ErrorTable.NO_SUCH_BUCKET, bucketName);
+ }
+ throw ex;
+ } catch (Exception e) {
+ getMetrics().updateGetBucketTaggingFailureStats(context.getStartNanos());
+ throw e;
+ }
+ }
+
+ private S3GAction resolveAction() {
+ if (queryParams().get(S3Consts.QueryParams.TAGGING) == null) {
+ return null;
+ }
+
+ switch (getContext().getMethod()) {
+ case HttpMethod.DELETE:
+ return S3GAction.DELETE_BUCKET_TAGGING;
+ case HttpMethod.GET:
+ return S3GAction.GET_BUCKET_TAGGING;
+ case HttpMethod.PUT:
+ return S3GAction.PUT_BUCKET_TAGGING;
+ default:
+ return null;
+ }
+ }
+}
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java
index 052dd34faa5..452303dc4c0 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java
@@ -403,6 +403,15 @@ protected static <KV> Map<String, String>
validateAndGetTagging(
List<KV> tagList,
Function<KV, String> getTagKey,
Function<KV, String> getTagValue
+ ) throws OS3Exception {
+ return validateAndGetTagging(tagList, getTagKey, getTagValue,
TAG_NUM_LIMIT);
+ }
+
+ protected static <KV> Map<String, String> validateAndGetTagging(
+ List<KV> tagList,
+ Function<KV, String> getTagKey,
+ Function<KV, String> getTagValue,
+ int maxTagCount
) throws OS3Exception {
final Map<String, String> tags = new HashMap<>();
for (KV tagPair : tagList) {
@@ -455,19 +464,18 @@ protected static <KV> Map<String, String>
validateAndGetTagging(
final String previous = tags.put(tagKey, tagValue);
if (previous != null) {
- // Tags that are associated with an object must have unique tag keys
- // Reject request if the same key is used twice on the same resource
+ // Tags must have unique keys on the same resource (object or bucket).
OS3Exception ex = S3ErrorTable.newError(INVALID_TAG, tagKey);
ex.setErrorMessage("There are tags with duplicate tag keys, tag keys
should be unique");
throw ex;
}
}
- if (tags.size() > TAG_NUM_LIMIT) {
- // You can associate up to 10 tags with an object.
+ if (tags.size() > maxTagCount) {
+ // You can associate up to 10 tags with an object and up to 50 tags with
a bucket.
OS3Exception ex = S3ErrorTable.newError(INVALID_TAG, TAG_HEADER);
ex.setErrorMessage("The number of tags " + tags.size() +
- " exceeded the maximum number of tags of " + TAG_NUM_LIMIT);
+ " exceeded the maximum number of tags of " + maxTagCount);
throw ex;
}
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/metrics/S3GatewayMetrics.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/metrics/S3GatewayMetrics.java
index 7f2b2f67863..160102e7290 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/metrics/S3GatewayMetrics.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/metrics/S3GatewayMetrics.java
@@ -101,6 +101,12 @@ public final class S3GatewayMetrics implements Closeable,
MetricsSource {
private @Metric MutableCounterLong putObjectTaggingFailure;
private @Metric MutableCounterLong deleteObjectTaggingSuccess;
private @Metric MutableCounterLong deleteObjectTaggingFailure;
+ private @Metric MutableCounterLong getBucketTaggingSuccess;
+ private @Metric MutableCounterLong getBucketTaggingFailure;
+ private @Metric MutableCounterLong putBucketTaggingSuccess;
+ private @Metric MutableCounterLong putBucketTaggingFailure;
+ private @Metric MutableCounterLong deleteBucketTaggingSuccess;
+ private @Metric MutableCounterLong deleteBucketTaggingFailure;
private @Metric MutableCounterLong putObjectAclSuccess;
private @Metric MutableCounterLong putObjectAclFailure;
@@ -272,6 +278,24 @@ public final class S3GatewayMetrics implements Closeable,
MetricsSource {
@Metric(about = "Latency for failing to delete object tagging of a key in
nanoseconds")
private PerformanceMetrics deleteObjectTaggingFailureLatencyNs;
+ @Metric(about = "Latency for successful get bucket tagging in nanoseconds")
+ private PerformanceMetrics getBucketTaggingSuccessLatencyNs;
+
+ @Metric(about = "Latency for failing to get bucket tagging in nanoseconds")
+ private PerformanceMetrics getBucketTaggingFailureLatencyNs;
+
+ @Metric(about = "Latency for successful put bucket tagging in nanoseconds")
+ private PerformanceMetrics putBucketTaggingSuccessLatencyNs;
+
+ @Metric(about = "Latency for failing to put bucket tagging in nanoseconds")
+ private PerformanceMetrics putBucketTaggingFailureLatencyNs;
+
+ @Metric(about = "Latency for successful delete bucket tagging in
nanoseconds")
+ private PerformanceMetrics deleteBucketTaggingSuccessLatencyNs;
+
+ @Metric(about = "Latency for failing to delete bucket tagging in
nanoseconds")
+ private PerformanceMetrics deleteBucketTaggingFailureLatencyNs;
+
@Metric(about = "Latency for successfully setting an S3 object ACL " +
"in nanoseconds")
private PerformanceMetrics putObjectAclSuccessLatencyNs;
@@ -427,6 +451,18 @@ public void getMetrics(MetricsCollector collector, boolean
all) {
deleteObjectTaggingSuccessLatencyNs.snapshot(recordBuilder, true);
deleteObjectTaggingFailure.snapshot(recordBuilder, true);
deleteObjectTaggingFailureLatencyNs.snapshot(recordBuilder, true);
+ getBucketTaggingSuccess.snapshot(recordBuilder, true);
+ getBucketTaggingSuccessLatencyNs.snapshot(recordBuilder, true);
+ getBucketTaggingFailure.snapshot(recordBuilder, true);
+ getBucketTaggingFailureLatencyNs.snapshot(recordBuilder, true);
+ putBucketTaggingSuccess.snapshot(recordBuilder, true);
+ putBucketTaggingSuccessLatencyNs.snapshot(recordBuilder, true);
+ putBucketTaggingFailure.snapshot(recordBuilder, true);
+ putBucketTaggingFailureLatencyNs.snapshot(recordBuilder, true);
+ deleteBucketTaggingSuccess.snapshot(recordBuilder, true);
+ deleteBucketTaggingSuccessLatencyNs.snapshot(recordBuilder, true);
+ deleteBucketTaggingFailure.snapshot(recordBuilder, true);
+ deleteBucketTaggingFailureLatencyNs.snapshot(recordBuilder, true);
putObjectAclSuccess.snapshot(recordBuilder, true);
putObjectAclSuccessLatencyNs.snapshot(recordBuilder, true);
putObjectAclFailure.snapshot(recordBuilder, true);
@@ -682,6 +718,36 @@ public void updateDeleteObjectTaggingFailureStats(long
startNanos) {
this.deleteObjectTaggingFailureLatencyNs.add(Time.monotonicNowNanos() -
startNanos);
}
+ public void updateGetBucketTaggingSuccessStats(long startNanos) {
+ this.getBucketTaggingSuccess.incr();
+ this.getBucketTaggingSuccessLatencyNs.add(Time.monotonicNowNanos() -
startNanos);
+ }
+
+ public void updateGetBucketTaggingFailureStats(long startNanos) {
+ this.getBucketTaggingFailure.incr();
+ this.getBucketTaggingFailureLatencyNs.add(Time.monotonicNowNanos() -
startNanos);
+ }
+
+ public void updatePutBucketTaggingSuccessStats(long startNanos) {
+ this.putBucketTaggingSuccess.incr();
+ this.putBucketTaggingSuccessLatencyNs.add(Time.monotonicNowNanos() -
startNanos);
+ }
+
+ public void updatePutBucketTaggingFailureStats(long startNanos) {
+ this.putBucketTaggingFailure.incr();
+ this.putBucketTaggingFailureLatencyNs.add(Time.monotonicNowNanos() -
startNanos);
+ }
+
+ public void updateDeleteBucketTaggingSuccessStats(long startNanos) {
+ this.deleteBucketTaggingSuccess.incr();
+ this.deleteBucketTaggingSuccessLatencyNs.add(Time.monotonicNowNanos() -
startNanos);
+ }
+
+ public void updateDeleteBucketTaggingFailureStats(long startNanos) {
+ this.deleteBucketTaggingFailure.incr();
+ this.deleteBucketTaggingFailureLatencyNs.add(Time.monotonicNowNanos() -
startNanos);
+ }
+
public void updatePutObjectAclSuccessStats(long startNanos) {
this.putObjectAclSuccess.incr();
this.putObjectAclSuccessLatencyNs.add(Time.monotonicNowNanos() -
startNanos);
@@ -857,6 +923,30 @@ public long getDeleteObjectTaggingFailure() {
return deleteObjectTaggingFailure.value();
}
+ public long getGetBucketTaggingSuccess() {
+ return getBucketTaggingSuccess.value();
+ }
+
+ public long getGetBucketTaggingFailure() {
+ return getBucketTaggingFailure.value();
+ }
+
+ public long getPutBucketTaggingSuccess() {
+ return putBucketTaggingSuccess.value();
+ }
+
+ public long getPutBucketTaggingFailure() {
+ return putBucketTaggingFailure.value();
+ }
+
+ public long getDeleteBucketTaggingSuccess() {
+ return deleteBucketTaggingSuccess.value();
+ }
+
+ public long getDeleteBucketTaggingFailure() {
+ return deleteBucketTaggingFailure.value();
+ }
+
private long updateAndGetStats(PerformanceMetrics metric, long startNanos) {
long value = Time.monotonicNowNanos() - startNanos;
metric.add(value);
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java
index abc654f916e..92266adbcab 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java
@@ -87,6 +87,7 @@ public final class S3Consts {
public static final String TAG_COUNT_HEADER = "x-amz-tagging-count";
public static final String AWS_TAG_PREFIX = "aws:";
+ // tag limit for object
public static final int TAG_NUM_LIMIT = 10;
public static final int TAG_KEY_LENGTH_LIMIT = 128;
public static final int TAG_VALUE_LENGTH_LIMIT = 256;
@@ -117,6 +118,9 @@ public final class S3Consts {
public static final String DEFAULT_S3_REGION = "us-east-1";
public static final String S3_EXPRESS_SERVICE = "s3express";
+ // tag limit for bucket
+ public static final int TAG_BUCKET_NUM_LIMIT = 50;
+
//Never Constructed
private S3Consts() {
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/EndpointTestUtils.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/EndpointTestUtils.java
index 8827150069f..1e48f6c9be9 100644
---
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/EndpointTestUtils.java
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/EndpointTestUtils.java
@@ -177,6 +177,47 @@ public static Response deleteTagging(
return subject.delete(bucket, key);
}
+ /**
+ * Get bucket tags (?tagging).
+ */
+ public static Response getBucketTagging(
+ BucketEndpoint subject,
+ String bucket
+ ) throws IOException, OS3Exception {
+ subject.queryParamsForTest().set(S3Consts.QueryParams.TAGGING, "");
+ when(subject.getContext().getMethod()).thenReturn(HttpMethod.GET);
+ return subject.get(bucket);
+ }
+
+ /**
+ * Add tagging on bucket (?tagging).
+ */
+ public static Response putBucketTagging(
+ BucketEndpoint subject, String bucket, String content)
+ throws IOException, OS3Exception {
+ subject.queryParamsForTest().set(S3Consts.QueryParams.TAGGING, "");
+ when(subject.getContext().getMethod()).thenReturn(HttpMethod.PUT);
+ setLengthHeader(subject, content);
+
+ if (content == null) {
+ return subject.put(bucket, null);
+ } else {
+ try (ByteArrayInputStream body = new
ByteArrayInputStream(content.getBytes(UTF_8))) {
+ return subject.put(bucket, body);
+ }
+ }
+ }
+
+ /**
+ * Delete bucket tags (?tagging).
+ */
+ public static Response deleteBucketTagging(
+ BucketEndpoint subject, String bucket) throws IOException, OS3Exception {
+ subject.queryParamsForTest().set(S3Consts.QueryParams.TAGGING, "");
+ when(subject.getContext().getMethod()).thenReturn(HttpMethod.DELETE);
+ return subject.delete(bucket);
+ }
+
/** Initiate multipart upload.
* @return upload ID */
public static String initiateMultipartUpload(ObjectEndpoint subject, String
bucket, String key)
@@ -279,7 +320,7 @@ public static OS3Exception assertErrorResponse(S3ErrorTable
expected, CheckedSup
return actual;
}
- private static void setLengthHeader(ObjectEndpoint subject, String content) {
+ private static void setLengthHeader(EndpointBase subject, String content) {
when(subject.getHeaders().getHeaderString(HttpHeaders.CONTENT_LENGTH))
.thenReturn(String.valueOf(contentLength(content)));
}
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketTaggingDelete.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketTaggingDelete.java
new file mode 100644
index 00000000000..b8ad60a245e
--- /dev/null
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketTaggingDelete.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.s3.endpoint;
+
+import static java.net.HttpURLConnection.HTTP_NO_CONTENT;
+import static
org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.assertErrorResponse;
+import static
org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.assertSucceeds;
+import static
org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.deleteBucketTagging;
+import static
org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.putBucketTagging;
+import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.NO_SUCH_BUCKET;
+import static org.apache.hadoop.ozone.s3.util.S3Consts.STORAGE_CLASS_HEADER;
+import static org.apache.hadoop.ozone.s3.util.S3Consts.X_AMZ_CONTENT_SHA256;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.io.IOException;
+import javax.ws.rs.core.HttpHeaders;
+import javax.ws.rs.core.Response;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneClientStub;
+import org.apache.hadoop.ozone.s3.exception.OS3Exception;
+import org.apache.hadoop.ozone.s3.util.S3Consts;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.mockito.Mockito;
+
+/**
+ * Tests for DeleteBucketTagging.
+ */
+public class TestBucketTaggingDelete {
+
+ private static final String BUCKET_NAME = "b1";
+ private BucketEndpoint rest;
+ private OzoneClient client;
+
+ private static String twoTagsBody() {
+ return
+ "<Tagging xmlns=\"" + S3Consts.S3_XML_NAMESPACE + "\">" +
+ " <TagSet>" +
+ " <Tag><Key>tag1</Key><Value>value1</Value></Tag>" +
+ " <Tag><Key>tag2</Key><Value>value2</Value></Tag>" +
+ " </TagSet>" +
+ "</Tagging>";
+ }
+
+ @BeforeEach
+ public void init() throws OS3Exception, IOException {
+ client = new OzoneClientStub();
+ client.getObjectStore().createS3Bucket(BUCKET_NAME);
+
+ HttpHeaders headers = Mockito.mock(HttpHeaders.class);
+ Mockito.when(headers.getHeaderString(X_AMZ_CONTENT_SHA256))
+ .thenReturn("UNSIGNED-PAYLOAD");
+ Mockito.when(headers.getHeaderString(STORAGE_CLASS_HEADER))
+ .thenReturn("STANDARD");
+
+ rest = EndpointBuilder.newBucketEndpointBuilder()
+ .setClient(client)
+ .setHeaders(headers)
+ .build();
+
+ assertSucceeds(() -> putBucketTagging(rest, BUCKET_NAME, twoTagsBody()));
+ }
+
+ @Test
+ public void testDeleteBucketTagging() throws IOException, OS3Exception {
+ Response response = deleteBucketTagging(rest, BUCKET_NAME);
+ assertEquals(HTTP_NO_CONTENT, response.getStatus());
+
+
assertTrue(client.getObjectStore().getS3Bucket(BUCKET_NAME).getBucketTagging().isEmpty());
+ }
+
+ @Test
+ public void testDeleteBucketTaggingNoBucketFound() {
+ assertErrorResponse(NO_SUCH_BUCKET, () -> deleteBucketTagging(rest,
"nonexistent"));
+ }
+}
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketTaggingGet.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketTaggingGet.java
new file mode 100644
index 00000000000..d685e2c7ebf
--- /dev/null
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketTaggingGet.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.s3.endpoint;
+
+import static java.net.HttpURLConnection.HTTP_OK;
+import static
org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.assertErrorResponse;
+import static
org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.assertSucceeds;
+import static
org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.getBucketTagging;
+import static
org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.putBucketTagging;
+import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.NO_SUCH_BUCKET;
+import static
org.apache.hadoop.ozone.s3.exception.S3ErrorTable.NO_SUCH_TAG_SET;
+import static org.apache.hadoop.ozone.s3.util.S3Consts.STORAGE_CLASS_HEADER;
+import static org.apache.hadoop.ozone.s3.util.S3Consts.X_AMZ_CONTENT_SHA256;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.fail;
+
+import java.io.IOException;
+import javax.ws.rs.core.HttpHeaders;
+import javax.ws.rs.core.Response;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneClientStub;
+import org.apache.hadoop.ozone.s3.endpoint.S3Tagging.Tag;
+import org.apache.hadoop.ozone.s3.exception.OS3Exception;
+import org.apache.hadoop.ozone.s3.util.S3Consts;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.mockito.Mockito;
+
+/**
+ * Tests for GetBucketTagging.
+ */
+public class TestBucketTaggingGet {
+
+ private static final String BUCKET_NAME = "b1";
+ private BucketEndpoint rest;
+
+ private static String twoTagsBody() {
+ return
+ "<Tagging xmlns=\"" + S3Consts.S3_XML_NAMESPACE + "\">" +
+ " <TagSet>" +
+ " <Tag><Key>tag1</Key><Value>value1</Value></Tag>" +
+ " <Tag><Key>tag2</Key><Value>value2</Value></Tag>" +
+ " </TagSet>" +
+ "</Tagging>";
+ }
+
+ @BeforeEach
+ public void init() throws Exception {
+ OzoneClient client = new OzoneClientStub();
+ client.getObjectStore().createS3Bucket(BUCKET_NAME);
+
+ HttpHeaders headers = Mockito.mock(HttpHeaders.class);
+ Mockito.when(headers.getHeaderString(X_AMZ_CONTENT_SHA256))
+ .thenReturn("UNSIGNED-PAYLOAD");
+ Mockito.when(headers.getHeaderString(STORAGE_CLASS_HEADER))
+ .thenReturn("STANDARD");
+
+ rest = EndpointBuilder.newBucketEndpointBuilder()
+ .setClient(client)
+ .setHeaders(headers)
+ .build();
+
+ assertSucceeds(() -> putBucketTagging(rest, BUCKET_NAME, twoTagsBody()));
+ }
+
+ @Test
+ public void testGetBucketTagging() throws IOException, OS3Exception {
+ Response response = getBucketTagging(rest, BUCKET_NAME);
+
+ assertEquals(HTTP_OK, response.getStatus());
+ S3Tagging s3Tagging = (S3Tagging) response.getEntity();
+ assertNotNull(s3Tagging);
+ assertNotNull(s3Tagging.getTagSet());
+ assertEquals(2, s3Tagging.getTagSet().getTags().size());
+ for (Tag tag : s3Tagging.getTagSet().getTags()) {
+ if (tag.getKey().equals("tag1")) {
+ assertEquals("value1", tag.getValue());
+ } else if (tag.getKey().equals("tag2")) {
+ assertEquals("value2", tag.getValue());
+ } else {
+ fail("Unknown tag found");
+ }
+ }
+ }
+
+ @Test
+ public void testGetBucketTaggingNoTagSet() throws Exception {
+ OzoneClient client = new OzoneClientStub();
+ String emptyBucket = "empty-tags-bucket";
+ client.getObjectStore().createS3Bucket(emptyBucket);
+
+ HttpHeaders headers = Mockito.mock(HttpHeaders.class);
+ Mockito.when(headers.getHeaderString(X_AMZ_CONTENT_SHA256))
+ .thenReturn("UNSIGNED-PAYLOAD");
+ Mockito.when(headers.getHeaderString(STORAGE_CLASS_HEADER))
+ .thenReturn("STANDARD");
+
+ BucketEndpoint endpoint = EndpointBuilder.newBucketEndpointBuilder()
+ .setClient(client)
+ .setHeaders(headers)
+ .build();
+
+ assertErrorResponse(NO_SUCH_TAG_SET, () -> getBucketTagging(endpoint,
emptyBucket));
+ }
+
+ @Test
+ public void testGetBucketTaggingNoBucketFound() {
+ assertErrorResponse(NO_SUCH_BUCKET, () -> getBucketTagging(rest,
"nonexistent"));
+ }
+}
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketTaggingPut.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketTaggingPut.java
new file mode 100644
index 00000000000..6d19c39f20d
--- /dev/null
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketTaggingPut.java
@@ -0,0 +1,206 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.s3.endpoint;
+
+import static
org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.assertErrorResponse;
+import static
org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.assertSucceeds;
+import static
org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.putBucketTagging;
+import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.INVALID_TAG;
+import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.MALFORMED_XML;
+import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.NO_SUCH_BUCKET;
+import static org.apache.hadoop.ozone.s3.util.S3Consts.STORAGE_CLASS_HEADER;
+import static org.apache.hadoop.ozone.s3.util.S3Consts.TAG_BUCKET_NUM_LIMIT;
+import static org.apache.hadoop.ozone.s3.util.S3Consts.TAG_KEY_LENGTH_LIMIT;
+import static org.apache.hadoop.ozone.s3.util.S3Consts.TAG_VALUE_LENGTH_LIMIT;
+import static org.apache.hadoop.ozone.s3.util.S3Consts.X_AMZ_CONTENT_SHA256;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import com.google.common.collect.ImmutableMap;
+import java.io.IOException;
+import java.util.Map;
+import javax.ws.rs.core.HttpHeaders;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneClientStub;
+import org.apache.hadoop.ozone.s3.exception.OS3Exception;
+import org.apache.hadoop.ozone.s3.util.S3Consts;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Tests for PutBucketTagging.
+ */
+public class TestBucketTaggingPut {
+
+ private static final String BUCKET_NAME = "b1";
+ private BucketEndpoint rest;
+
+ @BeforeEach
+ public void init() throws OS3Exception, IOException {
+ OzoneClient client = new OzoneClientStub();
+ client.getObjectStore().createS3Bucket(BUCKET_NAME);
+
+ HttpHeaders headers = mock(HttpHeaders.class);
+
when(headers.getHeaderString(X_AMZ_CONTENT_SHA256)).thenReturn("UNSIGNED-PAYLOAD");
+ when(headers.getHeaderString(STORAGE_CLASS_HEADER)).thenReturn("STANDARD");
+
+ rest = EndpointBuilder.newBucketEndpointBuilder()
+ .setClient(client)
+ .setHeaders(headers)
+ .build();
+ }
+
+ @Test
+ public void testPutBucketTaggingWithEmptyBody() {
+ assertErrorResponse(MALFORMED_XML, () -> putBucketTagging(rest,
BUCKET_NAME, ""));
+ }
+
+ @Test
+ public void testPutValidBucketTagging() throws Exception {
+ assertSucceeds(() -> putBucketTagging(rest, BUCKET_NAME, twoTagsBody()));
+ Map<String, String> tags =
+
rest.getClient().getObjectStore().getS3Bucket(BUCKET_NAME).getBucketTagging();
+ assertThat(tags).containsExactlyInAnyOrderEntriesOf(
+ ImmutableMap.of("tag1", "value1", "tag2", "value2"));
+ }
+
+ @Test
+ public void testPutInvalidBucketTaggingXml() {
+ assertErrorResponse(MALFORMED_XML, () -> putBucketTagging(rest,
BUCKET_NAME, null));
+ assertErrorResponse(MALFORMED_XML, () -> putBucketTagging(rest,
BUCKET_NAME, invalidXmlStructure()));
+ assertErrorResponse(MALFORMED_XML, () -> putBucketTagging(rest,
BUCKET_NAME, noTagSet()));
+ assertErrorResponse(MALFORMED_XML, () -> putBucketTagging(rest,
BUCKET_NAME, emptyTags()));
+ assertErrorResponse(MALFORMED_XML, () -> putBucketTagging(rest,
BUCKET_NAME, tagKeyNotSpecified()));
+ assertErrorResponse(MALFORMED_XML, () -> putBucketTagging(rest,
BUCKET_NAME, tagValueNotSpecified()));
+ }
+
+ @Test
+ public void testPutBucketTaggingDuplicateKeys() {
+ assertErrorResponse(INVALID_TAG, () -> putBucketTagging(rest, BUCKET_NAME,
duplicateTagKeys()));
+ }
+
+ @Test
+ public void testPutBucketTaggingAwsPrefixKey() {
+ assertErrorResponse(INVALID_TAG, () -> putBucketTagging(rest, BUCKET_NAME,
awsPrefixTag()));
+ }
+
+ @Test
+ public void testPutBucketTaggingKeyTooLong() {
+ String longKey = StringUtils.repeat("k", TAG_KEY_LENGTH_LIMIT + 1);
+ assertErrorResponse(INVALID_TAG, () -> putBucketTagging(rest, BUCKET_NAME,
singleTag(longKey, "v")));
+ }
+
+ @Test
+ public void testPutBucketTaggingValueTooLong() {
+ String longValue = StringUtils.repeat("v", TAG_VALUE_LENGTH_LIMIT + 1);
+ assertErrorResponse(INVALID_TAG, () -> putBucketTagging(rest, BUCKET_NAME,
singleTag("k", longValue)));
+ }
+
+ @Test
+ public void testPutBucketTaggingExceedsLimit() {
+ StringBuilder tags = new StringBuilder("<Tagging xmlns=\"" +
S3Consts.S3_XML_NAMESPACE + "\"><TagSet>");
+ for (int i = 0; i < TAG_BUCKET_NUM_LIMIT + 1; i++) {
+
tags.append("<Tag><Key>tag").append(i).append("</Key><Value>value").append(i)
+ .append("</Value></Tag>");
+ }
+ tags.append("</TagSet></Tagging>");
+ assertErrorResponse(INVALID_TAG, () -> putBucketTagging(rest, BUCKET_NAME,
tags.toString()));
+ }
+
+ @Test
+ public void testPutBucketTaggingNoBucketFound() {
+ assertErrorResponse(NO_SUCH_BUCKET, () -> putBucketTagging(rest,
"nonexistent", twoTagsBody()));
+ }
+
+ private static String twoTagsBody() {
+ return
+ "<Tagging xmlns=\"" + S3Consts.S3_XML_NAMESPACE + "\">" +
+ " <TagSet>" +
+ " <Tag>" +
+ " <Key>tag1</Key>" +
+ " <Value>value1</Value>" +
+ " </Tag>" +
+ " <Tag>" +
+ " <Key>tag2</Key>" +
+ " <Value>value2</Value>" +
+ " </Tag>" +
+ " </TagSet>" +
+ "</Tagging>";
+ }
+
+ private static String singleTag(String key, String value) {
+ return
+ "<Tagging xmlns=\"" + S3Consts.S3_XML_NAMESPACE + "\">" +
+ " <TagSet>" +
+ " <Tag>" +
+ " <Key>" + key + "</Key>" +
+ " <Value>" + value + "</Value>" +
+ " </Tag>" +
+ " </TagSet>" +
+ "</Tagging>";
+ }
+
+ private static String duplicateTagKeys() {
+ return
+ "<Tagging xmlns=\"" + S3Consts.S3_XML_NAMESPACE + "\">" +
+ " <TagSet>" +
+ " <Tag><Key>dup</Key><Value>v1</Value></Tag>" +
+ " <Tag><Key>dup</Key><Value>v2</Value></Tag>" +
+ " </TagSet>" +
+ "</Tagging>";
+ }
+
+ private static String awsPrefixTag() {
+ return singleTag("aws:reserved", "value");
+ }
+
+ private static String invalidXmlStructure() {
+ return
+ "<Tagging xmlns=\"" + S3Consts.S3_XML_NAMESPACE + "\">" +
+ " <TagSet>" +
+ " </Ta" +
+ "Tagging>";
+ }
+
+ private static String noTagSet() {
+ return "<Tagging xmlns=\"" + S3Consts.S3_XML_NAMESPACE + "\"></Tagging>";
+ }
+
+ private static String emptyTags() {
+ return
+ "<Tagging xmlns=\"" + S3Consts.S3_XML_NAMESPACE + "\">" +
+ " <TagSet></TagSet>" +
+ "</Tagging>";
+ }
+
+ private static String tagKeyNotSpecified() {
+ return
+ "<Tagging xmlns=\"" + S3Consts.S3_XML_NAMESPACE + "\">" +
+ " <TagSet><Tag><Value>val1</Value></Tag></TagSet>" +
+ "</Tagging>";
+ }
+
+ private static String tagValueNotSpecified() {
+ return
+ "<Tagging xmlns=\"" + S3Consts.S3_XML_NAMESPACE + "\">" +
+ " <TagSet><Tag><Key>tag1</Key></Tag></TagSet>" +
+ "</Tagging>";
+ }
+}
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/metrics/TestS3GatewayMetrics.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/metrics/TestS3GatewayMetrics.java
index 5a4be9dd975..26c1e723128 100644
---
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/metrics/TestS3GatewayMetrics.java
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/metrics/TestS3GatewayMetrics.java
@@ -26,11 +26,14 @@
import static
org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.assertSucceeds;
import static
org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.completeMultipartUpload;
import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.delete;
+import static
org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.deleteBucketTagging;
import static
org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.deleteTagging;
import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.get;
+import static
org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.getBucketTagging;
import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.getTagging;
import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.listParts;
import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.put;
+import static
org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.putBucketTagging;
import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.putTagging;
import static
org.apache.hadoop.ozone.s3.exception.S3ErrorTable.BUCKET_ALREADY_EXISTS;
import static org.apache.hadoop.ozone.s3.util.S3Consts.COPY_SOURCE_HEADER;
@@ -566,6 +569,68 @@ public void testDeleteObjectTaggingFailure() {
assertEquals(1L, curMetric - oriMetric);
}
+ @Test
+ public void testPutBucketTaggingSuccess() throws Exception {
+ long oriMetric = metrics.getPutBucketTaggingSuccess();
+ assertSucceeds(() -> putBucketTagging(bucketEndpoint, bucketName,
getPutBucketTaggingBody()));
+
+ long curMetric = metrics.getPutBucketTaggingSuccess();
+ assertEquals(1L, curMetric - oriMetric);
+ }
+
+ @Test
+ public void testPutBucketTaggingFailure() {
+ long oriMetric = metrics.getPutBucketTaggingFailure();
+
+ assertErrorResponse(S3ErrorTable.NO_SUCH_BUCKET,
+ () -> putBucketTagging(bucketEndpoint, "nonexistent",
getPutBucketTaggingBody()));
+
+ long curMetric = metrics.getPutBucketTaggingFailure();
+ assertEquals(1L, curMetric - oriMetric);
+ }
+
+ @Test
+ public void testGetBucketTaggingSuccess() throws Exception {
+ long oriMetric = metrics.getGetBucketTaggingSuccess();
+ assertSucceeds(() -> putBucketTagging(bucketEndpoint, bucketName,
getPutBucketTaggingBody()));
+ assertSucceeds(() -> getBucketTagging(bucketEndpoint, bucketName));
+
+ long curMetric = metrics.getGetBucketTaggingSuccess();
+ assertEquals(1L, curMetric - oriMetric);
+ }
+
+ @Test
+ public void testGetBucketTaggingFailure() {
+ long oriMetric = metrics.getGetBucketTaggingFailure();
+
+ assertErrorResponse(S3ErrorTable.NO_SUCH_TAG_SET,
+ () -> getBucketTagging(bucketEndpoint, bucketName));
+
+ long curMetric = metrics.getGetBucketTaggingFailure();
+ assertEquals(1L, curMetric - oriMetric);
+ }
+
+ @Test
+ public void testDeleteBucketTaggingSuccess() throws Exception {
+ long oriMetric = metrics.getDeleteBucketTaggingSuccess();
+ assertSucceeds(() -> putBucketTagging(bucketEndpoint, bucketName,
getPutBucketTaggingBody()));
+ deleteBucketTagging(bucketEndpoint, bucketName);
+
+ long curMetric = metrics.getDeleteBucketTaggingSuccess();
+ assertEquals(1L, curMetric - oriMetric);
+ }
+
+ @Test
+ public void testDeleteBucketTaggingFailure() {
+ long oriMetric = metrics.getDeleteBucketTaggingFailure();
+
+ assertErrorResponse(S3ErrorTable.NO_SUCH_BUCKET,
+ () -> deleteBucketTagging(bucketEndpoint, "nonexistent"));
+
+ long curMetric = metrics.getDeleteBucketTaggingFailure();
+ assertEquals(1L, curMetric - oriMetric);
+ }
+
private String initiateMultipartUpload(String bktName, String key) throws
IOException, OS3Exception {
return EndpointTestUtils.initiateMultipartUpload(keyEndpoint, bktName,
key);
}
@@ -582,6 +647,10 @@ private static String getPutTaggingBody() {
"</Tagging>";
}
+ private static String getPutBucketTaggingBody() {
+ return getPutTaggingBody();
+ }
+
@Test
public void testPutObjectAclLatencyMetricsSnapshot() {
MetricsCollectorImpl collector = new MetricsCollectorImpl();
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
