This is an automated email from the ASF dual-hosted git repository.
ChenSammi pushed a commit to branch HDDS-13323-sts
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/HDDS-13323-sts by this push:
new 113e3d0f100 HDDS-16187. [STS] Fix Latent S3 DeleteObjects Issue
(#11019)
113e3d0f100 is described below
commit 113e3d0f100e71fa347f20e380f205759cd01606
Author: fmorg-git <[email protected]>
AuthorDate: Mon Aug 17 01:50:09 2026 -0700
HDDS-16187. [STS] Fix Latent S3 DeleteObjects Issue (#11019)
---
.../main/smoketest/security/ozone-secure-sts.robot | 28 +++++++++
.../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 32 +++++++---
.../ozone/s3/endpoint/TestPermissionCheck.java | 70 +++++++++++++++++-----
3 files changed, 108 insertions(+), 22 deletions(-)
diff --git
a/hadoop-ozone/dist/src/main/smoketest/security/ozone-secure-sts.robot
b/hadoop-ozone/dist/src/main/smoketest/security/ozone-secure-sts.robot
index 68e12e40b5e..a1f34e818b7 100644
--- a/hadoop-ozone/dist/src/main/smoketest/security/ozone-secure-sts.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/security/ozone-secure-sts.robot
@@ -1209,6 +1209,34 @@ STS session policy s3:* on * must allow
ListAllMyBuckets, Create/ListBucket, and
${output} = Execute aws s3api
--endpoint-url ${S3G_ENDPOINT_URL} delete-bucket --bucket ${bucket} --profile
sts
Should Not Contain ${output} AccessDenied
+STS session policy containing only GetObject must deny DeleteObjects
+ ${bucket_suffix} = Generate Random String 8 [LOWER]
+ ${bucket} = Set Variable
sts-bucket-deleteobjects-${bucket_suffix}
+ ${key_suffix} = Generate Random String 8 [LOWER]
+ ${key} = Set Variable
sts-deny-deleteobjects-${key_suffix}.txt
+ ${local_path} = Set Variable
${TEMP_DIR}/${key}
+ Create File ${local_path} deleteobjects
deny test content
+
+ # Create bucket and object with full STS temp-bucket role permissions.
+ Assume Role And Configure STS Profile
perm_access_key_id=${PERMANENT_ACCESS_KEY_ID}
perm_secret_key=${PERMANENT_SECRET_KEY} role_arn=${STS_TEMP_BUCKET_ROLE_ARN}
+ ${output} = Execute aws s3api
--endpoint-url ${S3G_ENDPOINT_URL} create-bucket --bucket ${bucket} --profile
sts
+ Should Contain ${output} Location
+ ${output} = Execute aws s3api
--endpoint-url ${S3G_ENDPOINT_URL} put-object --bucket ${bucket} --key ${key}
--body ${local_path} --profile sts
+ Should Contain ${output} "ETag"
+
+ # Restrict token to GetObject-only via session policy. DeleteObjects must
return AccessDenied.
+ ${session_policy} = Set Variable
{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:GetObject","Resource":"arn:aws:s3:::${bucket}/*"}]}
+ Assume Role And Configure STS Profile
policy_json=${session_policy} perm_access_key_id=${PERMANENT_ACCESS_KEY_ID}
perm_secret_key=${PERMANENT_SECRET_KEY} role_arn=${STS_TEMP_BUCKET_ROLE_ARN}
+ ${output} = Execute And Ignore Error aws s3api
--endpoint-url ${S3G_ENDPOINT_URL} delete-objects --bucket ${bucket} --delete
'Objects=[{Key=${key}}],Quiet=false' --profile sts
+ Run Keyword And Continue On Failure Should Contain ${output}
AccessDenied
+
+ # Cleanup using a full-permission token.
+ Assume Role And Configure STS Profile
perm_access_key_id=${PERMANENT_ACCESS_KEY_ID}
perm_secret_key=${PERMANENT_SECRET_KEY} role_arn=${STS_TEMP_BUCKET_ROLE_ARN}
+ ${output} = Execute aws s3api
--endpoint-url ${S3G_ENDPOINT_URL} delete-object --bucket ${bucket} --key
${key} --profile sts
+ Should Not Contain ${output} AccessDenied
+ ${output} = Execute aws s3api
--endpoint-url ${S3G_ENDPOINT_URL} delete-bucket --bucket ${bucket} --profile
sts
+ Should Not Contain ${output} AccessDenied
+
Revoking Permanent User Must Revoke Existing Session Token
# Create session tokens for both buckets, verify they work, then revoke
permanent user secret and verify both fail.
Assume Role And Get Temporary Credentials
perm_access_key_id=${PERMANENT_ACCESS_KEY_ID}
perm_secret_key=${PERMANENT_SECRET_KEY}
role_arn=${ICEBERG_ALL_ACCESS_ROLE_OBS_ARN}
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java
index 8830e59921b..c833d5f22f4 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java
@@ -46,6 +46,7 @@
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
import org.apache.hadoop.ozone.audit.AuditEventStatus;
import org.apache.hadoop.ozone.audit.AuditMessage;
import org.apache.hadoop.ozone.audit.S3GAction;
@@ -351,7 +352,14 @@ public MultiDeleteResponse multiDelete(
throw newError(S3ErrorTable.MALFORMED_XML, bucketName);
}
- OzoneBucket bucket = getVolume().getBucket(bucketName);
+ final OzoneBucket bucket;
+ try {
+ bucket = getVolume().getBucket(bucketName);
+ } catch (OMException ex) {
+ throw newError(bucketName, ex);
+ } catch (IOException ex) {
+ throw newError(S3ErrorTable.INTERNAL_ERROR, bucketName, ex);
+ }
MultiDeleteResponse result = new MultiDeleteResponse();
List<String> deleteKeys = new ArrayList<>();
@@ -380,27 +388,37 @@ public MultiDeleteResponse multiDelete(
}
getMetrics().updateDeleteKeySuccessStats(startNanos);
} catch (IOException ex) {
- LOG.error("Delete key failed: {}", ex.getMessage());
getMetrics().updateDeleteKeyFailureStats(startNanos);
+ final OMException omEx = (OMException)
HddsClientUtils.containsException(ex, OMException.class);
+ if (omEx != null) {
+ auditMultiDeleteFailure(context, deleteKeys, omEx);
+ throw newError(bucketName, omEx);
+ }
+ LOG.error("Delete key failed: {}", ex.getMessage());
result.addError(
new Error("ALL", "InternalError",
ex.getMessage()));
}
}
- AuditMessage.Builder message = auditMessageFor(context.getAction());
- message.getParams().put("failedDeletes", deleteKeys.toString());
-
if (!result.getErrors().isEmpty()) {
- AUDIT.logWriteFailure(message.withResult(AuditEventStatus.FAILURE)
- .withException(new Exception("MultiDelete Exception")).build());
+ auditMultiDeleteFailure(context, deleteKeys, new Exception("MultiDelete
Exception"));
} else {
+ AuditMessage.Builder message = auditMessageFor(context.getAction());
+ message.getParams().put("failedDeletes", deleteKeys.toString());
AUDIT.logWriteSuccess(message.withResult(AuditEventStatus.SUCCESS).build());
}
return result;
}
+ void auditMultiDeleteFailure(S3RequestContext context, List<String>
deleteKeys, Throwable ex) {
+ final AuditMessage.Builder message = auditMessageFor(context.getAction());
+ message.getParams().put("failedDeletes", deleteKeys.toString());
+ AUDIT.logWriteFailure(message.withResult(AuditEventStatus.FAILURE)
+ .withException(ex).build());
+ }
+
private void addKey(ListObjectResponse response, OzoneKey next, boolean
includeOwner) {
KeyMetadata keyMetadata = new KeyMetadata();
keyMetadata.setKey(EncodingTypeObject.createNullable(next.getName(),
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java
index bc9e19db1b6..5229ca0e898 100644
---
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java
@@ -35,15 +35,17 @@
import static org.mockito.Mockito.anyMap;
import static org.mockito.Mockito.anyString;
import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.eq;
import static org.mockito.Mockito.isNull;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
+import java.util.Collections;
import java.util.Map;
+import java.util.stream.Stream;
import javax.ws.rs.core.HttpHeaders;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.OzoneConfigKeys;
@@ -53,6 +55,7 @@
import org.apache.hadoop.ozone.client.OzoneVolume;
import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes;
import org.apache.hadoop.ozone.om.helpers.ErrorInfo;
import org.apache.hadoop.ozone.s3.exception.OS3Exception;
import org.apache.hadoop.ozone.s3.exception.S3ErrorTable;
@@ -61,6 +64,9 @@
import org.apache.hadoop.ozone.s3.util.S3Consts.QueryParams;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
/**
* Test operation permission check result.
@@ -87,8 +93,7 @@ public void setup() {
bucket = mock(OzoneBucket.class);
volume = mock(OzoneVolume.class);
when(volume.getName()).thenReturn("s3Volume");
- exception = new OMException("Permission Denied",
- OMException.ResultCodes.PERMISSION_DENIED);
+ exception = new OMException("Permission Denied",
ResultCodes.PERMISSION_DENIED);
when(client.getObjectStore()).thenReturn(objectStore);
when(client.getConfiguration()).thenReturn(conf);
headers = mock(HttpHeaders.class);
@@ -186,25 +191,42 @@ public void testDeleteKeys() throws IOException,
OS3Exception {
when(objectStore.getVolume(anyString())).thenReturn(volume);
when(objectStore.getS3Volume()).thenReturn(volume);
when(volume.getBucket(anyString())).thenReturn(bucket);
- Map<String, ErrorInfo> deleteErrors = new HashMap<>();
- deleteErrors.put("deleteKeyName", new ErrorInfo("ACCESS_DENIED", "ACL
check failed"));
+ final Map<String, ErrorInfo> deleteErrors = Collections.singletonMap(
+ "deleteKeyName", new ErrorInfo("ACCESS_DENIED", "ACL check failed"));
when(bucket.deleteKeys(any(), anyBoolean())).thenReturn(deleteErrors);
- BucketEndpoint bucketEndpoint = EndpointBuilder.newBucketEndpointBuilder()
+ final BucketEndpoint bucketEndpoint =
EndpointBuilder.newBucketEndpointBuilder()
.setClient(client)
.build();
- MultiDeleteRequest request = new MultiDeleteRequest();
- List<MultiDeleteRequest.DeleteObject> objectList = new ArrayList<>();
- objectList.add(new MultiDeleteRequest.DeleteObject("deleteKeyName"));
- request.setQuiet(false);
- request.setObjects(objectList);
+ final MultiDeleteRequest request = createMultiDeleteRequest();
- MultiDeleteResponse response =
- bucketEndpoint.multiDelete("BucketName", "keyName", request);
+ final MultiDeleteResponse response =
bucketEndpoint.multiDelete("BucketName", "keyName", request);
assertEquals(1, response.getErrors().size());
assertEquals("ACCESS_DENIED", response.getErrors().get(0).getCode());
}
+ @ParameterizedTest
+ @MethodSource("deleteKeysTranslatedOMFailures")
+ public void testDeleteKeysTranslatesContainedOMFailures(ResultCodes
resultCode, S3ErrorTable expectedError)
+ throws IOException {
+ when(objectStore.getVolume(anyString())).thenReturn(volume);
+ when(objectStore.getS3Volume()).thenReturn(volume);
+ when(volume.getBucket(anyString())).thenReturn(bucket);
+ doThrow(new IOException("DeleteObjects failed", new OMException("OM
failure", resultCode)))
+ .when(bucket).deleteKeys(any(), anyBoolean());
+
+ final BucketEndpoint bucketEndpoint = spy(new BucketEndpoint());
+ EndpointBuilder.newBucketEndpointBuilder()
+ .setBase(bucketEndpoint)
+ .setClient(client)
+ .build();
+ final MultiDeleteRequest request = createMultiDeleteRequest();
+
+ assertErrorResponse(expectedError, () ->
bucketEndpoint.multiDelete("BucketName", "keyName", request));
+ verify(bucketEndpoint).auditMultiDeleteFailure(
+ any(), eq(Collections.singletonList("deleteKeyName")),
any(OMException.class));
+ }
+
@Test
public void testGetAcl() throws Exception {
when(objectStore.getS3Volume()).thenReturn(volume);
@@ -333,4 +355,22 @@ public void testObjectTagging() throws Exception {
assertErrorResponse(S3ErrorTable.ACCESS_DENIED,
() -> getTagging(objectEndpoint, "bucketName", "keyPath"));
}
+
+ private static MultiDeleteRequest createMultiDeleteRequest() {
+ final MultiDeleteRequest request = new MultiDeleteRequest();
+ request.setQuiet(false);
+ request.setObjects(Collections.singletonList(new
MultiDeleteRequest.DeleteObject("deleteKeyName")));
+ return request;
+ }
+
+ private static Stream<Arguments> deleteKeysTranslatedOMFailures() {
+ return Stream.of(
+ Arguments.of(ResultCodes.ACCESS_DENIED, S3ErrorTable.ACCESS_DENIED),
+ Arguments.of(ResultCodes.PERMISSION_DENIED,
S3ErrorTable.ACCESS_DENIED),
+ Arguments.of(ResultCodes.INVALID_TOKEN, S3ErrorTable.ACCESS_DENIED),
+ Arguments.of(ResultCodes.REVOKED_TOKEN, S3ErrorTable.ACCESS_DENIED),
+ Arguments.of(ResultCodes.TOKEN_EXPIRED, S3ErrorTable.EXPIRED_TOKEN),
+ Arguments.of(ResultCodes.BUCKET_NOT_FOUND, S3ErrorTable.NO_SUCH_BUCKET)
+ );
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
