This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 7b68fcd3414 HDDS-16282. Avoid O(n^2) key removal in S3 DeleteObjects
(#11126)
7b68fcd3414 is described below
commit 7b68fcd3414278f5ee5930edb7cd1d2a4827d6e9
Author: rjgoyln <[email protected]>
AuthorDate: Thu Sep 24 12:59:56 2026 +0800
HDDS-16282. Avoid O(n^2) key removal in S3 DeleteObjects (#11126)
---
.../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 11 +--
.../ozone/s3/endpoint/TestObjectMultiDelete.java | 84 ++++++++++++++++++++++
2 files changed, 91 insertions(+), 4 deletions(-)
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java
index c833d5f22f4..497f4efc76b 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java
@@ -362,6 +362,7 @@ public MultiDeleteResponse multiDelete(
}
MultiDeleteResponse result = new MultiDeleteResponse();
List<String> deleteKeys = new ArrayList<>();
+ List<String> failedDeletes = new ArrayList<>();
if (request.getObjects() != null) {
Map<String, ErrorInfo> undeletedKeyResultMap;
@@ -378,20 +379,22 @@ public MultiDeleteResponse multiDelete(
// if the key is not found, it is assumed to be successfully
deleted
ResultCodes.KEY_NOT_FOUND.name().equals(error.getCode());
if (deleted) {
- deleteKeys.remove(d.getKey());
if (!request.isQuiet()) {
result.addDeleted(new DeletedObject(d.getKey()));
}
} else {
+ failedDeletes.add(d.getKey());
result.addError(new Error(d.getKey(), error.getCode(),
error.getMessage()));
}
}
getMetrics().updateDeleteKeySuccessStats(startNanos);
} catch (IOException ex) {
getMetrics().updateDeleteKeyFailureStats(startNanos);
+ // the batch delete failed as a whole, so no key is reported as deleted
+ failedDeletes.addAll(deleteKeys);
final OMException omEx = (OMException)
HddsClientUtils.containsException(ex, OMException.class);
if (omEx != null) {
- auditMultiDeleteFailure(context, deleteKeys, omEx);
+ auditMultiDeleteFailure(context, failedDeletes, omEx);
throw newError(bucketName, omEx);
}
LOG.error("Delete key failed: {}", ex.getMessage());
@@ -402,10 +405,10 @@ public MultiDeleteResponse multiDelete(
}
if (!result.getErrors().isEmpty()) {
- auditMultiDeleteFailure(context, deleteKeys, new Exception("MultiDelete
Exception"));
+ auditMultiDeleteFailure(context, failedDeletes, new
Exception("MultiDelete Exception"));
} else {
AuditMessage.Builder message = auditMessageFor(context.getAction());
- message.getParams().put("failedDeletes", deleteKeys.toString());
+ message.getParams().put("failedDeletes", failedDeletes.toString());
AUDIT.logWriteSuccess(message.withResult(AuditEventStatus.SUCCESS).build());
}
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectMultiDelete.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectMultiDelete.java
index 1d657460346..f4316152ce8 100644
---
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectMultiDelete.java
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectMultiDelete.java
@@ -17,23 +17,39 @@
package org.apache.hadoop.ozone.s3.endpoint;
+import static java.util.Arrays.asList;
import static java.util.Collections.singleton;
import static
org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.assertErrorResponse;
import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.MALFORMED_XML;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.anyBoolean;
+import static org.mockito.Mockito.anyString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
import com.google.common.collect.Sets;
import java.io.IOException;
+import java.util.HashMap;
import java.util.HashSet;
+import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import javax.xml.bind.JAXBException;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.client.ObjectStore;
import org.apache.hadoop.ozone.client.OzoneBucket;
import org.apache.hadoop.ozone.client.OzoneClient;
import org.apache.hadoop.ozone.client.OzoneClientStub;
import org.apache.hadoop.ozone.client.OzoneKey;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
+import org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes;
+import org.apache.hadoop.ozone.om.helpers.ErrorInfo;
import org.apache.hadoop.ozone.s3.endpoint.MultiDeleteRequest.DeleteObject;
+import org.apache.hadoop.ozone.s3.endpoint.MultiDeleteResponse.DeletedObject;
import org.apache.hadoop.ozone.s3.exception.OS3Exception;
+import org.apache.hadoop.ozone.s3.metrics.S3GatewayMetrics;
import org.apache.hadoop.ozone.s3.util.S3Consts;
import org.junit.jupiter.api.Test;
@@ -140,6 +156,74 @@ public void multiDeleteAllowsMaxKeysPerRequest() throws
Exception {
assertEquals(3, Sets.newHashSet(bucket.listKeys("")).size());
}
+ @Test
+ public void multiDeleteAuditsOnlyFailedKeys() throws Exception {
+ OzoneBucket bucket = mock(OzoneBucket.class);
+ Map<String, ErrorInfo> undeletedKeys = new HashMap<>();
+ undeletedKeys.put("key2", new ErrorInfo("ACCESS_DENIED", "ACL check
failed"));
+ undeletedKeys.put("key3", new ErrorInfo(ResultCodes.KEY_NOT_FOUND.name(),
"Key does not exist"));
+ when(bucket.deleteKeys(any(), anyBoolean())).thenReturn(undeletedKeys);
+
+ Map<String, String> auditParams = new HashMap<>();
+ MultiDeleteResponse response =
+ newEndpointFor(bucket, auditParams).multiDelete("b1", "",
threeKeyRequest());
+
+ assertEquals(asList("key1", "key3"), response.getDeletedObjects().stream()
+ .map(DeletedObject::getKey)
+ .collect(Collectors.toList()));
+ assertEquals(1, response.getErrors().size());
+ assertEquals("key2", response.getErrors().get(0).getKey());
+ assertEquals("[key2]", auditParams.get("failedDeletes"));
+ }
+
+ @Test
+ public void multiDeleteAuditsAllKeysWhenDeleteFails() throws Exception {
+ OzoneBucket bucket = mock(OzoneBucket.class);
+ when(bucket.deleteKeys(any(), anyBoolean())).thenThrow(new
IOException("Ozone Manager is unavailable"));
+
+ Map<String, String> auditParams = new HashMap<>();
+ MultiDeleteResponse response =
+ newEndpointFor(bucket, auditParams).multiDelete("b1", "",
threeKeyRequest());
+
+ assertEquals(0, response.getDeletedObjects().size());
+ assertEquals(1, response.getErrors().size());
+ assertEquals("ALL", response.getErrors().get(0).getKey());
+ assertEquals("InternalError", response.getErrors().get(0).getCode());
+ assertEquals("[key1, key2, key3]", auditParams.get("failedDeletes"));
+ }
+
+ private MultiDeleteRequest threeKeyRequest() {
+ MultiDeleteRequest mdr = new MultiDeleteRequest();
+ mdr.getObjects().add(new DeleteObject("key1"));
+ mdr.getObjects().add(new DeleteObject("key2"));
+ mdr.getObjects().add(new DeleteObject("key3"));
+ return mdr;
+ }
+
+ private BucketEndpoint newEndpointFor(OzoneBucket bucket, Map<String,
String> auditParams) throws IOException {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ S3GatewayMetrics.create(conf);
+
+ OzoneClient client = mock(OzoneClient.class);
+ ObjectStore objectStore = mock(ObjectStore.class);
+ OzoneVolume volume = mock(OzoneVolume.class);
+ when(client.getObjectStore()).thenReturn(objectStore);
+ when(client.getConfiguration()).thenReturn(conf);
+ when(objectStore.getClientProxy()).thenReturn(mock(ClientProtocol.class));
+ when(objectStore.getS3Volume()).thenReturn(volume);
+ when(volume.getBucket(anyString())).thenReturn(bucket);
+
+ return EndpointBuilder.newBucketEndpointBuilder()
+ .setBase(new BucketEndpoint() {
+ @Override
+ protected Map<String, String> getAuditParameters() {
+ return auditParams;
+ }
+ })
+ .setClient(client)
+ .build();
+ }
+
private OzoneBucket initTestData(OzoneClient client) throws IOException {
client.getObjectStore().createS3Bucket("b1");
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
