This is an automated email from the ASF dual-hosted git repository.
chungen0126 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 8638daf7f3e HDDS-16636. Remove redundant InfoBucket RPC from S3
ListParts (#11358)
8638daf7f3e is described below
commit 8638daf7f3e4911257df2d7283b99c7939598f69
Author: KUAN-HAO HUANG <[email protected]>
AuthorDate: Wed Sep 30 14:42:21 2026 +0800
HDDS-16636. Remove redundant InfoBucket RPC from S3 ListParts (#11358)
---
.../ozone/s3/endpoint/MultipartKeyHandler.java | 12 +--
.../hadoop/ozone/s3/endpoint/TestListParts.java | 108 +++++++++++++++++++++
2 files changed, 114 insertions(+), 6 deletions(-)
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/MultipartKeyHandler.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/MultipartKeyHandler.java
index b9487b0f590..71d9c12f7ba 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/MultipartKeyHandler.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/MultipartKeyHandler.java
@@ -29,7 +29,6 @@
import org.apache.hadoop.ozone.audit.AuditLogger;
import org.apache.hadoop.ozone.audit.AuditLogger.PerformanceStringBuilder;
import org.apache.hadoop.ozone.audit.S3GAction;
-import org.apache.hadoop.ozone.client.OzoneBucket;
import org.apache.hadoop.ozone.client.OzoneMultipartUploadPartListParts;
import org.apache.hadoop.ozone.client.OzoneVolume;
import org.apache.hadoop.ozone.om.exceptions.OMException;
@@ -61,7 +60,7 @@ Response handleGetRequest(ObjectEndpoint.ObjectRequestContext
context, String ke
try {
int partMarker = parsePartNumberMarker(partNumberMarker);
- Response response = listParts(context.getBucket(), keyPath, uploadId,
+ Response response = listParts(context.getVolume(),
context.getBucketName(), keyPath, uploadId,
partMarker, maxParts, perf);
long opLatencyNs =
getMetrics().updateListPartsSuccessStats(context.getStartNanos());
perf.appendOpLatencyNanos(opLatencyNs);
@@ -114,7 +113,8 @@ private Response abortMultipartUpload(OzoneVolume volume,
String bucket,
/**
* Returns response for the listParts request.
* See:
https://docs.aws.amazon.com/AmazonS3/latest/API/mpUploadListParts.html
- * @param ozoneBucket
+ * @param volume
+ * @param bucketName
* @param key
* @param uploadId
* @param partNumberMarker
@@ -123,17 +123,17 @@ private Response abortMultipartUpload(OzoneVolume volume,
String bucket,
* @throws IOException
* @throws OS3Exception
*/
- private Response listParts(OzoneBucket ozoneBucket, String key, String
uploadId,
+ private Response listParts(OzoneVolume volume, String bucketName, String
key, String uploadId,
int partNumberMarker, int maxParts,
PerformanceStringBuilder perf)
throws IOException, OS3Exception {
ListPartsResponse resp = new ListPartsResponse();
- String bucketName = ozoneBucket.getName();
try {
OzoneMultipartUploadPartListParts parts =
- ozoneBucket.listParts(key, uploadId, partNumberMarker, maxParts);
+ getClientProtocol().listParts(volume.getName(), bucketName, key,
uploadId,
+ partNumberMarker, maxParts);
resp.setBucket(bucketName);
resp.setKey(key);
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestListParts.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestListParts.java
index d9828eb2e8e..7a4184f6960 100644
---
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestListParts.java
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestListParts.java
@@ -20,24 +20,42 @@
import static
org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.assertErrorResponse;
import static
org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.initiateMultipartUpload;
import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.uploadPart;
+import static
org.apache.hadoop.ozone.s3.util.S3Consts.EXPECTED_BUCKET_OWNER_HEADER;
import static org.apache.hadoop.ozone.s3.util.S3Consts.STORAGE_CLASS_HEADER;
import static org.apache.hadoop.ozone.s3.util.S3Consts.X_AMZ_CONTENT_SHA256;
+import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.io.IOException;
+import java.util.Collections;
import javax.ws.rs.core.HttpHeaders;
import javax.ws.rs.core.Response;
+import org.apache.hadoop.hdds.client.RatisReplicationConfig;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneBucket;
import org.apache.hadoop.ozone.client.OzoneClient;
import org.apache.hadoop.ozone.client.OzoneClientStub;
+import org.apache.hadoop.ozone.client.OzoneMultipartUploadPartListParts;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes;
import org.apache.hadoop.ozone.s3.exception.OS3Exception;
import org.apache.hadoop.ozone.s3.exception.S3ErrorTable;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
/**
* This class test list parts request.
@@ -99,6 +117,96 @@ public void testListPartsWithUnknownUploadID() {
() -> EndpointTestUtils.listParts(rest, OzoneConsts.S3_BUCKET,
"no-such-key", "no-such-upload", 2, 0));
}
+ @ParameterizedTest
+ @CsvSource({", 0", "'', 0", "owner, 1"})
+ public void testListPartsOnlyLooksUpBucketForOwner(String expectedOwner, int
bucketLookups) throws Exception {
+ ClientProtocol proxy = mock(ClientProtocol.class);
+ HttpHeaders headers = mock(HttpHeaders.class);
+
when(headers.getHeaderString(EXPECTED_BUCKET_OWNER_HEADER)).thenReturn(expectedOwner);
+ ObjectEndpoint endpoint = newEndpoint(proxy, headers);
+ OzoneMultipartUploadPartListParts parts = new
OzoneMultipartUploadPartListParts(
+
RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.THREE), 4,
true);
+ parts.addPart(new OzoneMultipartUploadPartListParts.PartInfo(4, "part4",
0, 10, "etag4"));
+ when(proxy.listParts("volume1", "bucket1", "key1", "upload1", 3,
2)).thenReturn(parts);
+
+ try (Response response = EndpointTestUtils.listParts(endpoint, "bucket1",
"key1", "upload1", 2, 3)) {
+ assertThat(response.getStatus()).isEqualTo(200);
+ ListPartsResponse result = (ListPartsResponse) response.getEntity();
+ assertThat(result.getBucket()).isEqualTo("bucket1");
+ assertThat(result.getPartNumberMarker()).isEqualTo(3);
+ assertThat(result.getMaxParts()).isEqualTo(2);
+ assertThat(result.getNextPartNumberMarker()).isEqualTo(4);
+ assertThat(result.getTruncated()).isTrue();
+ assertThat(result.getPartList()).hasSize(1);
+ assertThat(result.getPartList().get(0).getETag()).isEqualTo("etag4");
+ }
+ verify(proxy, times(bucketLookups)).getBucketDetails("volume1", "bucket1");
+ verify(proxy).listParts("volume1", "bucket1", "key1", "upload1", 3, 2);
+ }
+
+ @Test
+ public void testListPartsRejectsIncorrectOwner() throws Exception {
+ ClientProtocol proxy = mock(ClientProtocol.class);
+ HttpHeaders headers = mock(HttpHeaders.class);
+
when(headers.getHeaderString(EXPECTED_BUCKET_OWNER_HEADER)).thenReturn("other-owner");
+ ObjectEndpoint endpoint = newEndpoint(proxy, headers);
+
+ assertErrorResponse(S3ErrorTable.BUCKET_OWNER_MISMATCH,
+ () -> EndpointTestUtils.listParts(endpoint, "bucket1", "key1",
"upload1", 2, 3));
+
+ verify(proxy, never()).listParts("volume1", "bucket1", "key1", "upload1",
3, 2);
+ }
+
+ @ParameterizedTest
+ @CsvSource({
+ "NO_SUCH_MULTIPART_UPLOAD_ERROR, NO_SUCH_UPLOAD",
+ "BUCKET_NOT_FOUND, NO_SUCH_BUCKET",
+ "VOLUME_NOT_FOUND, NO_SUCH_BUCKET",
+ "PERMISSION_DENIED, ACCESS_DENIED"
+ })
+ public void testListPartsProtocolErrors(ResultCodes resultCode, S3ErrorTable
expectedError) throws Exception {
+ ClientProtocol proxy = mock(ClientProtocol.class);
+ ObjectEndpoint endpoint = newEndpoint(proxy, mock(HttpHeaders.class));
+ when(proxy.listParts("volume1", "bucket1", "key1", "upload1", 3, 2))
+ .thenThrow(new OMException("List parts failed", resultCode));
+
+ assertErrorResponse(expectedError,
+ () -> EndpointTestUtils.listParts(endpoint, "bucket1", "key1",
"upload1", 2, 3));
+
+ verify(proxy).listParts("volume1", "bucket1", "key1", "upload1", 3, 2);
+ }
+
+ @ParameterizedTest
+ @CsvSource({"PERMISSION_DENIED, ACCESS_DENIED", "TOKEN_EXPIRED,
EXPIRED_TOKEN"})
+ public void testListPartsVolumeLookupErrors(ResultCodes resultCode,
S3ErrorTable expectedError) throws Exception {
+ ClientProtocol proxy = mock(ClientProtocol.class);
+ ObjectEndpoint endpoint = newEndpoint(proxy, mock(HttpHeaders.class));
+ when(endpoint.getClient().getObjectStore().getS3Volume())
+ .thenThrow(new OMException("Volume lookup failed", resultCode));
+
+ OS3Exception error = assertErrorResponse(expectedError,
+ () -> EndpointTestUtils.listParts(endpoint, "bucket1", "key1",
"upload1", 2, 3));
+
+ assertThat(error.getResource()).isEqualTo("key1");
+ verify(proxy, never()).listParts("volume1", "bucket1", "key1", "upload1",
3, 2);
+ }
+
+ private ObjectEndpoint newEndpoint(ClientProtocol proxy, HttpHeaders
headers) throws Exception {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ OzoneVolume volume = OzoneVolume.newBuilder(conf, proxy)
+ .setName("volume1").setAcls(Collections.emptyList()).build();
+ OzoneBucket bucket = OzoneBucket.newBuilder(conf, proxy)
+ .setVolumeName("volume1").setName("bucket1").setOwner("owner").build();
+ when(proxy.getBucketDetails("volume1", "bucket1")).thenReturn(bucket);
+ ObjectStore objectStore = mock(ObjectStore.class);
+ when(objectStore.getS3Volume()).thenReturn(volume);
+ when(objectStore.getClientProxy()).thenReturn(proxy);
+ OzoneClient client = mock(OzoneClient.class);
+ when(client.getObjectStore()).thenReturn(objectStore);
+ when(client.getProxy()).thenReturn(proxy);
+ return
EndpointBuilder.newObjectEndpointBuilder().setClient(client).setHeaders(headers).build();
+ }
+
private ListPartsResponse listParts(int maxParts, int nextPart) throws
IOException, OS3Exception {
try (Response response = EndpointTestUtils.listParts(rest,
OzoneConsts.S3_BUCKET, OzoneConsts.KEY,
uploadID, maxParts, nextPart)) {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
