This is an automated email from the ASF dual-hosted git repository.
Gargi-jais11 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new e09eeaa7285 HDDS-15616. ListBuckets API ignores MaxBuckets parameter
and returns all buckets (#10639)
e09eeaa7285 is described below
commit e09eeaa728577552e08c4e4fdd8be5ae53b47dec
Author: Gargi Jaiswal <[email protected]>
AuthorDate: Tue Jul 14 14:32:34 2026 +0530
HDDS-15616. ListBuckets API ignores MaxBuckets parameter and returns all
buckets (#10639)
---
.../hadoop/ozone/s3/awssdk/S3SDKTestUtils.java | 59 +++++++
.../ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java | 169 ++++++++++++++++++--
.../ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java | 170 +++++++++++++++++++--
.../ozone/s3/endpoint/ListBucketResponse.java | 13 +-
.../hadoop/ozone/s3/endpoint/RootEndpoint.java | 41 ++++-
.../org/apache/hadoop/ozone/s3/util/S3Consts.java | 3 +
.../hadoop/ozone/s3/endpoint/TestRootList.java | 94 ++++++++++++
7 files changed, 515 insertions(+), 34 deletions(-)
diff --git
a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/S3SDKTestUtils.java
b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/S3SDKTestUtils.java
index ec3793c68d1..2b3375052ce 100644
---
a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/S3SDKTestUtils.java
+++
b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/S3SDKTestUtils.java
@@ -25,12 +25,17 @@
import java.net.HttpURLConnection;
import java.net.URL;
import java.security.MessageDigest;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Set;
+import java.util.function.BiFunction;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import java.util.stream.Collectors;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.RandomUtils;
import org.apache.ozone.test.InputSubstream;
@@ -49,6 +54,60 @@ public final class S3SDKTestUtils {
public static final Pattern UPLOAD_ID_PATTERN =
Pattern.compile("<UploadId>(.+?)</UploadId>");
+ /**
+ * One page of a paginated ListBuckets response.
+ */
+ public static final class BucketListPage {
+ private final List<String> bucketNames;
+ private final String continuationToken;
+
+ public BucketListPage(List<String> bucketNames, String continuationToken) {
+ this.bucketNames = bucketNames;
+ this.continuationToken = continuationToken;
+ }
+
+ public List<String> getBucketNames() {
+ return bucketNames;
+ }
+
+ public String getContinuationToken() {
+ return continuationToken;
+ }
+ }
+
+ /**
+ * Lists buckets one per page and returns all bucket names from the pages.
+ *
+ * @param listPage fetches one page; first arg is continuation token
(nullable),
+ * second arg is max buckets per page
+ */
+ public static List<String> collectBucketsOnePerPage(
+ BiFunction<String, Integer, BucketListPage> listPage) {
+ List<String> found = new ArrayList<>();
+ String continuationToken = null;
+ do {
+ BucketListPage page = listPage.apply(continuationToken, 1);
+ if (page.getBucketNames().size() != 1) {
+ throw new AssertionError(
+ "Expected 1 bucket per page, got " + page.getBucketNames().size());
+ }
+ found.add(page.getBucketNames().get(0));
+ continuationToken = page.getContinuationToken();
+ } while (continuationToken != null);
+ return found;
+ }
+
+ /**
+ * Filters a paginated bucket list down to the buckets created by the test.
+ */
+ public static List<String> filterToExpectedBuckets(
+ List<String> paginatedBuckets, String... expectedBuckets) {
+ Set<String> expected = new HashSet<>(Arrays.asList(expectedBuckets));
+ return paginatedBuckets.stream()
+ .filter(expected::contains)
+ .collect(Collectors.toList());
+ }
+
private S3SDKTestUtils() {
}
diff --git
a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java
b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java
index c31e3154c32..618dc73212e 100644
---
a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java
+++
b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java
@@ -55,6 +55,8 @@
import com.amazonaws.services.s3.model.Grantee;
import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest;
import com.amazonaws.services.s3.model.InitiateMultipartUploadResult;
+import com.amazonaws.services.s3.model.ListBucketsPaginatedRequest;
+import com.amazonaws.services.s3.model.ListBucketsPaginatedResult;
import com.amazonaws.services.s3.model.ListMultipartUploadsRequest;
import com.amazonaws.services.s3.model.ListObjectsRequest;
import com.amazonaws.services.s3.model.ListObjectsV2Request;
@@ -267,28 +269,163 @@ public void testBucketACLOperations() {
//assertEquals(aclList, s3Client.getBucketAcl(bucketName));
}
- @Test
- public void testListBuckets() throws IOException {
- List<String> bucketNames = new ArrayList<>();
- for (int i = 0; i <= 5; i++) {
- String bucketName = getBucketName(String.valueOf(i));
- s3Client.createBucket(bucketName);
- bucketNames.add(bucketName);
+ /**
+ * Integration tests for ListBuckets (GET / ListAllMyBuckets).
+ */
+ @Nested
+ class ListBucketsTests {
+
+ @Test
+ public void testListBuckets() throws IOException {
+ List<String> bucketNames = new ArrayList<>();
+ for (int i = 0; i <= 5; i++) {
+ String bucketName = getBucketName(String.valueOf(i));
+ s3Client.createBucket(bucketName);
+ bucketNames.add(bucketName);
+ }
+
+ List<Bucket> bucketList = s3Client.listBuckets();
+ List<String> listBucketNames = bucketList.stream()
+ .map(Bucket::getName).collect(Collectors.toList());
+
+ assertThat(listBucketNames).containsAll(bucketNames);
+
+ UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+ String expectOwner = ugi.getShortUserName();
+
+ Owner s3AccountOwner = s3Client.getS3AccountOwner();
+
+ assertThat(s3AccountOwner.getDisplayName()).isEqualTo(expectOwner);
+ assertThat(s3AccountOwner.getId()).isEqualTo(S3Owner.DEFAULT_S3OWNER_ID);
+ }
+
+ /**
+ * Verifies {@code maxBuckets=1} returns one bucket per page and a
continuation token
+ * when more buckets exist.
+ */
+ @Test
+ public void testListBucketsPaginatedMaxBucketsOne() {
+ final String bucketA = uniqueObjectName("bucket-a");
+ final String bucketB = uniqueObjectName("bucket-b");
+ s3Client.createBucket(bucketA);
+ s3Client.createBucket(bucketB);
+ try {
+ List<String> found = S3SDKTestUtils.collectBucketsOnePerPage((token,
max) -> {
+ ListBucketsPaginatedRequest request = new
ListBucketsPaginatedRequest()
+ .withMaxBuckets(max);
+ if (token != null) {
+ request.withContinuationToken(token);
+ }
+
+ ListBucketsPaginatedResult page = s3Client.listBuckets(request);
+ return new S3SDKTestUtils.BucketListPage(
+
page.getBuckets().stream().map(Bucket::getName).collect(Collectors.toList()),
+ page.getContinuationToken());
+ });
+ List<String> foundTestBuckets = S3SDKTestUtils.filterToExpectedBuckets(
+ found, bucketA, bucketB);
+ assertThat(foundTestBuckets).containsExactlyInAnyOrder(bucketA,
bucketB);
+ } finally {
+ s3Client.deleteBucket(bucketA);
+ s3Client.deleteBucket(bucketB);
+ }
+ }
+
+ /**
+ * Verifies pagination: listing buckets page-by-page using {@code
maxBuckets}
+ * and the returned continuation token, until all buckets are retrieved.
+ */
+ @Test
+ public void testListBucketsPaginationReturnsAllBuckets() {
+ final int totalBuckets = 5;
+ final int pageSize = 2;
+ List<String> created = new ArrayList<>();
+
+ for (int i = 0; i < totalBuckets; i++) {
+ String name = uniqueObjectName("paginated-" + i);
+ s3Client.createBucket(name);
+ created.add(name);
+ }
+
+ try {
+ List<String> retrieved = new ArrayList<>();
+ String continuationToken = null;
+
+ do {
+ ListBucketsPaginatedRequest request = new
ListBucketsPaginatedRequest()
+ .withMaxBuckets(pageSize);
+ if (continuationToken != null) {
+ request.withContinuationToken(continuationToken);
+ }
+
+ ListBucketsPaginatedResult response = s3Client.listBuckets(request);
+
+ response.getBuckets().stream()
+ .map(Bucket::getName)
+ .filter(created::contains)
+ .forEach(retrieved::add);
+
+ continuationToken = response.getContinuationToken();
+ } while (continuationToken != null);
+
+ assertThat(retrieved).containsExactlyInAnyOrderElementsOf(created);
+ } finally {
+ for (String name : created) {
+ s3Client.deleteBucket(name);
+ }
+ }
}
- List<Bucket> bucketList = s3Client.listBuckets();
- List<String> listBucketNames = bucketList.stream()
- .map(Bucket::getName).collect(Collectors.toList());
+ /**
+ * verifies that Page 1 uses maxBuckets=1;
+ * later pages send only continuationToken (no maxBuckets).
+ */
+ @Test
+ public void testListBucketsContinuationTokenWithoutMaxBuckets() {
+ List<String> created = new ArrayList<>();
+ for (int i = 0; i < 3; i++) {
+ String name = uniqueObjectName("token-only-" + i);
+ s3Client.createBucket(name);
+ created.add(name);
+ }
- assertThat(listBucketNames).containsAll(bucketNames);
+ try {
+ List<String> retrieved = new ArrayList<>();
+ String continuationToken = null;
+ boolean firstPage = true;
+
+ do {
+ ListBucketsPaginatedRequest request = new
ListBucketsPaginatedRequest();
+ if (firstPage) {
+ request.withMaxBuckets(1);
+ firstPage = false;
+ } else {
+ request.withContinuationToken(continuationToken);
+ }
- UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
- String expectOwner = ugi.getShortUserName();
+ ListBucketsPaginatedResult response = s3Client.listBuckets(request);
+ if (continuationToken == null) {
+ assertEquals(1, response.getBuckets().size());
+ assertNotNull(response.getContinuationToken());
+ } else {
+ assertFalse(response.getBuckets().isEmpty());
+ }
- Owner s3AccountOwner = s3Client.getS3AccountOwner();
+ response.getBuckets().stream()
+ .map(Bucket::getName)
+ .filter(created::contains)
+ .forEach(retrieved::add);
- assertThat(s3AccountOwner.getDisplayName()).isEqualTo(expectOwner);
- assertThat(s3AccountOwner.getId()).isEqualTo(S3Owner.DEFAULT_S3OWNER_ID);
+ continuationToken = response.getContinuationToken();
+ } while (continuationToken != null);
+
+ assertThat(retrieved).containsExactlyInAnyOrderElementsOf(created);
+ } finally {
+ for (String name : created) {
+ s3Client.deleteBucket(name);
+ }
+ }
+ }
}
@Test
diff --git
a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java
b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java
index 53cb48e99e9..ecab86ed323 100644
---
a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java
+++
b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java
@@ -131,6 +131,7 @@
import software.amazon.awssdk.services.s3.model.HeadBucketRequest;
import software.amazon.awssdk.services.s3.model.HeadObjectRequest;
import software.amazon.awssdk.services.s3.model.HeadObjectResponse;
+import software.amazon.awssdk.services.s3.model.ListBucketsRequest;
import software.amazon.awssdk.services.s3.model.ListBucketsResponse;
import software.amazon.awssdk.services.s3.model.ListDirectoryBucketsRequest;
import software.amazon.awssdk.services.s3.model.ListDirectoryBucketsResponse;
@@ -217,20 +218,6 @@ void closeClient() {
}
}
- @Test
- public void listBuckets() throws Exception {
- final String bucketName = getBucketName();
- final String expectedOwner =
UserGroupInformation.getCurrentUser().getUserName();
-
- s3Client.createBucket(b -> b.bucket(bucketName));
-
- ListBucketsResponse syncResponse = s3Client.listBuckets();
- assertEquals(1, syncResponse.buckets().size());
- assertEquals(bucketName, syncResponse.buckets().get(0).name());
- assertEquals(expectedOwner, syncResponse.owner().displayName());
- assertEquals(S3Owner.DEFAULT_S3OWNER_ID, syncResponse.owner().id());
- }
-
@Test
public void testPutObject() {
final String bucketName = getBucketName();
@@ -2857,6 +2844,161 @@ private void
verifyBucketOwnershipVerificationAccessDenied(Executable function)
}
}
+ /**
+ * Integration tests for ListBuckets (GET / ListAllMyBuckets).
+ */
+ @Nested
+ class ListBucketsTests {
+
+ @Test
+ public void testListBuckets() throws Exception {
+ List<String> bucketNames = new ArrayList<>();
+ for (int i = 0; i <= 5; i++) {
+ String bucketName = getBucketName(String.valueOf(i));
+ s3Client.createBucket(b -> b.bucket(bucketName));
+ bucketNames.add(bucketName);
+ }
+
+ ListBucketsResponse syncResponse = s3Client.listBuckets();
+ List<String> listBucketNames = syncResponse.buckets().stream()
+ .map(Bucket::name)
+ .collect(Collectors.toList());
+
+ assertThat(listBucketNames).containsAll(bucketNames);
+
+ String expectedOwner =
UserGroupInformation.getCurrentUser().getShortUserName();
+ assertEquals(expectedOwner, syncResponse.owner().displayName());
+ assertEquals(S3Owner.DEFAULT_S3OWNER_ID, syncResponse.owner().id());
+ }
+
+ /**
+ * Verifies {@code maxBuckets=1} returns one bucket per page and a
continuation token
+ * when more buckets exist.
+ */
+ @Test
+ public void testListBucketsPaginatedMaxBucketsOne() throws Exception {
+ final String bucketA = uniqueObjectName("bucket-a");
+ final String bucketB = uniqueObjectName("bucket-b");
+ s3Client.createBucket(b -> b.bucket(bucketA));
+ s3Client.createBucket(b -> b.bucket(bucketB));
+ try {
+ List<String> found = S3SDKTestUtils.collectBucketsOnePerPage((token,
max) -> {
+ ListBucketsRequest.Builder reqBuilder = ListBucketsRequest.builder()
+ .maxBuckets(max);
+ if (token != null) {
+ reqBuilder.continuationToken(token);
+ }
+ ListBucketsResponse page = s3Client.listBuckets(reqBuilder.build());
+ return new S3SDKTestUtils.BucketListPage(
+
page.buckets().stream().map(Bucket::name).collect(Collectors.toList()),
+ page.continuationToken());
+ });
+ List<String> foundTestBuckets = S3SDKTestUtils.filterToExpectedBuckets(
+ found, bucketA, bucketB);
+ assertThat(foundTestBuckets).containsExactlyInAnyOrder(bucketA,
bucketB);
+ } finally {
+ s3Client.deleteBucket(b -> b.bucket(bucketA));
+ s3Client.deleteBucket(b -> b.bucket(bucketB));
+ }
+ }
+
+ /**
+ * Verifies pagination: listing buckets page-by-page using {@code
maxBuckets}
+ * and the returned continuation token, until all buckets are retrieved.
+ */
+ @Test
+ public void testListBucketsPaginationReturnsAllBuckets() throws Exception {
+ final int totalBuckets = 5;
+ final int pageSize = 2;
+ List<String> created = new ArrayList<>();
+
+ for (int i = 0; i < totalBuckets; i++) {
+ String name = uniqueObjectName("paginated-" + i);
+ s3Client.createBucket(b -> b.bucket(name));
+ created.add(name);
+ }
+
+ try {
+ List<String> retrieved = new ArrayList<>();
+ String continuationToken = null;
+
+ do {
+ ListBucketsRequest.Builder reqBuilder = ListBucketsRequest.builder()
+ .maxBuckets(pageSize);
+ if (continuationToken != null) {
+ reqBuilder.continuationToken(continuationToken);
+ }
+
+ ListBucketsResponse response =
s3Client.listBuckets(reqBuilder.build());
+
+ response.buckets().stream()
+ .map(Bucket::name)
+ .filter(created::contains)
+ .forEach(retrieved::add);
+
+ continuationToken = response.continuationToken();
+ } while (continuationToken != null);
+
+ assertThat(retrieved).containsExactlyInAnyOrderElementsOf(created);
+ } finally {
+ for (String name : created) {
+ s3Client.deleteBucket(b -> b.bucket(name));
+ }
+ }
+ }
+
+ /**
+ * Verifies that page 2 can use only {@code continuationToken} without
{@code maxBuckets}.
+ * The first page uses {@code maxBuckets=1}; subsequent pages send only
the token.
+ */
+ @Test
+ public void testListBucketsContinuationTokenWithoutMaxBuckets() throws
Exception {
+ List<String> created = new ArrayList<>();
+ for (int i = 0; i < 3; i++) {
+ String name = uniqueObjectName("token-only-" + i);
+ s3Client.createBucket(b -> b.bucket(name));
+ created.add(name);
+ }
+
+ try {
+ List<String> retrieved = new ArrayList<>();
+ String continuationToken = null;
+ boolean firstPage = true;
+
+ do {
+ ListBucketsRequest.Builder reqBuilder = ListBucketsRequest.builder();
+ if (firstPage) {
+ reqBuilder.maxBuckets(1);
+ firstPage = false;
+ } else {
+ reqBuilder.continuationToken(continuationToken);
+ }
+
+ ListBucketsResponse response =
s3Client.listBuckets(reqBuilder.build());
+ if (continuationToken == null) {
+ assertEquals(1, response.buckets().size());
+ assertNotNull(response.continuationToken());
+ } else {
+ assertFalse(response.buckets().isEmpty());
+ }
+
+ response.buckets().stream()
+ .map(Bucket::name)
+ .filter(created::contains)
+ .forEach(retrieved::add);
+
+ continuationToken = response.continuationToken();
+ } while (continuationToken != null);
+
+ assertThat(retrieved).containsExactlyInAnyOrderElementsOf(created);
+ } finally {
+ for (String name : created) {
+ s3Client.deleteBucket(b -> b.bucket(name));
+ }
+ }
+ }
+ }
+
/**
* Integration tests for the ListDirectoryBuckets S3 API (HDDS-15450).
*
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ListBucketResponse.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ListBucketResponse.java
index 6515f1f9dda..30db1de61b4 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ListBucketResponse.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ListBucketResponse.java
@@ -41,7 +41,10 @@ public class ListBucketResponse {
@XmlElement(name = "Owner")
private S3Owner owner;
-
+
+ @XmlElement(name = "ContinuationToken")
+ private String continuationToken;
+
public List<BucketMetadata> getBuckets() {
return buckets;
}
@@ -66,4 +69,12 @@ public S3Owner getOwner() {
public void setOwner(S3Owner owner) {
this.owner = owner;
}
+
+ public String getContinuationToken() {
+ return continuationToken;
+ }
+
+ public void setContinuationToken(String continuationToken) {
+ this.continuationToken = continuationToken;
+ }
}
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/RootEndpoint.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/RootEndpoint.java
index 4cb52eaf3d5..e89d09a5432 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/RootEndpoint.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/RootEndpoint.java
@@ -153,22 +153,47 @@ private Response listAllBuckets()
long startNanos = Time.monotonicNowNanos();
boolean auditSuccess = true;
try {
+ final String continueToken =
queryParams().get(QueryParams.CONTINUATION_TOKEN);
+ final String maxBucketsParam =
queryParams().get(QueryParams.MAX_BUCKETS);
+ final boolean paginated = maxBucketsParam != null || continueToken !=
null;
+ final int maxBuckets = getMaxBuckets(paginated,
+ queryParams().getInt(QueryParams.MAX_BUCKETS,
S3Consts.MAX_BUCKETS_LIMIT));
+
ListBucketResponse response = new ListBucketResponse();
+ String previousBucket = null;
+ if (continueToken != null) {
+ previousBucket =
ContinueToken.decodeFromString(continueToken).getLastKey();
+ }
+
Iterator<? extends OzoneBucket> bucketIterator;
try {
- bucketIterator =
- listS3Buckets(null, volume ->
response.setOwner(S3Owner.of(volume.getOwner())));
+ if (previousBucket == null) {
+ bucketIterator = listS3Buckets(null,
+ volume -> response.setOwner(S3Owner.of(volume.getOwner())));
+ } else {
+ bucketIterator = listS3Buckets(null, previousBucket,
+ volume -> response.setOwner(S3Owner.of(volume.getOwner())));
+ }
} catch (Exception e) {
getMetrics().updateListS3BucketsFailureStats(startNanos);
throw e;
}
- while (bucketIterator.hasNext()) {
+ int count = 0;
+ String lastBucketName = null;
+ while (bucketIterator.hasNext() && count < maxBuckets) {
OzoneBucket next = bucketIterator.next();
BucketMetadata bucketMetadata = new BucketMetadata();
bucketMetadata.setName(next.getName());
bucketMetadata.setCreationDate(next.getCreationTime());
response.addBucket(bucketMetadata);
+ lastBucketName = next.getName();
+ count++;
+ }
+
+ if (paginated && lastBucketName != null && bucketIterator.hasNext()) {
+ response.setContinuationToken(
+ new ContinueToken(lastBucketName, null).encodeToString());
}
getMetrics().updateListS3BucketsSuccessStats(startNanos);
@@ -184,6 +209,16 @@ private Response listAllBuckets()
}
}
+ private int getMaxBuckets(boolean paginated, int requestedMaxBuckets) throws
OS3Exception {
+ if (!paginated) {
+ return Integer.MAX_VALUE;
+ }
+ if (requestedMaxBuckets < 1) {
+ throw newError(INVALID_ARGUMENT, "max-buckets must be >= 1");
+ }
+ return Math.min(requestedMaxBuckets, S3Consts.MAX_BUCKETS_LIMIT);
+ }
+
private int validateMaxDirectoryBuckets(int maxDirectoryBuckets) throws
OS3Exception {
if (maxDirectoryBuckets < 0) {
throw newError(INVALID_ARGUMENT, "max-directory-buckets must be >= 0");
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java
index 92266adbcab..df1a5e279d2 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java
@@ -115,6 +115,8 @@ public final class S3Consts {
// Constants related to S3 Express / ListDirectoryBuckets
public static final int MAX_DIRECTORY_BUCKETS_LIMIT = 1000;
+ // Maximum number of buckets per ListBuckets response page.
+ public static final int MAX_BUCKETS_LIMIT = 10000;
public static final String DEFAULT_S3_REGION = "us-east-1";
public static final String S3_EXPRESS_SERVICE = "s3express";
@@ -146,6 +148,7 @@ public static final class QueryParams {
public static final String LOCATION = "location";
public static final String MARKER = "marker";
public static final String MAX_DIRECTORY_BUCKETS = "max-directory-buckets";
+ public static final String MAX_BUCKETS = "max-buckets";
public static final String MAX_KEYS = "max-keys";
public static final String MAX_PARTS = "max-parts";
public static final String MAX_UPLOADS = "max-uploads";
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestRootList.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestRootList.java
index 915bb8ed372..4622564e743 100644
---
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestRootList.java
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestRootList.java
@@ -18,10 +18,15 @@
package org.apache.hadoop.ozone.s3.endpoint;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.client.OzoneClient;
import org.apache.hadoop.ozone.client.OzoneClientStub;
+import org.apache.hadoop.ozone.s3.exception.OS3Exception;
+import org.apache.hadoop.ozone.s3.util.S3Consts.QueryParams;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@@ -68,4 +73,93 @@ public void testListBucket() throws Exception {
assertEquals(S3Owner.DEFAULT_S3OWNER_ID, response.getOwner().getId());
}
+ @Test
+ public void testListAllBucketsPaginated() throws Exception {
+ ListBucketResponse response = listWithMaxBuckets(1);
+ assertEquals(0, response.getBucketsNum());
+ assertNull(response.getContinuationToken());
+
+ clientStub.getObjectStore().createS3Bucket("bucket-a");
+ response = listWithMaxBuckets(1);
+ assertEquals(1, response.getBucketsNum());
+ assertEquals("bucket-a", response.getBuckets().get(0).getName());
+ assertNull(response.getContinuationToken());
+
+ clientStub.getObjectStore().createS3Bucket("bucket-b");
+ response = listWithMaxBuckets(1);
+ assertEquals(1, response.getBucketsNum());
+ assertEquals("bucket-a", response.getBuckets().get(0).getName());
+ assertNotNull(response.getContinuationToken());
+
+ rootEndpoint.queryParamsForTest().set(QueryParams.CONTINUATION_TOKEN,
+ response.getContinuationToken());
+ rootEndpoint.queryParamsForTest().setInt(QueryParams.MAX_BUCKETS, 1);
+ response = (ListBucketResponse) rootEndpoint.get().getEntity();
+ assertEquals(1, response.getBucketsNum());
+ assertEquals("bucket-b", response.getBuckets().get(0).getName());
+ assertNull(response.getContinuationToken());
+ }
+
+ @Test
+ public void testListAllBucketsPaginationMultiplePages() throws Exception {
+ String bucketBaseName = "bucket-" + getClass().getName();
+ for (int i = 0; i < 5; i++) {
+ clientStub.getObjectStore().createS3Bucket(bucketBaseName + i);
+ }
+
+ ListBucketResponse response = listWithMaxBuckets(2);
+
+ assertEquals(2, response.getBucketsNum());
+ assertEquals(bucketBaseName + 0, response.getBuckets().get(0).getName());
+ assertEquals(bucketBaseName + 1, response.getBuckets().get(1).getName());
+ assertNotNull(response.getContinuationToken());
+
+ rootEndpoint.queryParamsForTest().set(QueryParams.CONTINUATION_TOKEN,
+ response.getContinuationToken());
+ response = (ListBucketResponse) rootEndpoint.get().getEntity();
+
+ assertEquals(2, response.getBucketsNum());
+ assertEquals(bucketBaseName + 2, response.getBuckets().get(0).getName());
+ assertEquals(bucketBaseName + 3, response.getBuckets().get(1).getName());
+ assertNotNull(response.getContinuationToken());
+
+ rootEndpoint.queryParamsForTest().set(QueryParams.CONTINUATION_TOKEN,
+ response.getContinuationToken());
+ response = (ListBucketResponse) rootEndpoint.get().getEntity();
+
+ assertEquals(1, response.getBucketsNum());
+ assertEquals(bucketBaseName + 4, response.getBuckets().get(0).getName());
+ assertNull(response.getContinuationToken());
+ }
+
+ @Test
+ public void testListAllBucketsInvalidMaxBuckets() {
+ rootEndpoint.queryParamsForTest().setInt(QueryParams.MAX_BUCKETS, 0);
+ assertThrows(OS3Exception.class, () -> rootEndpoint.get());
+
+ rootEndpoint.queryParamsForTest().setInt(QueryParams.MAX_BUCKETS, -1);
+ assertThrows(OS3Exception.class, () -> rootEndpoint.get());
+ }
+
+ @Test
+ public void testListAllBucketsUnpaginatedReturnsAll() throws Exception {
+ for (int i = 0; i < 3; i++) {
+ clientStub.getObjectStore().createS3Bucket("unpaginated-bucket-" + i);
+ }
+
+ rootEndpoint.queryParamsForTest().unset(QueryParams.MAX_BUCKETS);
+ rootEndpoint.queryParamsForTest().unset(QueryParams.CONTINUATION_TOKEN);
+ ListBucketResponse response =
+ (ListBucketResponse) rootEndpoint.get().getEntity();
+
+ assertEquals(3, response.getBucketsNum());
+ assertNull(response.getContinuationToken());
+ }
+
+ private ListBucketResponse listWithMaxBuckets(int maxBuckets) throws
Exception {
+ rootEndpoint.queryParamsForTest().unset(QueryParams.CONTINUATION_TOKEN);
+ rootEndpoint.queryParamsForTest().setInt(QueryParams.MAX_BUCKETS,
maxBuckets);
+ return (ListBucketResponse) rootEndpoint.get().getEntity();
+ }
+
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
