This is an automated email from the ASF dual-hosted git repository.
ivandika3 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new dac25dc6b15 HDDS-15450. Implement ListDirectoryBuckets S3 API for
Directory Buckets support (#10408)
dac25dc6b15 is described below
commit dac25dc6b15788c8e24aebc4f9d9eee65979839f
Author: Priyesh Karatha <[email protected]>
AuthorDate: Sat Jun 13 09:06:29 2026 +0530
HDDS-15450. Implement ListDirectoryBuckets S3 API for Directory Buckets
support (#10408)
---
.../hadoop/ozone/s3/awssdk/OzoneS3SDKTests.java | 1 +
.../ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java | 243 +++++++++++++++++++++
.../org/apache/hadoop/ozone/audit/S3GAction.java | 1 +
.../s3/commontypes/DirectoryBucketMetadata.java | 79 +++++++
.../s3/endpoint/ListDirectoryBucketsResponse.java | 69 ++++++
.../hadoop/ozone/s3/endpoint/RootEndpoint.java | 128 +++++++++++
.../org/apache/hadoop/ozone/s3/util/S3Consts.java | 6 +
.../hadoop/ozone/client/OzoneVolumeStub.java | 35 +--
.../s3/endpoint/TestListDirectoryBuckets.java | 187 ++++++++++++++++
9 files changed, 736 insertions(+), 13 deletions(-)
diff --git
a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/OzoneS3SDKTests.java
b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/OzoneS3SDKTests.java
index 665c9458f3c..885c3b60b17 100644
---
a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/OzoneS3SDKTests.java
+++
b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/OzoneS3SDKTests.java
@@ -48,4 +48,5 @@ public MiniOzoneCluster cluster() {
return getCluster();
}
}
+
}
diff --git
a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java
b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java
index 3a1df68154c..753e3826949 100644
---
a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java
+++
b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java
@@ -27,6 +27,7 @@
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static software.amazon.awssdk.core.sync.RequestBody.fromString;
@@ -70,6 +71,7 @@
import org.apache.hadoop.ozone.client.OzoneKeyDetails;
import org.apache.hadoop.ozone.client.OzoneVolume;
import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
+import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.ozone.s3.S3ClientFactory;
import org.apache.hadoop.ozone.s3.awssdk.S3SDKTestUtils;
import org.apache.hadoop.ozone.s3.endpoint.S3Owner;
@@ -103,6 +105,7 @@
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.S3Configuration;
import software.amazon.awssdk.services.s3.model.AbortMultipartUploadRequest;
+import software.amazon.awssdk.services.s3.model.Bucket;
import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest;
import
software.amazon.awssdk.services.s3.model.CompleteMultipartUploadResponse;
import software.amazon.awssdk.services.s3.model.CompletedMultipartUpload;
@@ -125,6 +128,8 @@
import software.amazon.awssdk.services.s3.model.HeadObjectRequest;
import software.amazon.awssdk.services.s3.model.HeadObjectResponse;
import software.amazon.awssdk.services.s3.model.ListBucketsResponse;
+import software.amazon.awssdk.services.s3.model.ListDirectoryBucketsRequest;
+import software.amazon.awssdk.services.s3.model.ListDirectoryBucketsResponse;
import software.amazon.awssdk.services.s3.model.ListMultipartUploadsRequest;
import software.amazon.awssdk.services.s3.model.ListObjectsRequest;
import software.amazon.awssdk.services.s3.model.ListObjectsResponse;
@@ -2701,4 +2706,242 @@ private void
verifyBucketOwnershipVerificationAccessDenied(Executable function)
assertEquals("Access Denied", exception.awsErrorDetails().errorCode());
}
}
+
+ /**
+ * Integration tests for the ListDirectoryBuckets S3 API (HDDS-15450).
+ *
+ * <p>These tests verify that GET / with the {@code max-directory-buckets}
query parameter
+ * correctly routes to the ListDirectoryBuckets handler and returns only FSO
(File System
+ * Optimized) buckets, while {@code ListBuckets} continues to return all
bucket types.
+ *
+ * <p>Note: passing {@code maxDirectoryBuckets} explicitly is required to
trigger the
+ * ListDirectoryBuckets routing in Ozone's S3 Gateway, because both
ListBuckets and
+ * ListDirectoryBuckets share the same {@code GET /} endpoint and must be
distinguished
+ * by either the {@code max-directory-buckets} query parameter or S3 Express
credential
+ * scope. See {@code RootEndpoint#isListDirectoryBucketsRequest()}.
+ */
+ @Nested
+ class ListDirectoryBucketsTests {
+
+ /**
+ * Verifies that only FSO (directory) buckets are returned, and OBS
buckets are excluded.
+ * Also verifies that the standard ListBuckets still returns all bucket
types.
+ */
+ @Test
+ public void testListDirectoryBucketsReturnsOnlyFSOBuckets() throws
Exception {
+ final String obsBucketName = uniqueObjectName();
+ final String fsoBucketName1 = uniqueObjectName();
+ final String fsoBucketName2 = uniqueObjectName();
+
+ s3Client.createBucket(b -> b.bucket(obsBucketName));
+ createFsoBucket(fsoBucketName1);
+ createFsoBucket(fsoBucketName2);
+ try {
+ ListDirectoryBucketsResponse response = s3Client.listDirectoryBuckets(
+ ListDirectoryBucketsRequest.builder()
+ .maxDirectoryBuckets(1000)
+ .build());
+
+ List<String> dirBucketNames = response.buckets().stream()
+ .map(Bucket::name)
+ .collect(Collectors.toList());
+
+ assertThat(dirBucketNames).contains(fsoBucketName1, fsoBucketName2);
+ assertThat(dirBucketNames).doesNotContain(obsBucketName);
+ } finally {
+ s3Client.deleteBucket(b -> b.bucket(obsBucketName));
+ deleteFsoBucket(fsoBucketName1);
+ deleteFsoBucket(fsoBucketName2);
+ }
+ }
+
+ /**
+ * Verifies that an empty result with no continuation token is returned
when no FSO
+ * buckets exist (only OBS buckets present).
+ */
+ @Test
+ public void testListDirectoryBucketsEmptyWhenNoFSOBuckets() throws
Exception {
+ final String obsBucketName = uniqueObjectName();
+
+ s3Client.createBucket(b -> b.bucket(obsBucketName));
+ try {
+ ListDirectoryBucketsResponse response = s3Client.listDirectoryBuckets(
+ ListDirectoryBucketsRequest.builder()
+ .maxDirectoryBuckets(1000)
+ .build());
+
+ List<String> dirBucketNames = response.buckets().stream()
+ .map(Bucket::name)
+ .collect(Collectors.toList());
+
+ assertThat(dirBucketNames).doesNotContain(obsBucketName);
+ assertNull(response.continuationToken());
+ } finally {
+ s3Client.deleteBucket(b -> b.bucket(obsBucketName));
+ }
+ }
+
+ /**
+ * Verifies pagination: listing FSO buckets page-by-page using {@code
maxDirectoryBuckets}
+ * and the returned continuation token, until all buckets are retrieved.
+ */
+ @Test
+ public void testListDirectoryBucketsPaginationReturnsAllBuckets() throws
Exception {
+ final int totalBuckets = 5;
+ final int pageSize = 2;
+ List<String> created = new ArrayList<>();
+
+ for (int i = 0; i < totalBuckets; i++) {
+ String name = uniqueObjectName();
+ createFsoBucket(name);
+ created.add(name);
+ }
+
+ try {
+ List<String> retrieved = new ArrayList<>();
+ String continuationToken = null;
+
+ do {
+ ListDirectoryBucketsRequest.Builder reqBuilder =
ListDirectoryBucketsRequest.builder()
+ .maxDirectoryBuckets(pageSize);
+ if (continuationToken != null) {
+ reqBuilder.continuationToken(continuationToken);
+ }
+
+ ListDirectoryBucketsResponse response =
s3Client.listDirectoryBuckets(reqBuilder.build());
+
+ response.buckets().stream()
+ .map(Bucket::name)
+ .filter(created::contains)
+ .forEach(retrieved::add);
+
+ continuationToken = response.continuationToken();
+ } while (continuationToken != null);
+
+ assertThat(retrieved).containsExactlyInAnyOrderElementsOf(created);
+ } finally {
+ for (String name : created) {
+ deleteFsoBucket(name);
+ }
+ }
+ }
+
+ /**
+ * Verifies that a single page returns no continuation token when fewer
buckets exist
+ * than the requested max.
+ */
+ @Test
+ public void
testListDirectoryBucketsNoContinuationTokenWhenResultFitsOnePage() throws
Exception {
+ final String fsoBucketName = uniqueObjectName();
+ createFsoBucket(fsoBucketName);
+
+ try {
+ ListDirectoryBucketsResponse response = s3Client.listDirectoryBuckets(
+ ListDirectoryBucketsRequest.builder()
+ .maxDirectoryBuckets(1000)
+ .build());
+
+ List<String> dirBucketNames = response.buckets().stream()
+ .map(Bucket::name)
+ .collect(Collectors.toList());
+
+ assertThat(dirBucketNames).contains(fsoBucketName);
+ assertNull(response.continuationToken(),
+ "No continuation token expected when all results fit on one page");
+ } finally {
+ deleteFsoBucket(fsoBucketName);
+ }
+ }
+
+ /**
+ * Verifies response fields: name, creationDate, bucketRegion, and
bucketArn are populated.
+ * The BucketArn must match the expected S3 Express ARN format.
+ */
+ @Test
+ public void testListDirectoryBucketsResponseFieldsArePopulated() throws
Exception {
+ final String fsoBucketName = uniqueObjectName();
+ createFsoBucket(fsoBucketName);
+
+ try {
+ ListDirectoryBucketsResponse response = s3Client.listDirectoryBuckets(
+ ListDirectoryBucketsRequest.builder()
+ .maxDirectoryBuckets(1000)
+ .build());
+
+ Bucket bucket = response.buckets().stream()
+ .filter(b -> b.name().equals(fsoBucketName))
+ .findFirst()
+ .orElse(null);
+
+ assertNotNull(bucket, "FSO bucket should be present in response");
+ assertEquals(fsoBucketName, bucket.name());
+ assertNotNull(bucket.creationDate(), "CreationDate must be set");
+ assertNotNull(bucket.bucketRegion(), "BucketRegion must be set");
+
+ // BucketArn should follow the S3 Express ARN format:
arn:aws:s3express:<region>:<accountId>:bucket/<name>
+ assertNotNull(bucket.bucketArn(), "BucketArn must be set");
+ assertThat(bucket.bucketArn())
+ .startsWith("arn:aws:s3express:")
+ .endsWith(":bucket/" + fsoBucketName);
+ } finally {
+ deleteFsoBucket(fsoBucketName);
+ }
+ }
+
+ /**
+ * Verifies that maxDirectoryBuckets=0 returns an empty result immediately.
+ */
+ @Test
+ public void testListDirectoryBucketsMaxZeroReturnsEmpty() throws Exception
{
+ final String fsoBucketName = uniqueObjectName();
+ createFsoBucket(fsoBucketName);
+
+ try {
+ ListDirectoryBucketsResponse response = s3Client.listDirectoryBuckets(
+ ListDirectoryBucketsRequest.builder()
+ .maxDirectoryBuckets(0)
+ .build());
+
+ assertEquals(0, response.buckets().size());
+ } finally {
+ deleteFsoBucket(fsoBucketName);
+ }
+ }
+
+ /**
+ * Verifies that creating an FSO bucket does not affect the standard
ListBuckets result —
+ * FSO buckets must also appear in ListBuckets (they are still
S3-accessible buckets).
+ */
+ @Test
+ public void testListBucketsIncludesFSOBuckets() throws Exception {
+ final String fsoBucketName = uniqueObjectName();
+ createFsoBucket(fsoBucketName);
+
+ try {
+ List<String> allBuckets = s3Client.listBuckets().buckets().stream()
+ .map(Bucket::name)
+ .collect(Collectors.toList());
+
+ assertThat(allBuckets).contains(fsoBucketName);
+ } finally {
+ deleteFsoBucket(fsoBucketName);
+ }
+ }
+
+ private void createFsoBucket(String bucketName) throws Exception {
+ try (OzoneClient ozoneClient = cluster.newClient()) {
+ OzoneVolume volume = ozoneClient.getObjectStore().getS3Volume();
+ volume.createBucket(bucketName, BucketArgs.newBuilder()
+ .setBucketLayout(BucketLayout.FILE_SYSTEM_OPTIMIZED)
+ .build());
+ }
+ }
+
+ private void deleteFsoBucket(String bucketName) throws Exception {
+ try (OzoneClient ozoneClient = cluster.newClient()) {
+ OzoneVolume volume = ozoneClient.getObjectStore().getS3Volume();
+ volume.deleteBucket(bucketName);
+ }
+ }
+ }
}
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/audit/S3GAction.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/audit/S3GAction.java
index 2561faa3226..969b2c04809 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/audit/S3GAction.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/audit/S3GAction.java
@@ -35,6 +35,7 @@ public enum S3GAction implements AuditAction {
//RootEndpoint
LIST_S3_BUCKETS,
+ LIST_DIRECTORY_BUCKETS,
//ObjectEndpoint
CREATE_MULTIPART_KEY,
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/commontypes/DirectoryBucketMetadata.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/commontypes/DirectoryBucketMetadata.java
new file mode 100644
index 00000000000..dd0f50114e5
--- /dev/null
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/commontypes/DirectoryBucketMetadata.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.s3.commontypes;
+
+import java.time.Instant;
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
+
+/**
+ * Metadata for one S3 directory bucket in a ListDirectoryBuckets response.
+ *
+ * @see <a
href="https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListDirectoryBuckets.html";>
+ * ListDirectoryBuckets</a>
+ */
+@XmlAccessorType(XmlAccessType.FIELD)
+public class DirectoryBucketMetadata {
+
+ @XmlElement(name = "Name")
+ private String name;
+
+ @XmlJavaTypeAdapter(IsoDateAdapter.class)
+ @XmlElement(name = "CreationDate")
+ private Instant creationDate;
+
+ @XmlElement(name = "BucketRegion")
+ private String bucketRegion;
+
+ @XmlElement(name = "BucketArn")
+ private String bucketArn;
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public Instant getCreationDate() {
+ return creationDate;
+ }
+
+ public void setCreationDate(Instant creationDate) {
+ this.creationDate = creationDate;
+ }
+
+ public String getBucketRegion() {
+ return bucketRegion;
+ }
+
+ public void setBucketRegion(String bucketRegion) {
+ this.bucketRegion = bucketRegion;
+ }
+
+ public String getBucketArn() {
+ return bucketArn;
+ }
+
+ public void setBucketArn(String bucketArn) {
+ this.bucketArn = bucketArn;
+ }
+}
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ListDirectoryBucketsResponse.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ListDirectoryBucketsResponse.java
new file mode 100644
index 00000000000..f68baa10e73
--- /dev/null
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ListDirectoryBucketsResponse.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.s3.endpoint;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.util.ArrayList;
+import java.util.List;
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlElementWrapper;
+import javax.xml.bind.annotation.XmlRootElement;
+import org.apache.hadoop.ozone.s3.commontypes.DirectoryBucketMetadata;
+import org.apache.hadoop.ozone.s3.util.S3Consts;
+
+/**
+ * Response from the ListDirectoryBuckets API call.
+ *
+ * @see <a
href="https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListDirectoryBuckets.html";>
+ * ListDirectoryBuckets</a>
+ */
+@XmlAccessorType(XmlAccessType.FIELD)
+@XmlRootElement(name = "ListDirectoryBucketsResult",
+ namespace = S3Consts.S3_XML_NAMESPACE)
+public class ListDirectoryBucketsResponse {
+
+ @XmlElementWrapper(name = "Buckets")
+ @XmlElement(name = "Bucket")
+ private List<DirectoryBucketMetadata> buckets = new ArrayList<>();
+
+ @XmlElement(name = "ContinuationToken")
+ private String continuationToken;
+
+ public List<DirectoryBucketMetadata> getBuckets() {
+ return buckets;
+ }
+
+ @VisibleForTesting
+ public int getBucketsNum() {
+ return buckets.size();
+ }
+
+ public void addBucket(DirectoryBucketMetadata bucket) {
+ buckets.add(bucket);
+ }
+
+ public String getContinuationToken() {
+ return continuationToken;
+ }
+
+ public void setContinuationToken(String continuationToken) {
+ this.continuationToken = continuationToken;
+ }
+}
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/RootEndpoint.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/RootEndpoint.java
index 9e638a112a7..4cb52eaf3d5 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/RootEndpoint.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/RootEndpoint.java
@@ -17,6 +17,9 @@
package org.apache.hadoop.ozone.s3.endpoint;
+import static
org.apache.hadoop.ozone.s3.exception.S3ErrorTable.INVALID_ARGUMENT;
+import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.newError;
+
import java.io.IOException;
import java.util.Iterator;
import javax.ws.rs.GET;
@@ -25,7 +28,11 @@
import org.apache.hadoop.ozone.audit.S3GAction;
import org.apache.hadoop.ozone.client.OzoneBucket;
import org.apache.hadoop.ozone.s3.commontypes.BucketMetadata;
+import org.apache.hadoop.ozone.s3.commontypes.DirectoryBucketMetadata;
import org.apache.hadoop.ozone.s3.exception.OS3Exception;
+import org.apache.hadoop.ozone.s3.util.ContinueToken;
+import org.apache.hadoop.ozone.s3.util.S3Consts;
+import org.apache.hadoop.ozone.s3.util.S3Consts.QueryParams;
import org.apache.hadoop.util.Time;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -48,6 +55,101 @@ public class RootEndpoint extends EndpointBase {
@GET
public Response get()
throws OS3Exception, IOException {
+ if (isS3ExpressSignedRequest()) {
+ return listDirectoryBuckets();
+ }
+ return listAllBuckets();
+ }
+
+ /**
+ * AWS SDKs and CLI sign requests to S3 Express Regional endpoints
(s3express-control.<region>.amazonaws.com)
+ * using "s3express" as the SigV4 service name in the credential scope
+ * (<date>/<region>/s3express/aws4_request), even though this is not
explicitly documented
+ * on the ListDirectoryBuckets API page.
+ */
+ private boolean isS3ExpressSignedRequest() {
+ if (signatureInfo == null) {
+ return false;
+ }
+ String credentialScope = signatureInfo.getCredentialScope();
+ if (credentialScope == null || credentialScope.isEmpty()) {
+ return false;
+ }
+ String[] parts = credentialScope.split("/");
+ return parts.length >= 3 && S3Consts.S3_EXPRESS_SERVICE.equals(parts[2]);
+ }
+
+ /**
+ * Rest endpoint to list FSO (directory) buckets owned by the current user.
+ */
+ private Response listDirectoryBuckets()
+ throws OS3Exception, IOException {
+ long startNanos = Time.monotonicNowNanos();
+ boolean auditSuccess = true;
+ try {
+ final String continueToken =
queryParams().get(QueryParams.CONTINUATION_TOKEN);
+ int maxDirectoryBuckets =
+ queryParams().getInt(QueryParams.MAX_DIRECTORY_BUCKETS,
S3Consts.MAX_DIRECTORY_BUCKETS_LIMIT);
+ maxDirectoryBuckets = validateMaxDirectoryBuckets(maxDirectoryBuckets);
+
+ ListDirectoryBucketsResponse response = new
ListDirectoryBucketsResponse();
+ String previousBucket = null;
+ if (continueToken != null) {
+ previousBucket =
ContinueToken.decodeFromString(continueToken).getLastKey();
+ }
+
+ Iterator<? extends OzoneBucket> bucketIterator;
+ try {
+ if (previousBucket == null) {
+ bucketIterator = listS3Buckets(null, volume -> { });
+ } else {
+ bucketIterator = listS3Buckets(null, previousBucket, volume -> { });
+ }
+ } catch (Exception e) {
+ getMetrics().updateListS3BucketsFailureStats(startNanos);
+ throw e;
+ }
+
+ String bucketRegion = resolveBucketRegion();
+ String accountId = S3Owner.DEFAULT_S3OWNER_ID;
+ int count = 0;
+ String lastBucketName = null;
+ while (bucketIterator.hasNext() && count < maxDirectoryBuckets) {
+ OzoneBucket bucket = bucketIterator.next();
+ if (!bucket.getBucketLayout().isFileSystemOptimized()) {
+ continue;
+ }
+ DirectoryBucketMetadata metadata = new DirectoryBucketMetadata();
+ metadata.setName(bucket.getName());
+ metadata.setCreationDate(bucket.getCreationTime());
+ metadata.setBucketRegion(bucketRegion);
+ metadata.setBucketArn(buildDirectoryBucketArn(bucketRegion, accountId,
+ bucket.getName()));
+ response.addBucket(metadata);
+ lastBucketName = bucket.getName();
+ count++;
+ }
+
+ if (lastBucketName != null && bucketIterator.hasNext()) {
+ response.setContinuationToken(
+ new ContinueToken(lastBucketName, null).encodeToString());
+ }
+
+ getMetrics().updateListS3BucketsSuccessStats(startNanos);
+ return Response.ok(response).build();
+ } catch (Exception ex) {
+ auditSuccess = false;
+ auditReadFailure(S3GAction.LIST_DIRECTORY_BUCKETS, ex);
+ throw ex;
+ } finally {
+ if (auditSuccess) {
+ auditReadSuccess(S3GAction.LIST_DIRECTORY_BUCKETS);
+ }
+ }
+ }
+
+ private Response listAllBuckets()
+ throws OS3Exception, IOException {
long startNanos = Time.monotonicNowNanos();
boolean auditSuccess = true;
try {
@@ -81,4 +183,30 @@ public Response get()
}
}
}
+
+ private int validateMaxDirectoryBuckets(int maxDirectoryBuckets) throws
OS3Exception {
+ if (maxDirectoryBuckets < 0) {
+ throw newError(INVALID_ARGUMENT, "max-directory-buckets must be >= 0");
+ }
+ return Math.min(maxDirectoryBuckets, S3Consts.MAX_DIRECTORY_BUCKETS_LIMIT);
+ }
+
+ private String resolveBucketRegion() {
+ if (signatureInfo != null) {
+ String credentialScope = signatureInfo.getCredentialScope();
+ if (credentialScope != null && !credentialScope.isEmpty()) {
+ String[] parts = credentialScope.split("/");
+ if (parts.length >= 2 && !parts[1].isEmpty()) {
+ return parts[1];
+ }
+ }
+ }
+ return S3Consts.DEFAULT_S3_REGION;
+ }
+
+ static String buildDirectoryBucketArn(String region, String accountId,
+ String bucketName) {
+ return String.format("arn:aws:s3express:%s:%s:bucket/%s",
+ region, accountId, bucketName);
+ }
}
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java
index 157324d4a24..c898ea78e4a 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java
@@ -109,6 +109,11 @@ public final class S3Consts {
public static final String IF_UNMODIFIED_SINCE_HEADER =
"If-Unmodified-Since";
+ // Constants related to S3 Express / ListDirectoryBuckets
+ public static final int MAX_DIRECTORY_BUCKETS_LIMIT = 1000;
+ public static final String DEFAULT_S3_REGION = "us-east-1";
+ public static final String S3_EXPRESS_SERVICE = "s3express";
+
//Never Constructed
private S3Consts() {
@@ -133,6 +138,7 @@ public static final class QueryParams {
// GetBucketLocation is not implemented
public static final String LOCATION = "location";
public static final String MARKER = "marker";
+ public static final String MAX_DIRECTORY_BUCKETS = "max-directory-buckets";
public static final String MAX_KEYS = "max-keys";
public static final String MAX_PARTS = "max-parts";
public static final String MAX_UPLOADS = "max-uploads";
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneVolumeStub.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneVolumeStub.java
index b19187d45c0..2919c85203b 100644
---
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneVolumeStub.java
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneVolumeStub.java
@@ -19,11 +19,13 @@
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Comparator;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
+import java.util.stream.Stream;
import org.apache.hadoop.hdds.client.DefaultReplicationConfig;
import org.apache.hadoop.hdds.client.RatisReplicationConfig;
import org.apache.hadoop.hdds.protocol.StorageType;
@@ -144,15 +146,7 @@ public OzoneBucket getBucket(String bucketName) throws
IOException {
@Override
public Iterator<? extends OzoneBucket> listBuckets(String bucketPrefix) {
- return buckets.values()
- .stream()
- .filter(bucket -> {
- if (bucketPrefix != null) {
- return bucket.getName().startsWith(bucketPrefix);
- } else {
- return true;
- }
- })
+ return listBucketsStream(bucketPrefix, null)
.collect(Collectors.toList())
.iterator();
}
@@ -160,14 +154,29 @@ public Iterator<? extends OzoneBucket> listBuckets(String
bucketPrefix) {
@Override
public Iterator<? extends OzoneBucket> listBuckets(String bucketPrefix,
String prevBucket) {
- return buckets.values()
- .stream()
- .filter(bucket -> bucket.getName().compareTo(prevBucket) > 0)
- .filter(bucket -> bucket.getName().startsWith(bucketPrefix))
+ return listBucketsStream(bucketPrefix, prevBucket)
.collect(Collectors.toList())
.iterator();
}
+ private Stream<? extends OzoneBucket> listBucketsStream(String bucketPrefix,
+ String prevBucket) {
+ return buckets.values()
+ .stream()
+ .filter(bucket -> matchesBucketPrefix(bucket, bucketPrefix))
+ .filter(bucket -> prevBucket == null
+ || bucket.getName().compareTo(prevBucket) > 0)
+ .sorted(Comparator.comparing(OzoneBucket::getName));
+ }
+
+ private static boolean matchesBucketPrefix(OzoneBucket bucket,
+ String bucketPrefix) {
+ if (bucketPrefix != null) {
+ return bucket.getName().startsWith(bucketPrefix);
+ }
+ return true;
+ }
+
@Override
public void deleteBucket(String bucketName) throws IOException {
if (!buckets.containsKey(bucketName)) {
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestListDirectoryBuckets.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestListDirectoryBuckets.java
new file mode 100644
index 00000000000..cc967a5d606
--- /dev/null
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestListDirectoryBuckets.java
@@ -0,0 +1,187 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.s3.endpoint;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.client.BucketArgs;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneClientStub;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.om.helpers.BucketLayout;
+import org.apache.hadoop.ozone.s3.commontypes.DirectoryBucketMetadata;
+import org.apache.hadoop.ozone.s3.exception.OS3Exception;
+import org.apache.hadoop.ozone.s3.signature.SignatureInfo;
+import org.apache.hadoop.ozone.s3.util.S3Consts.QueryParams;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Tests for ListDirectoryBuckets (FSO bucket listing).
+ */
+public class TestListDirectoryBuckets {
+
+ private OzoneClient clientStub;
+ private RootEndpoint rootEndpoint;
+
+ private static final String DEFAULT_VOLUME =
OzoneConfigKeys.OZONE_S3_VOLUME_NAME_DEFAULT;
+
+ @BeforeEach
+ public void setup() throws Exception {
+ clientStub = new OzoneClientStub();
+ rootEndpoint = EndpointBuilder.newRootEndpointBuilder()
+ .setClient(clientStub)
+ .setSignatureInfo(new SignatureInfo.Builder(SignatureInfo.Version.V4)
+ .setCredentialScope("20260101/us-west-2/s3express/aws4_request")
+ .build())
+ .build();
+
+ clientStub.getObjectStore().createVolume(DEFAULT_VOLUME);
+ }
+
+ @Test
+ public void testListDirectoryBucketsEmpty() throws Exception {
+
rootEndpoint.queryParamsForTest().setInt(QueryParams.MAX_DIRECTORY_BUCKETS,
1000);
+
+ ListDirectoryBucketsResponse response =
+ (ListDirectoryBucketsResponse) rootEndpoint.get().getEntity();
+
+ assertEquals(0, response.getBucketsNum());
+ assertNull(response.getContinuationToken());
+ }
+
+ @Test
+ public void testListDirectoryBucketsFiltersNonFsoBuckets() throws Exception {
+ clientStub.getObjectStore().createS3Bucket("obs-bucket");
+ createFsoBucket("fso-bucket-1");
+ createFsoBucket("fso-bucket-2");
+
+
rootEndpoint.queryParamsForTest().setInt(QueryParams.MAX_DIRECTORY_BUCKETS,
1000);
+ ListDirectoryBucketsResponse response =
+ (ListDirectoryBucketsResponse) rootEndpoint.get().getEntity();
+
+ assertEquals(2, response.getBucketsNum());
+ assertNull(response.getContinuationToken());
+
+ DirectoryBucketMetadata first = response.getBuckets().get(0);
+ assertEquals("fso-bucket-1", first.getName());
+ assertEquals("us-west-2", first.getBucketRegion());
+ assertEquals(
+ RootEndpoint.buildDirectoryBucketArn("us-west-2",
S3Owner.DEFAULT_S3OWNER_ID,
+ "fso-bucket-1"),
+ first.getBucketArn());
+ assertNotNull(first.getCreationDate());
+ }
+
+ @Test
+ public void testListDirectoryBucketsPagination() throws Exception {
+ for (int i = 0; i < 5; i++) {
+ createFsoBucket("fso-bucket-" + i);
+ }
+
+
rootEndpoint.queryParamsForTest().setInt(QueryParams.MAX_DIRECTORY_BUCKETS, 2);
+ ListDirectoryBucketsResponse response =
+ (ListDirectoryBucketsResponse) rootEndpoint.get().getEntity();
+
+ assertEquals(2, response.getBucketsNum());
+ assertEquals("fso-bucket-0", response.getBuckets().get(0).getName());
+ assertEquals("fso-bucket-1", response.getBuckets().get(1).getName());
+ assertNotNull(response.getContinuationToken());
+
+ rootEndpoint.queryParamsForTest().set(QueryParams.CONTINUATION_TOKEN,
+ response.getContinuationToken());
+ response = (ListDirectoryBucketsResponse) rootEndpoint.get().getEntity();
+
+ assertEquals(2, response.getBucketsNum());
+ assertEquals("fso-bucket-2", response.getBuckets().get(0).getName());
+ assertEquals("fso-bucket-3", response.getBuckets().get(1).getName());
+ assertNotNull(response.getContinuationToken());
+
+ rootEndpoint.queryParamsForTest().set(QueryParams.CONTINUATION_TOKEN,
+ response.getContinuationToken());
+ response = (ListDirectoryBucketsResponse) rootEndpoint.get().getEntity();
+
+ assertEquals(1, response.getBucketsNum());
+ assertEquals("fso-bucket-4", response.getBuckets().get(0).getName());
+ assertNull(response.getContinuationToken());
+ }
+
+ @Test
+ public void testListDirectoryBucketsZeroMaxBuckets() throws Exception {
+ createFsoBucket("fso-bucket");
+
+
rootEndpoint.queryParamsForTest().setInt(QueryParams.MAX_DIRECTORY_BUCKETS, 0);
+ ListDirectoryBucketsResponse response =
+ (ListDirectoryBucketsResponse) rootEndpoint.get().getEntity();
+
+ assertEquals(0, response.getBucketsNum());
+ assertNull(response.getContinuationToken());
+ }
+
+ @Test
+ public void testListDirectoryBucketsInvalidMaxBuckets() {
+
rootEndpoint.queryParamsForTest().setInt(QueryParams.MAX_DIRECTORY_BUCKETS, -1);
+ assertThrows(OS3Exception.class, () -> rootEndpoint.get());
+ }
+
+ @Test
+ public void testListDirectoryBucketsWithS3ExpressSigningOnly() throws
Exception {
+ clientStub.getObjectStore().createS3Bucket("obs-bucket");
+ createFsoBucket("fso-bucket");
+
+ rootEndpoint = EndpointBuilder.newRootEndpointBuilder()
+ .setClient(clientStub)
+ .setSignatureInfo(new SignatureInfo.Builder(SignatureInfo.Version.V4)
+ .setCredentialScope("20260101/aws-global/s3express/aws4_request")
+ .build())
+ .build();
+
+ ListDirectoryBucketsResponse response =
+ (ListDirectoryBucketsResponse) rootEndpoint.get().getEntity();
+
+ assertEquals(1, response.getBucketsNum());
+ assertEquals("fso-bucket", response.getBuckets().get(0).getName());
+ assertEquals("aws-global", response.getBuckets().get(0).getBucketRegion());
+ assertNull(response.getContinuationToken());
+ }
+
+ @Test
+ public void testListDirectoryBucketsCapsAtLimit() throws Exception {
+ for (int i = 0; i < 3; i++) {
+ createFsoBucket("fso-bucket-" + i);
+ }
+
+
rootEndpoint.queryParamsForTest().setInt(QueryParams.MAX_DIRECTORY_BUCKETS,
2000);
+ ListDirectoryBucketsResponse response =
+ (ListDirectoryBucketsResponse) rootEndpoint.get().getEntity();
+
+ assertEquals(3, response.getBucketsNum());
+ assertNull(response.getContinuationToken());
+ }
+
+ private void createFsoBucket(String bucketName) throws Exception {
+ OzoneVolume volume = clientStub.getObjectStore().getVolume(DEFAULT_VOLUME);
+ volume.createBucket(bucketName, BucketArgs.newBuilder()
+ .setBucketLayout(BucketLayout.FILE_SYSTEM_OPTIMIZED)
+ .build());
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
