HDDS-658. Implement s3 bucket list backend call and use it from rest endpoint. Contributed by Bharat Viswanadham.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b12e6947 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b12e6947 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b12e6947 Branch: refs/heads/HDFS-12943 Commit: b12e69475bffb90b82cb7a905c96738b3fd91a7b Parents: 9b01f03 Author: Márton Elek <e...@apache.org> Authored: Thu Nov 15 12:11:05 2018 +0100 Committer: Márton Elek <e...@apache.org> Committed: Thu Nov 15 12:22:16 2018 +0100 ---------------------------------------------------------------------- .../org/apache/hadoop/ozone/OzoneConsts.java | 2 + .../apache/hadoop/ozone/client/ObjectStore.java | 113 ++++++++++++++++++- .../ozone/client/protocol/ClientProtocol.java | 16 +++ .../hadoop/ozone/client/rest/RestClient.java | 8 ++ .../hadoop/ozone/client/rpc/RpcClient.java | 19 ++++ .../org/apache/hadoop/ozone/audit/OMAction.java | 3 +- .../ozone/om/protocol/OzoneManagerProtocol.java | 23 ++++ ...neManagerProtocolClientSideTranslatorPB.java | 38 +++++++ .../src/main/proto/OzoneManagerProtocol.proto | 15 +++ .../ozone/client/rpc/TestOzoneRpcClient.java | 34 ++++++ .../org/apache/hadoop/ozone/om/OMMetrics.java | 23 ++++ .../apache/hadoop/ozone/om/OzoneManager.java | 30 +++++ .../apache/hadoop/ozone/om/S3BucketManager.java | 6 + .../hadoop/ozone/om/S3BucketManagerImpl.java | 10 +- ...neManagerProtocolServerSideTranslatorPB.java | 24 ++++ .../hadoop/ozone/om/TestS3BucketManager.java | 25 ++++ .../hadoop/ozone/s3/endpoint/EndpointBase.java | 32 ++++++ .../hadoop/ozone/s3/endpoint/RootEndpoint.java | 24 ++-- .../hadoop/ozone/client/ObjectStoreStub.java | 72 +++++++++++- .../hadoop/ozone/s3/endpoint/TestObjectGet.java | 1 - .../hadoop/ozone/s3/endpoint/TestRootList.java | 11 +- 21 files changed, 489 insertions(+), 40 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/b12e6947/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java index b77d621..d37ce0f 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java @@ -144,6 +144,8 @@ public final class OzoneConsts { public static final String OM_KEY_PREFIX = "/"; public static final String OM_USER_PREFIX = "$"; public static final String OM_S3_PREFIX ="S3:"; + public static final String OM_S3_VOLUME_PREFIX = "s3"; + /** * Max OM Quota size of 1024 PB. http://git-wip-us.apache.org/repos/asf/hadoop/blob/b12e6947/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java index 1ea5c03..136a684 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java @@ -18,17 +18,20 @@ package org.apache.hadoop.ozone.client; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Strings; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.NoSuchElementException; +import java.util.Objects; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdds.scm.client.HddsClientUtils; import org.apache.hadoop.ozone.client.protocol.ClientProtocol; import org.apache.hadoop.security.UserGroupInformation; -import java.io.IOException; -import java.util.Iterator; -import java.util.List; -import java.util.NoSuchElementException; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Strings; /** * ObjectStore class is responsible for the client operations that can be @@ -150,6 +153,36 @@ public class ObjectStore { return volume; } + /** + * Returns Iterator to iterate over all buckets for a user. + * The result can be restricted using bucket prefix, will return all + * buckets if bucket prefix is null. + * + * @param userName user name + * @param bucketPrefix Bucket prefix to match + * @return {@code Iterator<OzoneBucket>} + */ + public Iterator<? extends OzoneBucket> listS3Buckets(String userName, + String bucketPrefix) { + return listS3Buckets(userName, bucketPrefix, null); + } + + /** + * Returns Iterator to iterate over all buckets after prevBucket for a + * specific user. If prevBucket is null it returns an iterator to iterate over + * all the buckets of a user. The result can be restricted using bucket + * prefix, will return all buckets if bucket prefix is null. + * + * @param userName user name + * @param bucketPrefix Bucket prefix to match + * @param prevBucket Buckets are listed after this bucket + * @return {@code Iterator<OzoneBucket>} + */ + public Iterator<? extends OzoneBucket> listS3Buckets(String userName, + String bucketPrefix, + String prevBucket) { + return new S3BucketIterator(userName, bucketPrefix, prevBucket); + } /** * Returns Iterator to iterate over all the volumes in object store. @@ -270,4 +303,72 @@ public class ObjectStore { } } + /** + * An Iterator to iterate over {@link OzoneBucket} list. + */ + public class S3BucketIterator implements Iterator<OzoneBucket> { + + private String bucketPrefix = null; + private String userName; + + private Iterator<OzoneBucket> currentIterator; + private OzoneBucket currentValue; + + + /** + * Creates an Iterator to iterate over all buckets after prevBucket for + * a user. If prevBucket is null it returns an iterator which list all + * the buckets of the user. + * The returned buckets match bucket prefix. + * @param user + * @param bucketPrefix + * @param prevBucket + */ + public S3BucketIterator(String user, String bucketPrefix, String + prevBucket) { + Objects.requireNonNull(user); + this.userName = user; + this.bucketPrefix = bucketPrefix; + this.currentValue = null; + this.currentIterator = getNextListOfS3Buckets(prevBucket).iterator(); + } + + @Override + public boolean hasNext() { + if(!currentIterator.hasNext()) { + currentIterator = getNextListOfS3Buckets( + currentValue != null ? currentValue.getName() : null) + .iterator(); + } + return currentIterator.hasNext(); + } + + @Override + public OzoneBucket next() { + if(hasNext()) { + currentValue = currentIterator.next(); + return currentValue; + } + throw new NoSuchElementException(); + } + + /** + * Gets the next set of bucket list using proxy. + * @param prevBucket + * @return {@code List<OzoneVolume>} + */ + private List<OzoneBucket> getNextListOfS3Buckets(String prevBucket) { + try { + return proxy.listS3Buckets(userName, bucketPrefix, prevBucket, + listCacheSize); + } catch (IOException e) { + if (e.getMessage().contains("VOLUME_NOT_FOUND")) { + return new ArrayList<OzoneBucket>(); + } else { + throw new RuntimeException(e); + } + } + } + } + } http://git-wip-us.apache.org/repos/asf/hadoop/blob/b12e6947/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java index a3e49e9..be5670d 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java @@ -365,6 +365,22 @@ public interface ClientProtocol { String getOzoneBucketName(String s3BucketName) throws IOException; /** + * Returns Iterator to iterate over all buckets after prevBucket for a + * specific user. If prevBucket is null it returns an iterator to iterate over + * all the buckets of a user. The result can be restricted using bucket + * prefix, will return all buckets if bucket prefix is null. + * + * @param userName user name + * @param bucketPrefix Bucket prefix to match + * @param prevBucket Buckets are listed after this bucket + * @return {@code Iterator<OzoneBucket>} + * @throws IOException + */ + List<OzoneBucket> listS3Buckets(String userName, String bucketPrefix, + String prevBucket, int maxListResult) + throws IOException; + + /** * Close and release the resources. */ void close() throws IOException; http://git-wip-us.apache.org/repos/asf/hadoop/blob/b12e6947/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java index 8e106a5..8889d99 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java @@ -854,6 +854,14 @@ public class RestClient implements ClientProtocol { "support this operation."); } + @Override + public List<OzoneBucket> listS3Buckets(String userName, String bucketPrefix, + String prevBucket, int maxListResult) + throws IOException { + throw new UnsupportedOperationException("Ozone REST protocol does not " + + "support this operation."); + } + /** * Adds Ozone headers to http request. * http://git-wip-us.apache.org/repos/asf/hadoop/blob/b12e6947/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index 826f04b..ea002ec 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@ -623,6 +623,25 @@ public class RpcClient implements ClientProtocol { } @Override + public List<OzoneBucket> listS3Buckets(String userName, String bucketPrefix, + String prevBucket, int maxListResult) + throws IOException { + List<OmBucketInfo> buckets = ozoneManagerClient.listS3Buckets( + userName, prevBucket, bucketPrefix, maxListResult); + + return buckets.stream().map(bucket -> new OzoneBucket( + conf, + this, + bucket.getVolumeName(), + bucket.getBucketName(), + bucket.getAcls(), + bucket.getStorageType(), + bucket.getIsVersionEnabled(), + bucket.getCreationTime())) + .collect(Collectors.toList()); + } + + @Override public void close() throws IOException { IOUtils.cleanupWithLogger(LOG, storageContainerLocationClient); IOUtils.cleanupWithLogger(LOG, ozoneManagerClient); http://git-wip-us.apache.org/repos/asf/hadoop/blob/b12e6947/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/audit/OMAction.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/audit/OMAction.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/audit/OMAction.java index a0ae455..1d4d646 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/audit/OMAction.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/audit/OMAction.java @@ -45,7 +45,8 @@ public enum OMAction implements AuditAction { LIST_KEYS("LIST_KEYS"), READ_VOLUME("READ_VOLUME"), READ_BUCKET("READ_BUCKET"), - READ_KEY("READ_BUCKET"); + READ_KEY("READ_BUCKET"), + LIST_S3BUCKETS("LIST_S3BUCKETS"); private String action; http://git-wip-us.apache.org/repos/asf/hadoop/blob/b12e6947/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java index c021e64..e4cce65 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java @@ -281,5 +281,28 @@ public interface OzoneManagerProtocol { */ String getOzoneBucketMapping(String s3BucketName) throws IOException; + /** + * Returns a list of buckets represented by {@link OmBucketInfo} + * for the given user. Argument username is required, others + * are optional. + * + * @param userName + * user Name. + * @param startBucketName + * the start bucket name, only the buckets whose name is + * after this value will be included in the result. + * @param bucketPrefix + * bucket name prefix, only the buckets whose name has + * this prefix will be included in the result. + * @param maxNumOfBuckets + * the maximum number of buckets to return. It ensures + * the size of the result will not exceed this limit. + * @return a list of buckets. + * @throws IOException + */ + List<OmBucketInfo> listS3Buckets(String userName, String startBucketName, + String bucketPrefix, int maxNumOfBuckets) + throws IOException; + } http://git-wip-us.apache.org/repos/asf/hadoop/blob/b12e6947/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java index 94c57e5..6c8c932 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java @@ -126,6 +126,10 @@ import org.apache.hadoop.ozone.protocol.proto .OzoneManagerProtocolProtos.S3BucketInfoRequest; import org.apache.hadoop.ozone.protocol.proto .OzoneManagerProtocolProtos.S3BucketInfoResponse; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos + .S3ListBucketsRequest; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos + .S3ListBucketsResponse; @@ -837,6 +841,40 @@ public final class OzoneManagerProtocolClientSideTranslatorPB return resp.getOzoneMapping(); } + @Override + public List<OmBucketInfo> listS3Buckets(String userName, String startKey, + String prefix, int count) + throws IOException { + List<OmBucketInfo> buckets = new ArrayList<>(); + S3ListBucketsRequest.Builder reqBuilder = S3ListBucketsRequest.newBuilder(); + reqBuilder.setUserName(userName); + reqBuilder.setCount(count); + if (startKey != null) { + reqBuilder.setStartKey(startKey); + } + if (prefix != null) { + reqBuilder.setPrefix(prefix); + } + S3ListBucketsRequest request = reqBuilder.build(); + final S3ListBucketsResponse resp; + try { + resp = rpcProxy.listS3Buckets(NULL_RPC_CONTROLLER, request); + } catch (ServiceException e) { + throw ProtobufHelper.getRemoteException(e); + } + + if (resp.getStatus() == Status.OK) { + buckets.addAll( + resp.getBucketInfoList().stream() + .map(OmBucketInfo::getFromProtobuf) + .collect(Collectors.toList())); + return buckets; + } else { + throw new IOException("List S3 Buckets failed, error: " + + resp.getStatus()); + } + } + /** * Return the proxy object underlying this protocol translator. * http://git-wip-us.apache.org/repos/asf/hadoop/blob/b12e6947/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto ---------------------------------------------------------------------- diff --git a/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto b/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto index d3c0777..6320cf1 100644 --- a/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto +++ b/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto @@ -393,6 +393,18 @@ message S3DeleteBucketResponse { required Status status = 1; } +message S3ListBucketsRequest { + required string userName = 1; + optional string startKey = 2; + optional string prefix = 3; + optional int32 count = 4; +} + +message S3ListBucketsResponse { + required Status status = 1; + repeated BucketInfo bucketInfo = 2; +} + /** The OM service that takes care of Ozone namespace. @@ -526,4 +538,7 @@ service OzoneManagerService { */ rpc getS3Bucketinfo(S3BucketInfoRequest) returns(S3BucketInfoResponse); + + rpc listS3Buckets(S3ListBucketsRequest) + returns(S3ListBucketsResponse); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/b12e6947/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClient.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClient.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClient.java index 555b895..1ed5f67 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClient.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClient.java @@ -68,6 +68,10 @@ import java.util.Iterator; import java.util.List; import java.util.UUID; +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.either; +import static org.junit.Assert.assertThat; + /** * This class is to test all the public facing APIs of Ozone Client. */ @@ -222,6 +226,36 @@ public class TestOzoneRpcClient { Assert.assertTrue(volume.getCreationTime() >= currentTime); } + + @Test + public void testListS3Buckets() + throws IOException, OzoneException { + String userName = "ozone100"; + String bucketName1 = UUID.randomUUID().toString(); + String bucketName2 = UUID.randomUUID().toString(); + store.createS3Bucket(userName, bucketName1); + store.createS3Bucket(userName, bucketName2); + Iterator<? extends OzoneBucket> iterator = store.listS3Buckets(userName, + null); + + while (iterator.hasNext()) { + assertThat(iterator.next().getName(), either(containsString(bucketName1)) + .or(containsString(bucketName2))); + } + + } + + @Test + public void testListS3BucketsFail() + throws IOException, OzoneException { + String userName = "randomUser"; + Iterator<? extends OzoneBucket> iterator = store.listS3Buckets(userName, + null); + + Assert.assertFalse(iterator.hasNext()); + + } + @Test public void testDeleteS3Bucket() throws IOException, OzoneException { http://git-wip-us.apache.org/repos/asf/hadoop/blob/b12e6947/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetrics.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetrics.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetrics.java index b8cfc97..1b396f9 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetrics.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetrics.java @@ -60,6 +60,8 @@ public class OMMetrics { private @Metric MutableCounterLong numKeyCommits; private @Metric MutableCounterLong numAllocateBlockCalls; private @Metric MutableCounterLong numGetServiceLists; + private @Metric MutableCounterLong numListS3Buckets; + // Failure Metrics private @Metric MutableCounterLong numVolumeCreateFails; @@ -81,6 +83,7 @@ public class OMMetrics { private @Metric MutableCounterLong numKeyCommitFails; private @Metric MutableCounterLong numBlockAllocateCallFails; private @Metric MutableCounterLong numGetServiceListFails; + private @Metric MutableCounterLong numListS3BucketsFails; public OMMetrics() { } @@ -152,6 +155,16 @@ public class OMMetrics { numVolumeLists.incr(); } + public void incNumListS3Buckets() { + numBucketOps.incr(); + numListS3Buckets.incr(); + } + + public void incNumListS3BucketsFails() { + numBucketOps.incr(); + numListS3BucketsFails.incr(); + } + public void incNumGetServiceLists() { numGetServiceLists.incr(); } @@ -452,6 +465,16 @@ public class OMMetrics { return numGetServiceListFails.value(); } + @VisibleForTesting + public long getNumListS3Buckets() { + return numListS3Buckets.value(); + } + + @VisibleForTesting + public long getNumListS3BucketsFails() { + return numListS3BucketsFails.value(); + } + public void unRegister() { MetricsSystem ms = DefaultMetricsSystem.instance(); ms.unregisterSource(SOURCE_NAME); http://git-wip-us.apache.org/repos/asf/hadoop/blob/b12e6947/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index 63f0d52..da56850 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -1156,6 +1156,36 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl return s3BucketManager.getOzoneBucketMapping(s3BucketName); } + @Override + public List<OmBucketInfo> listS3Buckets(String userName, String startKey, + String prefix, int maxNumOfBuckets) + throws IOException { + boolean auditSuccess = true; + Map<String, String> auditMap = buildAuditMap(userName); + auditMap.put(OzoneConsts.START_KEY, startKey); + auditMap.put(OzoneConsts.PREFIX, prefix); + auditMap.put(OzoneConsts.MAX_NUM_OF_BUCKETS, + String.valueOf(maxNumOfBuckets)); + try { + metrics.incNumListS3Buckets(); + String volumeName = s3BucketManager.getOzoneVolumeNameForUser(userName); + return bucketManager.listBuckets(volumeName, startKey, prefix, + maxNumOfBuckets); + } catch (IOException ex) { + metrics.incNumListS3BucketsFails(); + auditSuccess = false; + AUDIT.logReadFailure(buildAuditMessageForFailure(OMAction.LIST_S3BUCKETS, + auditMap, ex)); + throw ex; + } finally { + if(auditSuccess){ + AUDIT.logReadSuccess(buildAuditMessageForSuccess(OMAction + .LIST_S3BUCKETS, auditMap)); + } + } + } + + /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/b12e6947/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/S3BucketManager.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/S3BucketManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/S3BucketManager.java index 4144662..f22af7f 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/S3BucketManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/S3BucketManager.java @@ -63,4 +63,10 @@ public interface S3BucketManager { * @throws IOException - in case of failure to retrieve mapping. */ String getOzoneBucketName(String s3BucketName) throws IOException; + + /** + * Returns volume Name for a user. + * @param userName + */ + String getOzoneVolumeNameForUser(String userName) throws IOException; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/b12e6947/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/S3BucketManagerImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/S3BucketManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/S3BucketManagerImpl.java index ca032c9..d6fd410 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/S3BucketManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/S3BucketManagerImpl.java @@ -31,7 +31,9 @@ import org.slf4j.LoggerFactory; import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.util.Objects; +import static org.apache.hadoop.ozone.OzoneConsts.OM_S3_VOLUME_PREFIX; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.FAILED_VOLUME_ALREADY_EXISTS; /** @@ -152,7 +154,7 @@ public class S3BucketManagerImpl implements S3BucketManager { } private String formatOzoneVolumeName(String userName) { - return String.format("s3%s", userName); + return String.format(OM_S3_VOLUME_PREFIX + "%s", userName); } private void createOzoneVolumeIfNeeded(String userName, String volumeName) @@ -227,4 +229,10 @@ public class S3BucketManagerImpl implements S3BucketManager { return mapping.split("/")[1]; } + @Override + public String getOzoneVolumeNameForUser(String userName) throws IOException { + Objects.requireNonNull(userName, "UserName cannot be null"); + return formatOzoneVolumeName(userName); + } + } http://git-wip-us.apache.org/repos/asf/hadoop/blob/b12e6947/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerProtocolServerSideTranslatorPB.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerProtocolServerSideTranslatorPB.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerProtocolServerSideTranslatorPB.java index 9416056..64806ed 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerProtocolServerSideTranslatorPB.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerProtocolServerSideTranslatorPB.java @@ -102,6 +102,10 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos .S3DeleteBucketResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos + .S3ListBucketsResponse; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos + .S3ListBucketsRequest; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos .ServiceListRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos .ServiceListResponse; @@ -627,4 +631,24 @@ public class OzoneManagerProtocolServerSideTranslatorPB implements } return resp.build(); } + + @Override + public S3ListBucketsResponse listS3Buckets(RpcController controller, + S3ListBucketsRequest request) { + S3ListBucketsResponse.Builder resp = S3ListBucketsResponse.newBuilder(); + try { + List<OmBucketInfo> buckets = impl.listS3Buckets( + request.getUserName(), + request.getStartKey(), + request.getPrefix(), + request.getCount()); + for(OmBucketInfo bucket : buckets) { + resp.addBucketInfo(bucket.getProtobuf()); + } + resp.setStatus(Status.OK); + } catch (IOException e) { + resp.setStatus(exceptionToResponseStatus(e)); + } + return resp.build(); + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/b12e6947/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestS3BucketManager.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestS3BucketManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestS3BucketManager.java index 75349fb..5160045 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestS3BucketManager.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestS3BucketManager.java @@ -20,6 +20,8 @@ package org.apache.hadoop.ozone.om; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.server.ServerUtils; +import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.test.GenericTestUtils; import org.junit.Assert; import org.junit.Before; import org.junit.Rule; @@ -76,6 +78,29 @@ public class TestS3BucketManager { } @Test + public void testOzoneVolumeNameForUser() throws IOException { + S3BucketManager s3BucketManager = new S3BucketManagerImpl(conf, metaMgr, + volumeManager, bucketManager); + String userName = "ozone"; + String volumeName = s3BucketManager.getOzoneVolumeNameForUser(userName); + assertEquals(OzoneConsts.OM_S3_VOLUME_PREFIX + userName, volumeName); + } + + @Test + public void testOzoneVolumeNameForUserFails() throws IOException { + S3BucketManager s3BucketManager = new S3BucketManagerImpl(conf, metaMgr, + volumeManager, bucketManager); + String userName = null; + try { + String volumeName = s3BucketManager.getOzoneVolumeNameForUser(userName); + fail("testOzoneVolumeNameForUserFails failed"); + } catch (NullPointerException ex) { + GenericTestUtils.assertExceptionContains("UserName cannot be null", ex); + } + + } + + @Test public void testDeleteS3Bucket() throws IOException { S3BucketManager s3BucketManager = new S3BucketManagerImpl(conf, metaMgr, volumeManager, bucketManager); http://git-wip-us.apache.org/repos/asf/hadoop/blob/b12e6947/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java index a5af7f1..cfa2117 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java @@ -20,6 +20,7 @@ package org.apache.hadoop.ozone.s3.endpoint; import javax.inject.Inject; import javax.ws.rs.NotFoundException; import java.io.IOException; +import java.util.Iterator; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; @@ -172,6 +173,37 @@ public class EndpointBase { return client.getObjectStore().getOzoneBucketName(s3BucketName); } + /** + * Returns Iterator to iterate over all buckets for a specific user. + * The result can be restricted using bucket prefix, will return all + * buckets if bucket prefix is null. + * + * @param userName + * @param prefix + * @return {@code Iterator<OzoneBucket>} + */ + public Iterator<? extends OzoneBucket> listS3Buckets(String userName, + String prefix) { + return client.getObjectStore().listS3Buckets(userName, prefix); + } + + /** + * Returns Iterator to iterate over all buckets after prevBucket for a + * specific user. If prevBucket is null it returns an iterator to iterate + * over all buckets for this user. The result can be restricted using + * bucket prefix, will return all buckets if bucket prefix is null. + * + * @param prefix Bucket prefix to match + * @param previousBucket Buckets are listed after this bucket + * @return {@code Iterator<OzoneBucket>} + */ + public Iterator<? extends OzoneBucket> listS3Buckets(String userName, + String prefix, + String previousBucket) { + return client.getObjectStore().listS3Buckets(userName, prefix, + previousBucket); + } + public AuthenticationHeaderParser getAuthenticationHeaderParser() { return authenticationHeaderParser; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/b12e6947/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/RootEndpoint.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/RootEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/RootEndpoint.java index 1b0e876..7ad374d 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/RootEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/RootEndpoint.java @@ -18,7 +18,6 @@ package org.apache.hadoop.ozone.s3.endpoint; import javax.ws.rs.GET; -import javax.ws.rs.NotFoundException; import javax.ws.rs.Path; import java.io.IOException; import java.time.Instant; @@ -53,25 +52,16 @@ public class RootEndpoint extends EndpointBase { OzoneVolume volume; ListBucketResponse response = new ListBucketResponse(); - String volumeName = "s3" + getAuthenticationHeaderParser().getAccessKeyID(); - try { - //TODO: we need a specific s3bucketlist endpoint instead - // of reimplement the naming convention here - volume = getVolume(volumeName); - } catch (NotFoundException ex) { - return response; - } catch (IOException e) { - throw e; - } - - Iterator<? extends OzoneBucket> volABucketIter = volume.listBuckets(null); + String userName = getAuthenticationHeaderParser().getAccessKeyID(); + Iterator<? extends OzoneBucket> bucketIterator = listS3Buckets(userName, + null); - while (volABucketIter.hasNext()) { - OzoneBucket next = volABucketIter.next(); + while (bucketIterator.hasNext()) { + OzoneBucket next = bucketIterator.next(); BucketMetadata bucketMetadata = new BucketMetadata(); bucketMetadata.setName(next.getName()); - bucketMetadata.setCreationDate( - Instant.ofEpochMilli(next.getCreationTime())); + bucketMetadata.setCreationDate(Instant.ofEpochMilli(next + .getCreationTime())); response.addBucket(bucketMetadata); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/b12e6947/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ObjectStoreStub.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ObjectStoreStub.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ObjectStoreStub.java index 4984d58..de15a40 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ObjectStoreStub.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ObjectStoreStub.java @@ -20,10 +20,7 @@ package org.apache.hadoop.ozone.client; import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Iterator; -import java.util.Map; +import java.util.*; import java.util.stream.Collectors; /** @@ -38,6 +35,7 @@ public class ObjectStoreStub extends ObjectStore { private Map<String, OzoneVolumeStub> volumes = new HashMap<>(); private Map<String, String> bucketVolumeMap = new HashMap<>(); private Map<String, Boolean> bucketEmptyStatus = new HashMap<>(); + private Map<String, List<OzoneBucket>> userBuckets = new HashMap<>(); @Override public void createVolume(String volumeName) throws IOException { @@ -113,8 +111,8 @@ public class ObjectStoreStub extends ObjectStore { @Override public void createS3Bucket(String userName, String s3BucketName) throws IOException { + String volumeName = "s3" + userName; if (bucketVolumeMap.get(s3BucketName) == null) { - String volumeName = "s3"+userName; bucketVolumeMap.put(s3BucketName, volumeName + "/" + s3BucketName); bucketEmptyStatus.put(s3BucketName, true); createVolume(volumeName); @@ -122,6 +120,70 @@ public class ObjectStoreStub extends ObjectStore { } else { throw new IOException("BUCKET_ALREADY_EXISTS"); } + + if (userBuckets.get(userName) == null) { + List<OzoneBucket> ozoneBuckets = new ArrayList<>(); + ozoneBuckets.add(volumes.get(volumeName).getBucket(s3BucketName)); + userBuckets.put(userName, ozoneBuckets); + } else { + userBuckets.get(userName).add(volumes.get(volumeName).getBucket( + s3BucketName)); + } + } + + public Iterator<? extends OzoneBucket> listS3Buckets(String userName, + String bucketPrefix) { + if (userBuckets.get(userName) == null) { + return new ArrayList<OzoneBucket>().iterator(); + } else { + return userBuckets.get(userName).parallelStream() + .filter(ozoneBucket -> { + if (bucketPrefix != null) { + return ozoneBucket.getName().startsWith(bucketPrefix); + } else { + return true; + } + }).collect(Collectors.toList()) + .iterator(); + } + } + + public Iterator<? extends OzoneBucket> listS3Buckets(String userName, + String bucketPrefix, + String prevBucket) { + + if (userBuckets.get(userName) == null) { + return new ArrayList<OzoneBucket>().iterator(); + } else { + //Sort buckets lexicographically + userBuckets.get(userName).sort( + (bucket1, bucket2) -> { + int compare = bucket1.getName().compareTo(bucket2.getName()); + if (compare < 0) { + return -1; + } else if (compare == 0) { + return 0; + } else { + return 1; + } + }); + return userBuckets.get(userName).stream() + .filter(ozoneBucket -> { + if (prevBucket != null) { + return ozoneBucket.getName().compareTo(prevBucket) > 0; + } else { + return true; + } + }) + .filter(ozoneBucket -> { + if (bucketPrefix != null) { + return ozoneBucket.getName().startsWith(bucketPrefix); + } else { + return true; + } + }).collect(Collectors.toList()) + .iterator(); + } } @Override http://git-wip-us.apache.org/repos/asf/hadoop/blob/b12e6947/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java index bd54896..2d0504d 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java @@ -51,7 +51,6 @@ public class TestObjectGet { client.getObjectStore().createS3Bucket("bilbo", "b1"); String volumeName = client.getObjectStore().getOzoneVolumeName("b1"); OzoneVolume volume = client.getObjectStore().getVolume(volumeName); - volume.createBucket("b1"); OzoneBucket bucket = volume.getBucket("b1"); OzoneOutputStream keyStream = http://git-wip-us.apache.org/repos/asf/hadoop/blob/b12e6947/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestRootList.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestRootList.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestRootList.java index 80cad3a..4f76067 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestRootList.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestRootList.java @@ -20,15 +20,12 @@ package org.apache.hadoop.ozone.s3.endpoint; - import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneClientStub; -import org.apache.hadoop.ozone.client.OzoneVolume; +import org.apache.hadoop.ozone.s3.header.AuthenticationHeaderParser; import org.apache.commons.lang3.RandomStringUtils; import static org.junit.Assert.assertEquals; - -import org.apache.hadoop.ozone.s3.header.AuthenticationHeaderParser; import org.junit.Before; import org.junit.Test; @@ -39,7 +36,6 @@ public class TestRootList { private OzoneClientStub clientStub; private ObjectStore objectStoreStub; - private OzoneVolume volumeStub; private RootEndpoint rootEndpoint; private String userName = "ozone"; @@ -49,9 +45,6 @@ public class TestRootList { //Create client stub and object store stub. clientStub = new OzoneClientStub(); objectStoreStub = clientStub.getObjectStore(); - String volumeName = "s3" + userName; - objectStoreStub.createVolume(volumeName); - volumeStub = objectStoreStub.getVolume(volumeName); // Create HeadBucket and setClient to OzoneClientStub rootEndpoint = new RootEndpoint(); @@ -71,7 +64,7 @@ public class TestRootList { String bucketBaseName = "bucket-"; for(int i = 0; i < 10; i++) { - volumeStub.createBucket( + objectStoreStub.createS3Bucket(userName, bucketBaseName + RandomStringUtils.randomNumeric(3)); } response = rootEndpoint.get(); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
