Repository: cloudstack Updated Branches: refs/heads/master 7c1590ffd -> 0b6c540a2
listDirectory method updated to use ObjectListing.isTruncated(). Because buckets can contain a virtually unlimited number of keys, the complete results of a list query can be extremely large. To manage large result sets, Amazon S3 uses pagination to split them into multiple responses. Signed-off-by: Rajani Karuturi <[email protected]> This closes #25 Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/0b6c540a Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/0b6c540a Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/0b6c540a Branch: refs/heads/master Commit: 0b6c540a203453580bb76a0d0a60be292fccfa0e Parents: 7c1590f Author: santhosh <[email protected]> Authored: Fri Oct 24 21:15:29 2014 +0530 Committer: Rajani Karuturi <[email protected]> Committed: Wed Mar 4 12:52:48 2015 +0530 ---------------------------------------------------------------------- utils/src/com/cloud/utils/S3Utils.java | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cloudstack/blob/0b6c540a/utils/src/com/cloud/utils/S3Utils.java ---------------------------------------------------------------------- diff --git a/utils/src/com/cloud/utils/S3Utils.java b/utils/src/com/cloud/utils/S3Utils.java index d9b850d..37aeb91 100644 --- a/utils/src/com/cloud/utils/S3Utils.java +++ b/utils/src/com/cloud/utils/S3Utils.java @@ -57,13 +57,14 @@ import com.amazonaws.services.s3.AmazonS3Client; import com.amazonaws.services.s3.model.Bucket; import com.amazonaws.services.s3.model.CannedAccessControlList; import com.amazonaws.services.s3.model.GetObjectRequest; +import com.amazonaws.services.s3.model.ListObjectsRequest; +import com.amazonaws.services.s3.model.ObjectListing; import com.amazonaws.services.s3.model.ObjectMetadata; import com.amazonaws.services.s3.model.PutObjectRequest; import com.amazonaws.services.s3.model.S3Object; import com.amazonaws.services.s3.model.S3ObjectSummary; import com.amazonaws.services.s3.transfer.TransferManager; import com.amazonaws.services.s3.transfer.Upload; - import com.cloud.utils.exception.CloudRuntimeException; public final class S3Utils { @@ -338,9 +339,19 @@ public final class S3Utils { private static List<S3ObjectSummary> listDirectory(final String bucketName, final String directory, final AmazonS3 client) { - final List<S3ObjectSummary> objects = client.listObjects(bucketName, directory + SEPARATOR).getObjectSummaries(); - - if (objects == null) { + List<S3ObjectSummary> objects = new ArrayList<S3ObjectSummary>(); + ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName).withPrefix(directory + SEPARATOR); + ObjectListing objectListing; + + do { + objectListing = client.listObjects(listObjectsRequest); + + if (objectListing != null ) + objects.addAll(objectListing.getObjectSummaries()); + listObjectsRequest.setMarker(objectListing.getNextMarker()); + } while (objectListing.isTruncated()); + + if (objects.isEmpty()) { return emptyList(); }
