Repository: jclouds Updated Branches: refs/heads/master c1ce819f6 -> 0bd295941
JCLOUDS-1125: portable list multipart uploads Only Azure, B2, and S3 support this operation. Some MultipartUpload fields become nullable. Project: http://git-wip-us.apache.org/repos/asf/jclouds/repo Commit: http://git-wip-us.apache.org/repos/asf/jclouds/commit/0bd29594 Tree: http://git-wip-us.apache.org/repos/asf/jclouds/tree/0bd29594 Diff: http://git-wip-us.apache.org/repos/asf/jclouds/diff/0bd29594 Branch: refs/heads/master Commit: 0bd29594104b9a77178aed2b721e66dd7e81da77 Parents: 445664c Author: Andrew Gaul <[email protected]> Authored: Sat Jun 11 11:17:19 2016 -0700 Committer: Andrew Gaul <[email protected]> Committed: Mon Jun 13 16:26:43 2016 -0700 ---------------------------------------------------------------------- .../jclouds/atmos/blobstore/AtmosBlobStore.java | 5 +++ .../integration/AtmosIntegrationLiveTest.java | 9 ++++ .../FilesystemBlobIntegrationTest.java | 9 ++++ .../blobstore/RegionScopedSwiftBlobStore.java | 5 +++ .../SwiftBlobIntegrationLiveTest.java | 9 ++++ .../org/jclouds/s3/blobstore/S3BlobStore.java | 20 +++++++++ .../java/org/jclouds/blobstore/BlobStore.java | 3 ++ .../blobstore/config/LocalBlobStore.java | 5 +++ .../blobstore/domain/MultipartUpload.java | 9 ++-- .../blobstore/util/ForwardingBlobStore.java | 5 +++ .../blobstore/util/ReadOnlyBlobStore.java | 6 +++ .../TransientBlobIntegrationTest.java | 9 ++++ .../internal/BaseBlobIntegrationTest.java | 44 ++++++++++++++++++++ .../azureblob/blobstore/AzureBlobStore.java | 31 ++++++++++++++ 14 files changed, 165 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/jclouds/blob/0bd29594/apis/atmos/src/main/java/org/jclouds/atmos/blobstore/AtmosBlobStore.java ---------------------------------------------------------------------- diff --git a/apis/atmos/src/main/java/org/jclouds/atmos/blobstore/AtmosBlobStore.java b/apis/atmos/src/main/java/org/jclouds/atmos/blobstore/AtmosBlobStore.java index a263af9..1a0e74b 100644 --- a/apis/atmos/src/main/java/org/jclouds/atmos/blobstore/AtmosBlobStore.java +++ b/apis/atmos/src/main/java/org/jclouds/atmos/blobstore/AtmosBlobStore.java @@ -322,6 +322,11 @@ public class AtmosBlobStore extends BaseBlobStore { } @Override + public List<MultipartUpload> listMultipartUploads(String container) { + throw new UnsupportedOperationException("Atmos does not support multipart uploads"); + } + + @Override public long getMinimumMultipartPartSize() { throw new UnsupportedOperationException("Atmos does not support multipart uploads"); } http://git-wip-us.apache.org/repos/asf/jclouds/blob/0bd29594/apis/atmos/src/test/java/org/jclouds/atmos/blobstore/integration/AtmosIntegrationLiveTest.java ---------------------------------------------------------------------- diff --git a/apis/atmos/src/test/java/org/jclouds/atmos/blobstore/integration/AtmosIntegrationLiveTest.java b/apis/atmos/src/test/java/org/jclouds/atmos/blobstore/integration/AtmosIntegrationLiveTest.java index 640524f..4f6e991 100644 --- a/apis/atmos/src/test/java/org/jclouds/atmos/blobstore/integration/AtmosIntegrationLiveTest.java +++ b/apis/atmos/src/test/java/org/jclouds/atmos/blobstore/integration/AtmosIntegrationLiveTest.java @@ -150,6 +150,15 @@ public class AtmosIntegrationLiveTest extends BaseBlobIntegrationTest { } @Override + public void testListMultipartUploads() throws Exception { + try { + super.testListMultipartUploads(); + } catch (UnsupportedOperationException uoe) { + throw new SkipException("Atmos does not support multipart uploads", uoe); + } + } + + @Override public void testPutMultipartByteSource() throws Exception { throw new SkipException("Atmos does not support multipart uploads"); } http://git-wip-us.apache.org/repos/asf/jclouds/blob/0bd29594/apis/filesystem/src/test/java/org/jclouds/filesystem/integration/FilesystemBlobIntegrationTest.java ---------------------------------------------------------------------- diff --git a/apis/filesystem/src/test/java/org/jclouds/filesystem/integration/FilesystemBlobIntegrationTest.java b/apis/filesystem/src/test/java/org/jclouds/filesystem/integration/FilesystemBlobIntegrationTest.java index d469037..fd75a94 100644 --- a/apis/filesystem/src/test/java/org/jclouds/filesystem/integration/FilesystemBlobIntegrationTest.java +++ b/apis/filesystem/src/test/java/org/jclouds/filesystem/integration/FilesystemBlobIntegrationTest.java @@ -93,4 +93,13 @@ public class FilesystemBlobIntegrationTest extends BaseBlobIntegrationTest { public void testSetBlobAccess() throws Exception { throw new SkipException("filesystem does not support anonymous access"); } + + @Test(groups = { "integration", "live" }) + public void testListMultipartUploads() throws Exception { + try { + super.testListMultipartUploads(); + } catch (UnsupportedOperationException uoe) { + throw new SkipException("filesystem does not support listing multipart uploads"); + } + } } http://git-wip-us.apache.org/repos/asf/jclouds/blob/0bd29594/apis/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/blobstore/RegionScopedSwiftBlobStore.java ---------------------------------------------------------------------- diff --git a/apis/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/blobstore/RegionScopedSwiftBlobStore.java b/apis/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/blobstore/RegionScopedSwiftBlobStore.java index cdd305a..20bdaef 100644 --- a/apis/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/blobstore/RegionScopedSwiftBlobStore.java +++ b/apis/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/blobstore/RegionScopedSwiftBlobStore.java @@ -517,6 +517,11 @@ public class RegionScopedSwiftBlobStore implements BlobStore { } @Override + public List<MultipartUpload> listMultipartUploads(String container) { + throw new UnsupportedOperationException(); + } + + @Override public long getMinimumMultipartPartSize() { return 1024 * 1024 + 1; } http://git-wip-us.apache.org/repos/asf/jclouds/blob/0bd29594/apis/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/blobstore/integration/SwiftBlobIntegrationLiveTest.java ---------------------------------------------------------------------- diff --git a/apis/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/blobstore/integration/SwiftBlobIntegrationLiveTest.java b/apis/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/blobstore/integration/SwiftBlobIntegrationLiveTest.java index 441b6a2..c60586f 100644 --- a/apis/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/blobstore/integration/SwiftBlobIntegrationLiveTest.java +++ b/apis/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/blobstore/integration/SwiftBlobIntegrationLiveTest.java @@ -112,6 +112,15 @@ public class SwiftBlobIntegrationLiveTest extends BaseBlobIntegrationTest { super.testCopyIfNoneMatchNegative(); } + @Override + public void testListMultipartUploads() throws Exception { + try { + super.testListMultipartUploads(); + } catch (UnsupportedOperationException uoe) { + throw new SkipException("Swift does not support listing multipart uploads", uoe); + } + } + // TODO: testCopyIfModifiedSinceNegative throws HTTP 304 not 412 error @Override http://git-wip-us.apache.org/repos/asf/jclouds/blob/0bd29594/apis/s3/src/main/java/org/jclouds/s3/blobstore/S3BlobStore.java ---------------------------------------------------------------------- diff --git a/apis/s3/src/main/java/org/jclouds/s3/blobstore/S3BlobStore.java b/apis/s3/src/main/java/org/jclouds/s3/blobstore/S3BlobStore.java index ef3729f..e5d56b9 100644 --- a/apis/s3/src/main/java/org/jclouds/s3/blobstore/S3BlobStore.java +++ b/apis/s3/src/main/java/org/jclouds/s3/blobstore/S3BlobStore.java @@ -64,6 +64,7 @@ import org.jclouds.s3.domain.AccessControlList.GroupGranteeURI; import org.jclouds.s3.domain.AccessControlList.Permission; import org.jclouds.s3.domain.BucketMetadata; import org.jclouds.s3.domain.CannedAccessPolicy; +import org.jclouds.s3.domain.ListMultipartUploadsResponse; import org.jclouds.s3.options.CopyObjectOptions; import org.jclouds.s3.options.ListBucketOptions; import org.jclouds.s3.options.PutBucketOptions; @@ -401,6 +402,25 @@ public class S3BlobStore extends BaseBlobStore { } @Override + public List<MultipartUpload> listMultipartUploads(String container) { + ImmutableList.Builder<MultipartUpload> builder = ImmutableList.builder(); + String keyMarker = null; + String uploadIdMarker = null; + while (true) { + ListMultipartUploadsResponse response = sync.listMultipartUploads(container, null, null, keyMarker, null, uploadIdMarker); + for (ListMultipartUploadsResponse.Upload upload : response.uploads()) { + builder.add(MultipartUpload.create(container, upload.key(), upload.uploadId(), null, null)); + } + keyMarker = response.keyMarker(); + uploadIdMarker = response.uploadIdMarker(); + if (response.uploads().isEmpty() || keyMarker == null || uploadIdMarker == null) { + break; + } + } + return builder.build(); + } + + @Override public long getMinimumMultipartPartSize() { return 5 * 1024 * 1024; } http://git-wip-us.apache.org/repos/asf/jclouds/blob/0bd29594/blobstore/src/main/java/org/jclouds/blobstore/BlobStore.java ---------------------------------------------------------------------- diff --git a/blobstore/src/main/java/org/jclouds/blobstore/BlobStore.java b/blobstore/src/main/java/org/jclouds/blobstore/BlobStore.java index 4cbc92e..933675c 100644 --- a/blobstore/src/main/java/org/jclouds/blobstore/BlobStore.java +++ b/blobstore/src/main/java/org/jclouds/blobstore/BlobStore.java @@ -349,6 +349,9 @@ public interface BlobStore { List<MultipartPart> listMultipartUpload(MultipartUpload mpu); @Beta + List<MultipartUpload> listMultipartUploads(String container); + + @Beta long getMinimumMultipartPartSize(); @Beta http://git-wip-us.apache.org/repos/asf/jclouds/blob/0bd29594/blobstore/src/main/java/org/jclouds/blobstore/config/LocalBlobStore.java ---------------------------------------------------------------------- diff --git a/blobstore/src/main/java/org/jclouds/blobstore/config/LocalBlobStore.java b/blobstore/src/main/java/org/jclouds/blobstore/config/LocalBlobStore.java index f48e2da..b441f1b 100644 --- a/blobstore/src/main/java/org/jclouds/blobstore/config/LocalBlobStore.java +++ b/blobstore/src/main/java/org/jclouds/blobstore/config/LocalBlobStore.java @@ -882,6 +882,11 @@ public final class LocalBlobStore implements BlobStore { } @Override + public List<MultipartUpload> listMultipartUploads(String container) { + throw new UnsupportedOperationException(); + } + + @Override public long getMinimumMultipartPartSize() { return 1; } http://git-wip-us.apache.org/repos/asf/jclouds/blob/0bd29594/blobstore/src/main/java/org/jclouds/blobstore/domain/MultipartUpload.java ---------------------------------------------------------------------- diff --git a/blobstore/src/main/java/org/jclouds/blobstore/domain/MultipartUpload.java b/blobstore/src/main/java/org/jclouds/blobstore/domain/MultipartUpload.java index 94f2401..385a9a1 100644 --- a/blobstore/src/main/java/org/jclouds/blobstore/domain/MultipartUpload.java +++ b/blobstore/src/main/java/org/jclouds/blobstore/domain/MultipartUpload.java @@ -18,6 +18,7 @@ package org.jclouds.blobstore.domain; import org.jclouds.blobstore.options.PutOptions; +import org.jclouds.javax.annotation.Nullable; import com.google.auto.value.AutoValue; @@ -26,11 +27,11 @@ public abstract class MultipartUpload { public abstract String containerName(); public abstract String blobName(); public abstract String id(); - public abstract BlobMetadata blobMetadata(); - public abstract PutOptions putOptions(); + @Nullable public abstract BlobMetadata blobMetadata(); + @Nullable public abstract PutOptions putOptions(); - public static MultipartUpload create(String containerName, String blobName, String id, BlobMetadata blobMetadata, - PutOptions putOptions) { + public static MultipartUpload create(String containerName, String blobName, String id, @Nullable BlobMetadata blobMetadata, + @Nullable PutOptions putOptions) { return new AutoValue_MultipartUpload(containerName, blobName, id, blobMetadata, putOptions); } } http://git-wip-us.apache.org/repos/asf/jclouds/blob/0bd29594/blobstore/src/main/java/org/jclouds/blobstore/util/ForwardingBlobStore.java ---------------------------------------------------------------------- diff --git a/blobstore/src/main/java/org/jclouds/blobstore/util/ForwardingBlobStore.java b/blobstore/src/main/java/org/jclouds/blobstore/util/ForwardingBlobStore.java index 2a4631b..b25d56e 100644 --- a/blobstore/src/main/java/org/jclouds/blobstore/util/ForwardingBlobStore.java +++ b/blobstore/src/main/java/org/jclouds/blobstore/util/ForwardingBlobStore.java @@ -245,6 +245,11 @@ public abstract class ForwardingBlobStore extends ForwardingObject } @Override + public List<MultipartUpload> listMultipartUploads(String container) { + return delegate().listMultipartUploads(container); + } + + @Override public long getMinimumMultipartPartSize() { return delegate().getMinimumMultipartPartSize(); } http://git-wip-us.apache.org/repos/asf/jclouds/blob/0bd29594/blobstore/src/main/java/org/jclouds/blobstore/util/ReadOnlyBlobStore.java ---------------------------------------------------------------------- diff --git a/blobstore/src/main/java/org/jclouds/blobstore/util/ReadOnlyBlobStore.java b/blobstore/src/main/java/org/jclouds/blobstore/util/ReadOnlyBlobStore.java index 3cfbb73..af82dab 100644 --- a/blobstore/src/main/java/org/jclouds/blobstore/util/ReadOnlyBlobStore.java +++ b/blobstore/src/main/java/org/jclouds/blobstore/util/ReadOnlyBlobStore.java @@ -159,4 +159,10 @@ public final class ReadOnlyBlobStore extends ForwardingBlobStore { public List<MultipartPart> listMultipartUpload(MultipartUpload mpu) { throw new UnsupportedOperationException("Read-only BlobStore"); } + + // TODO: should ReadOnlyBlobStore allow listing parts and uploads? + @Override + public List<MultipartUpload> listMultipartUploads(String container) { + throw new UnsupportedOperationException("Read-only BlobStore"); + } } http://git-wip-us.apache.org/repos/asf/jclouds/blob/0bd29594/blobstore/src/test/java/org/jclouds/blobstore/integration/TransientBlobIntegrationTest.java ---------------------------------------------------------------------- diff --git a/blobstore/src/test/java/org/jclouds/blobstore/integration/TransientBlobIntegrationTest.java b/blobstore/src/test/java/org/jclouds/blobstore/integration/TransientBlobIntegrationTest.java index e2b842e..739e7ba 100644 --- a/blobstore/src/test/java/org/jclouds/blobstore/integration/TransientBlobIntegrationTest.java +++ b/blobstore/src/test/java/org/jclouds/blobstore/integration/TransientBlobIntegrationTest.java @@ -31,4 +31,13 @@ public class TransientBlobIntegrationTest extends BaseBlobIntegrationTest { public void testSetBlobAccess() throws Exception { throw new SkipException("transient does not support anonymous access"); } + + @Test(groups = { "integration", "live" }) + public void testListMultipartUploads() throws Exception { + try { + super.testListMultipartUploads(); + } catch (UnsupportedOperationException uoe) { + throw new SkipException("transient does not support listing multipart uploads"); + } + } } http://git-wip-us.apache.org/repos/asf/jclouds/blob/0bd29594/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseBlobIntegrationTest.java ---------------------------------------------------------------------- diff --git a/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseBlobIntegrationTest.java b/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseBlobIntegrationTest.java index b2a501e..2acb165 100644 --- a/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseBlobIntegrationTest.java +++ b/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseBlobIntegrationTest.java @@ -1275,6 +1275,50 @@ public class BaseBlobIntegrationTest extends BaseBlobStoreIntegrationTest { } } + @Test(groups = { "integration", "live" }) + public void testListMultipartUploads() throws Exception { + BlobStore blobStore = view.getBlobStore(); + String name = "blob-name"; + PayloadBlobBuilder blobBuilder = blobStore.blobBuilder(name) + .userMetadata(ImmutableMap.of("key1", "value1", "key2", "value2")) + // TODO: fake payload to add content metadata + .payload(new byte[0]); + addContentMetadata(blobBuilder); + Blob blob = blobBuilder.build(); + MultipartUpload mpu = null; + + String container = getContainerName(); + try { + List<MultipartUpload> uploads = blobStore.listMultipartUploads(container); + assertThat(uploads).isEmpty(); + + mpu = blobStore.initiateMultipartUpload(container, blob.getMetadata(), new PutOptions()); + + // some providers like Azure cannot list an MPU until the first blob is uploaded + assertThat(uploads.size()).isBetween(0, 1); + + ByteSource byteSource = TestUtils.randomByteSource().slice(0, 1); + Payload payload = Payloads.newByteSourcePayload(byteSource); + payload.getContentMetadata().setContentLength(byteSource.size()); + MultipartPart part = blobStore.uploadMultipartPart(mpu, 1, payload); + + uploads = blobStore.listMultipartUploads(container); + assertThat(uploads).hasSize(1); + + blobStore.completeMultipartUpload(mpu, ImmutableList.of(part)); + + uploads = blobStore.listMultipartUploads(container); + assertThat(uploads).isEmpty(); + + // cannot test abort since Azure does not have explicit support + } finally { + if (mpu != null) { + blobStore.abortMultipartUpload(mpu); + } + returnContainer(container); + } + } + @Test(groups = { "integration", "live" }, expectedExceptions = {KeyNotFoundException.class}) public void testCopy404BlobFail() throws Exception { BlobStore blobStore = view.getBlobStore(); http://git-wip-us.apache.org/repos/asf/jclouds/blob/0bd29594/providers/azureblob/src/main/java/org/jclouds/azureblob/blobstore/AzureBlobStore.java ---------------------------------------------------------------------- diff --git a/providers/azureblob/src/main/java/org/jclouds/azureblob/blobstore/AzureBlobStore.java b/providers/azureblob/src/main/java/org/jclouds/azureblob/blobstore/AzureBlobStore.java index d839024..c7007e5 100644 --- a/providers/azureblob/src/main/java/org/jclouds/azureblob/blobstore/AzureBlobStore.java +++ b/providers/azureblob/src/main/java/org/jclouds/azureblob/blobstore/AzureBlobStore.java @@ -20,6 +20,7 @@ import static com.google.common.base.Preconditions.checkNotNull; import static org.jclouds.azure.storage.options.ListOptions.Builder.includeMetadata; import java.net.URI; +import java.util.EnumSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -38,8 +39,11 @@ import org.jclouds.azureblob.blobstore.functions.ListBlobsResponseToResourceList import org.jclouds.azureblob.blobstore.functions.ListOptionsToListBlobsOptions; import org.jclouds.azureblob.domain.AzureBlob; import org.jclouds.azureblob.domain.BlobBlockProperties; +import org.jclouds.azureblob.domain.BlobProperties; import org.jclouds.azureblob.domain.ContainerProperties; import org.jclouds.azureblob.domain.ListBlobBlocksResponse; +import org.jclouds.azureblob.domain.ListBlobsInclude; +import org.jclouds.azureblob.domain.ListBlobsResponse; import org.jclouds.azureblob.domain.PublicAccess; import org.jclouds.azureblob.options.CopyBlobOptions; import org.jclouds.azureblob.options.ListBlobsOptions; @@ -454,6 +458,33 @@ public class AzureBlobStore extends BaseBlobStore { } @Override + public List<MultipartUpload> listMultipartUploads(String container) { + ImmutableList.Builder<MultipartUpload> builder = ImmutableList.builder(); + String marker = null; + while (true) { + ListBlobsOptions options = new ListBlobsOptions().include(EnumSet.of(ListBlobsInclude.UNCOMMITTEDBLOBS)); + if (marker != null) { + options.marker(marker); + } + ListBlobsResponse response = sync.listBlobs(container, options); + for (BlobProperties properties : response) { + // only uncommitted blobs lack ETags + if (properties.getETag() != null) { + continue; + } + // TODO: bogus uploadId + String uploadId = UUID.randomUUID().toString(); + builder.add(MultipartUpload.create(properties.getContainer(), properties.getName(), uploadId, null, null)); + } + marker = response.getNextMarker(); + if (marker == null) { + break; + } + } + return builder.build(); + } + + @Override public long getMinimumMultipartPartSize() { return 1; }
