JCLOUDS-1368: Correct use of slicing algorithm MultipartUploadSlicingAlgorithm creates multiple equal-sized parts and a remaining amount. BaseBlobStore.putMultipartBlob used this interface incorrectly, which could create more parts than intended since the remaining size could be larger than the part size. This manifested with Google Cloud Storage which only allows 32 parts.
Project: http://git-wip-us.apache.org/repos/asf/jclouds/repo Commit: http://git-wip-us.apache.org/repos/asf/jclouds/commit/d05be896 Tree: http://git-wip-us.apache.org/repos/asf/jclouds/tree/d05be896 Diff: http://git-wip-us.apache.org/repos/asf/jclouds/diff/d05be896 Branch: refs/heads/master Commit: d05be89614b4539b2ac16e911439dcb3df2e23df Parents: 52c92a9 Author: Andrew Gaul <[email protected]> Authored: Thu Jan 4 14:52:10 2018 -0800 Committer: Andrew Gaul <[email protected]> Committed: Tue Jan 9 12:56:00 2018 -0800 ---------------------------------------------------------------------- .../org/jclouds/blobstore/internal/BaseBlobStore.java | 12 +++++++++++- .../internal/MultipartUploadSlicingAlgorithm.java | 8 +++----- 2 files changed, 14 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/jclouds/blob/d05be896/blobstore/src/main/java/org/jclouds/blobstore/internal/BaseBlobStore.java ---------------------------------------------------------------------- diff --git a/blobstore/src/main/java/org/jclouds/blobstore/internal/BaseBlobStore.java b/blobstore/src/main/java/org/jclouds/blobstore/internal/BaseBlobStore.java index 087e9f6..97d4553 100644 --- a/blobstore/src/main/java/org/jclouds/blobstore/internal/BaseBlobStore.java +++ b/blobstore/src/main/java/org/jclouds/blobstore/internal/BaseBlobStore.java @@ -354,14 +354,24 @@ public abstract class BaseBlobStore implements BlobStore { MultipartUpload mpu = initiateMultipartUpload(container, blob.getMetadata(), overrides); try { long contentLength = blob.getMetadata().getContentMetadata().getContentLength(); + // TODO: inject MultipartUploadSlicingAlgorithm to override default part size MultipartUploadSlicingAlgorithm algorithm = new MultipartUploadSlicingAlgorithm( getMinimumMultipartPartSize(), getMaximumMultipartPartSize(), getMaximumNumberOfParts()); long partSize = algorithm.calculateChunkSize(contentLength); int partNumber = 1; - for (Payload payload : slicer.slice(blob.getPayload(), partSize)) { + // TODO: for InputStream payloads, this buffers all parts in-memory! + while (partNumber < algorithm.getParts()) { + Payload payload = slicer.slice(blob.getPayload(), algorithm.getCopied(), partSize); BlobUploader b = new BlobUploader(mpu, partNumber++, payload); parts.add(executor.submit(b)); + algorithm.addCopied(partSize); + } + if (algorithm.getRemaining() != 0) { + Payload payload = slicer.slice(blob.getPayload(), algorithm.getCopied(), algorithm.getRemaining()); + BlobUploader b = + new BlobUploader(mpu, partNumber, payload); + parts.add(executor.submit(b)); } return completeMultipartUpload(mpu, Futures.getUnchecked(Futures.allAsList(parts))); } catch (RuntimeException re) { http://git-wip-us.apache.org/repos/asf/jclouds/blob/d05be896/blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/MultipartUploadSlicingAlgorithm.java ---------------------------------------------------------------------- diff --git a/blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/MultipartUploadSlicingAlgorithm.java b/blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/MultipartUploadSlicingAlgorithm.java index 455ab62..eb4074f 100644 --- a/blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/MultipartUploadSlicingAlgorithm.java +++ b/blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/MultipartUploadSlicingAlgorithm.java @@ -132,8 +132,7 @@ public final class MultipartUploadSlicingAlgorithm { this.copied = copied; } - @VisibleForTesting - protected int getParts() { + public int getParts() { return parts; } @@ -141,7 +140,7 @@ public final class MultipartUploadSlicingAlgorithm { return ++part; } - protected void addCopied(long copied) { + public void addCopied(long copied) { this.copied += copied; } @@ -156,8 +155,7 @@ public final class MultipartUploadSlicingAlgorithm { return chunkSize; } - @VisibleForTesting - protected long getRemaining() { + public long getRemaining() { return remaining; }
