Repository: jclouds Updated Branches: refs/heads/2.0.x 222099660 -> 382c9201e
JCLOUDS-1368: Correct use of slicing algorithm MultipartUploadSlicingAlgorithm creates multiple equal-sized parts and a remaining amount. BaseBlobStore.putMultipartBlob used this interface incorrectly, which could create more parts than intended since the remaining size could be larger than the part size. This manifested with Google Cloud Storage which only allows 32 parts. Project: http://git-wip-us.apache.org/repos/asf/jclouds/repo Commit: http://git-wip-us.apache.org/repos/asf/jclouds/commit/382c9201 Tree: http://git-wip-us.apache.org/repos/asf/jclouds/tree/382c9201 Diff: http://git-wip-us.apache.org/repos/asf/jclouds/diff/382c9201 Branch: refs/heads/2.0.x Commit: 382c9201e37eb5ddfe5e9ab06a6ec73e1fb2cba5 Parents: 2220996 Author: Andrew Gaul <[email protected]> Authored: Thu Jan 4 14:52:10 2018 -0800 Committer: Andrew Gaul <[email protected]> Committed: Tue Jan 9 12:58:18 2018 -0800 ---------------------------------------------------------------------- .../org/jclouds/blobstore/internal/BaseBlobStore.java | 12 +++++++++++- .../internal/MultipartUploadSlicingAlgorithm.java | 8 +++----- 2 files changed, 14 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/jclouds/blob/382c9201/blobstore/src/main/java/org/jclouds/blobstore/internal/BaseBlobStore.java ---------------------------------------------------------------------- diff --git a/blobstore/src/main/java/org/jclouds/blobstore/internal/BaseBlobStore.java b/blobstore/src/main/java/org/jclouds/blobstore/internal/BaseBlobStore.java index 4e050c8..201bc3d 100644 --- a/blobstore/src/main/java/org/jclouds/blobstore/internal/BaseBlobStore.java +++ b/blobstore/src/main/java/org/jclouds/blobstore/internal/BaseBlobStore.java @@ -356,14 +356,24 @@ public abstract class BaseBlobStore implements BlobStore { MultipartUpload mpu = initiateMultipartUpload(container, blob.getMetadata(), overrides); try { long contentLength = blob.getMetadata().getContentMetadata().getContentLength(); + // TODO: inject MultipartUploadSlicingAlgorithm to override default part size MultipartUploadSlicingAlgorithm algorithm = new MultipartUploadSlicingAlgorithm( getMinimumMultipartPartSize(), getMaximumMultipartPartSize(), getMaximumNumberOfParts()); long partSize = algorithm.calculateChunkSize(contentLength); int partNumber = 1; - for (Payload payload : slicer.slice(blob.getPayload(), partSize)) { + // TODO: for InputStream payloads, this buffers all parts in-memory! + while (partNumber < algorithm.getParts()) { + Payload payload = slicer.slice(blob.getPayload(), algorithm.getCopied(), partSize); BlobUploader b = new BlobUploader(mpu, partNumber++, payload); parts.add(executor.submit(b)); + algorithm.addCopied(partSize); + } + if (algorithm.getRemaining() != 0) { + Payload payload = slicer.slice(blob.getPayload(), algorithm.getCopied(), algorithm.getRemaining()); + BlobUploader b = + new BlobUploader(mpu, partNumber, payload); + parts.add(executor.submit(b)); } return completeMultipartUpload(mpu, Futures.getUnchecked(Futures.allAsList(parts))); } catch (RuntimeException re) { http://git-wip-us.apache.org/repos/asf/jclouds/blob/382c9201/blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/MultipartUploadSlicingAlgorithm.java ---------------------------------------------------------------------- diff --git a/blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/MultipartUploadSlicingAlgorithm.java b/blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/MultipartUploadSlicingAlgorithm.java index 455ab62..eb4074f 100644 --- a/blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/MultipartUploadSlicingAlgorithm.java +++ b/blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/MultipartUploadSlicingAlgorithm.java @@ -132,8 +132,7 @@ public final class MultipartUploadSlicingAlgorithm { this.copied = copied; } - @VisibleForTesting - protected int getParts() { + public int getParts() { return parts; } @@ -141,7 +140,7 @@ public final class MultipartUploadSlicingAlgorithm { return ++part; } - protected void addCopied(long copied) { + public void addCopied(long copied) { this.copied += copied; } @@ -156,8 +155,7 @@ public final class MultipartUploadSlicingAlgorithm { return chunkSize; } - @VisibleForTesting - protected long getRemaining() { + public long getRemaining() { return remaining; }
