Repository: jclouds-labs Updated Branches: refs/heads/2.0.x 9ffceea3a -> 6567192d4
JCLOUDS-1267: Allow B2 streaming uploads B2 now supports uploads without calculating the SHA-1 hash. This allows uploading without a repeatable payload. Project: http://git-wip-us.apache.org/repos/asf/jclouds-labs/repo Commit: http://git-wip-us.apache.org/repos/asf/jclouds-labs/commit/6567192d Tree: http://git-wip-us.apache.org/repos/asf/jclouds-labs/tree/6567192d Diff: http://git-wip-us.apache.org/repos/asf/jclouds-labs/diff/6567192d Branch: refs/heads/2.0.x Commit: 6567192d4f5a9a78049cfadea7d3a9328804659e Parents: 9ffceea Author: Andrew Gaul <[email protected]> Authored: Fri Apr 7 17:28:35 2017 -0700 Committer: Andrew Gaul <[email protected]> Committed: Fri Apr 7 19:49:46 2017 -0700 ---------------------------------------------------------------------- .../jclouds/b2/binders/UploadFileBinder.java | 5 +++ .../jclouds/b2/binders/UploadPartBinder.java | 6 ++++ .../org/jclouds/b2/blobstore/B2BlobStore.java | 32 +++++--------------- .../org/jclouds/b2/features/MultipartApi.java | 2 +- .../java/org/jclouds/b2/features/ObjectApi.java | 3 +- .../integration/B2BlobIntegrationLiveTest.java | 20 ------------ 6 files changed, 20 insertions(+), 48 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/jclouds-labs/blob/6567192d/b2/src/main/java/org/jclouds/b2/binders/UploadFileBinder.java ---------------------------------------------------------------------- diff --git a/b2/src/main/java/org/jclouds/b2/binders/UploadFileBinder.java b/b2/src/main/java/org/jclouds/b2/binders/UploadFileBinder.java index 23ddf05..aa4ebc9 100644 --- a/b2/src/main/java/org/jclouds/b2/binders/UploadFileBinder.java +++ b/b2/src/main/java/org/jclouds/b2/binders/UploadFileBinder.java @@ -33,10 +33,15 @@ public final class UploadFileBinder implements MapBinder { public <R extends HttpRequest> R bindToRequest(R request, Map<String, Object> postParams) { UploadUrlResponse uploadUrl = (UploadUrlResponse) postParams.get("uploadUrl"); String fileName = (String) postParams.get("fileName"); + String contentSha1 = (String) postParams.get("contentSha1"); + if (contentSha1 == null) { + contentSha1 = "do_not_verify"; + } Map<String, String> fileInfo = (Map<String, String>) postParams.get("fileInfo"); HttpRequest.Builder builder = request.toBuilder() .endpoint(uploadUrl.uploadUrl()) .replaceHeader(HttpHeaders.AUTHORIZATION, uploadUrl.authorizationToken()) + .replaceHeader(B2Headers.CONTENT_SHA1, contentSha1) .replaceHeader(B2Headers.FILE_NAME, escaper.escape(fileName)); for (Map.Entry<String, String> entry : fileInfo.entrySet()) { builder.replaceHeader(B2Headers.FILE_INFO_PREFIX + entry.getKey(), escaper.escape(entry.getValue())); http://git-wip-us.apache.org/repos/asf/jclouds-labs/blob/6567192d/b2/src/main/java/org/jclouds/b2/binders/UploadPartBinder.java ---------------------------------------------------------------------- diff --git a/b2/src/main/java/org/jclouds/b2/binders/UploadPartBinder.java b/b2/src/main/java/org/jclouds/b2/binders/UploadPartBinder.java index 0275412..e89e2df 100644 --- a/b2/src/main/java/org/jclouds/b2/binders/UploadPartBinder.java +++ b/b2/src/main/java/org/jclouds/b2/binders/UploadPartBinder.java @@ -20,6 +20,7 @@ import java.util.Map; import org.jclouds.http.HttpRequest; import org.jclouds.b2.domain.GetUploadPartResponse; +import org.jclouds.b2.reference.B2Headers; import org.jclouds.rest.MapBinder; import com.google.common.net.HttpHeaders; @@ -28,9 +29,14 @@ public final class UploadPartBinder implements MapBinder { @Override public <R extends HttpRequest> R bindToRequest(R request, Map<String, Object> postParams) { GetUploadPartResponse uploadUrl = (GetUploadPartResponse) postParams.get("response"); + String contentSha1 = (String) postParams.get("contentSha1"); + if (contentSha1 == null) { + contentSha1 = "do_not_verify"; + } return (R) request.toBuilder() .endpoint(uploadUrl.uploadUrl()) .replaceHeader(HttpHeaders.AUTHORIZATION, uploadUrl.authorizationToken()) + .replaceHeader(B2Headers.CONTENT_SHA1, contentSha1) .build(); } http://git-wip-us.apache.org/repos/asf/jclouds-labs/blob/6567192d/b2/src/main/java/org/jclouds/b2/blobstore/B2BlobStore.java ---------------------------------------------------------------------- diff --git a/b2/src/main/java/org/jclouds/b2/blobstore/B2BlobStore.java b/b2/src/main/java/org/jclouds/b2/blobstore/B2BlobStore.java index 00953b6..aa40fc5 100644 --- a/b2/src/main/java/org/jclouds/b2/blobstore/B2BlobStore.java +++ b/b2/src/main/java/org/jclouds/b2/blobstore/B2BlobStore.java @@ -16,7 +16,6 @@ */ package org.jclouds.b2.blobstore; -import java.io.IOException; import java.net.URI; import java.util.Date; import java.util.List; @@ -67,7 +66,6 @@ import org.jclouds.blobstore.options.PutOptions; import org.jclouds.blobstore.util.BlobUtils; import org.jclouds.collect.Memoized; import org.jclouds.domain.Location; -import org.jclouds.io.ByteStreams2; import org.jclouds.io.ContentMetadata; import org.jclouds.io.ContentMetadataBuilder; import org.jclouds.io.MutableContentMetadata; @@ -78,14 +76,12 @@ import org.jclouds.io.payloads.BaseMutableContentMetadata; import com.google.common.base.Preconditions; import com.google.common.base.Strings; import com.google.common.base.Supplier; -import com.google.common.base.Throwables; import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Sets; -import com.google.common.hash.Hashing; import com.google.common.net.HttpHeaders; import com.google.common.util.concurrent.UncheckedExecutionException; @@ -254,8 +250,6 @@ public final class B2BlobStore extends BaseBlobStore { @Override public String putBlob(String container, Blob blob, PutOptions options) { - Preconditions.checkArgument(blob.getPayload().isRepeatable(), "B2 requires repeatable payload to calculate SHA1"); - if (options.getBlobAccess() != BlobAccess.PRIVATE) { throw new UnsupportedOperationException("B2 only supports private access blobs"); } @@ -263,13 +257,6 @@ public final class B2BlobStore extends BaseBlobStore { if (options.isMultipart()) { return putMultipartBlob(container, blob, options); } else { - String contentSha1; - try { - contentSha1 = ByteStreams2.hashAndClose(blob.getPayload().openStream(), Hashing.sha1()).toString(); - } catch (IOException ioe) { - throw Throwables.propagate(ioe); - } - String name = blob.getMetadata().getName(); // B2 versions all files so we store the original fileId to delete it after the upload succeeds @@ -277,7 +264,7 @@ public final class B2BlobStore extends BaseBlobStore { Bucket bucket = getBucket(container); UploadUrlResponse uploadUrl = api.getObjectApi().getUploadUrl(bucket.bucketId()); - UploadFileResponse uploadFile = api.getObjectApi().uploadFile(uploadUrl, name, contentSha1, blob.getMetadata().getUserMetadata(), blob.getPayload()); + UploadFileResponse uploadFile = api.getObjectApi().uploadFile(uploadUrl, name, null, blob.getMetadata().getUserMetadata(), blob.getPayload()); if (oldFileId != null) { api.getObjectApi().deleteFileVersion(name, oldFileId); @@ -397,20 +384,15 @@ public final class B2BlobStore extends BaseBlobStore { @Override public MultipartPart uploadMultipartPart(MultipartUpload mpu, int partNumber, Payload payload) { - Preconditions.checkArgument(payload.isRepeatable()); - - String contentSha1; - try { - contentSha1 = ByteStreams2.hashAndClose(payload.openStream(), Hashing.sha1()).toString(); - } catch (IOException ioe) { - throw Throwables.propagate(ioe); - } - GetUploadPartResponse getUploadPart = api.getMultipartApi().getUploadPartUrl(mpu.id()); - UploadPartResponse uploadPart = api.getMultipartApi().uploadPart(getUploadPart, partNumber, contentSha1, payload); + UploadPartResponse uploadPart = api.getMultipartApi().uploadPart(getUploadPart, partNumber, null, payload); Date lastModified = null; // B2 does not return Last-Modified - return MultipartPart.create(uploadPart.partNumber(), uploadPart.contentLength(), uploadPart.contentSha1(), lastModified); + String contentSha1 = uploadPart.contentSha1(); + if (contentSha1.startsWith("unverified:")) { + contentSha1 = contentSha1.substring("unverified:".length()); + } + return MultipartPart.create(uploadPart.partNumber(), uploadPart.contentLength(), contentSha1, lastModified); } @Override http://git-wip-us.apache.org/repos/asf/jclouds-labs/blob/6567192d/b2/src/main/java/org/jclouds/b2/features/MultipartApi.java ---------------------------------------------------------------------- diff --git a/b2/src/main/java/org/jclouds/b2/features/MultipartApi.java b/b2/src/main/java/org/jclouds/b2/features/MultipartApi.java index 5affc4e..510fd0f 100644 --- a/b2/src/main/java/org/jclouds/b2/features/MultipartApi.java +++ b/b2/src/main/java/org/jclouds/b2/features/MultipartApi.java @@ -84,7 +84,7 @@ public interface MultipartApi { @Named("b2_upload_part") @POST @MapBinder(UploadPartBinder.class) - UploadPartResponse uploadPart(@PayloadParam("response") GetUploadPartResponse response, @HeaderParam("X-Bz-Part-Number") int partNumber, @HeaderParam("X-Bz-Content-Sha1") String sha1, @PayloadParam("payload") Payload payload); + UploadPartResponse uploadPart(@PayloadParam("response") GetUploadPartResponse response, @HeaderParam("X-Bz-Part-Number") int partNumber, @Nullable @PayloadParam("contentSha1") String sha1, @PayloadParam("payload") Payload payload); @Named("b2_list_parts") @POST http://git-wip-us.apache.org/repos/asf/jclouds-labs/blob/6567192d/b2/src/main/java/org/jclouds/b2/features/ObjectApi.java ---------------------------------------------------------------------- diff --git a/b2/src/main/java/org/jclouds/b2/features/ObjectApi.java b/b2/src/main/java/org/jclouds/b2/features/ObjectApi.java index 4864864..c8379eb 100644 --- a/b2/src/main/java/org/jclouds/b2/features/ObjectApi.java +++ b/b2/src/main/java/org/jclouds/b2/features/ObjectApi.java @@ -24,7 +24,6 @@ import java.util.Map; import javax.inject.Named; import javax.ws.rs.Consumes; import javax.ws.rs.GET; -import javax.ws.rs.HeaderParam; import javax.ws.rs.POST; import javax.ws.rs.Path; import javax.ws.rs.PathParam; @@ -68,7 +67,7 @@ public interface ObjectApi { @POST @MapBinder(UploadFileBinder.class) @Consumes(APPLICATION_JSON) - UploadFileResponse uploadFile(@PayloadParam("uploadUrl") UploadUrlResponse uploadUrl, @PayloadParam("fileName") String fileName, @HeaderParam("X-Bz-Content-Sha1") String contentSha1, @PayloadParam("fileInfo") Map<String, String> fileInfo, Payload payload); + UploadFileResponse uploadFile(@PayloadParam("uploadUrl") UploadUrlResponse uploadUrl, @PayloadParam("fileName") String fileName, @Nullable @PayloadParam("contentSha1") String contentSha1, @PayloadParam("fileInfo") Map<String, String> fileInfo, Payload payload); @Named("b2_delete_file_version") @POST http://git-wip-us.apache.org/repos/asf/jclouds-labs/blob/6567192d/b2/src/test/java/org/jclouds/b2/blobstore/integration/B2BlobIntegrationLiveTest.java ---------------------------------------------------------------------- diff --git a/b2/src/test/java/org/jclouds/b2/blobstore/integration/B2BlobIntegrationLiveTest.java b/b2/src/test/java/org/jclouds/b2/blobstore/integration/B2BlobIntegrationLiveTest.java index ae2326e..846b25b 100644 --- a/b2/src/test/java/org/jclouds/b2/blobstore/integration/B2BlobIntegrationLiveTest.java +++ b/b2/src/test/java/org/jclouds/b2/blobstore/integration/B2BlobIntegrationLiveTest.java @@ -123,26 +123,6 @@ public final class B2BlobIntegrationLiveTest extends BaseBlobIntegrationTest { } @Override - public void testPutInputStream() throws Exception { - try { - super.testPutInputStream(); - failBecauseExceptionWasNotThrown(IllegalArgumentException.class); - } catch (IllegalArgumentException iae) { - throw new SkipException("B2 requires repeatable payloads to calculate SHA1 hash", iae); - } - } - - @Override - public void testPutMultipartInputStream() throws Exception { - try { - super.testPutMultipartInputStream(); - failBecauseExceptionWasNotThrown(IllegalArgumentException.class); - } catch (IllegalArgumentException iae) { - throw new SkipException("B2 requires repeatable payloads to calculate SHA1 hash", iae); - } - } - - @Override public void testPutObjectStream() throws InterruptedException, IOException, ExecutionException { try { super.testPutObjectStream();
