Repository: jclouds Updated Branches: refs/heads/master 1d218b170 -> f4eca0422
Enforce correct MD5 for local blobstores Matches behavior of real blobstores. Project: http://git-wip-us.apache.org/repos/asf/jclouds/repo Commit: http://git-wip-us.apache.org/repos/asf/jclouds/commit/f4eca042 Tree: http://git-wip-us.apache.org/repos/asf/jclouds/tree/f4eca042 Diff: http://git-wip-us.apache.org/repos/asf/jclouds/diff/f4eca042 Branch: refs/heads/master Commit: f4eca0422db963a619a0215affec2cc905c78dc1 Parents: 1d218b1 Author: Andrew Gaul <[email protected]> Authored: Tue Oct 22 16:41:31 2013 -0700 Committer: Andrew Gaul <[email protected]> Committed: Thu Jul 10 22:19:42 2014 -0700 ---------------------------------------------------------------------- .../internal/FilesystemStorageStrategyImpl.java | 13 ++++- .../FilesystemBlobIntegrationTest.java | 5 -- .../jclouds/blobstore/LocalAsyncBlobStore.java | 5 ++ .../blobstore/TransientStorageStrategy.java | 59 ++++++++++---------- .../TransientBlobIntegrationTest.java | 5 -- 5 files changed, 45 insertions(+), 42 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/jclouds/blob/f4eca042/apis/filesystem/src/main/java/org/jclouds/filesystem/strategy/internal/FilesystemStorageStrategyImpl.java ---------------------------------------------------------------------- diff --git a/apis/filesystem/src/main/java/org/jclouds/filesystem/strategy/internal/FilesystemStorageStrategyImpl.java b/apis/filesystem/src/main/java/org/jclouds/filesystem/strategy/internal/FilesystemStorageStrategyImpl.java index bd013cb..b526fbf 100644 --- a/apis/filesystem/src/main/java/org/jclouds/filesystem/strategy/internal/FilesystemStorageStrategyImpl.java +++ b/apis/filesystem/src/main/java/org/jclouds/filesystem/strategy/internal/FilesystemStorageStrategyImpl.java @@ -47,6 +47,7 @@ import com.google.common.base.Throwables; import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; import com.google.common.collect.Sets; +import com.google.common.hash.HashCode; import com.google.common.hash.Hashing; import com.google.common.hash.HashingInputStream; import com.google.common.io.ByteSource; @@ -201,9 +202,15 @@ public class FilesystemStorageStrategyImpl implements LocalStorageStrategy { Files.createParentDirs(outputFile); his = new HashingInputStream(Hashing.md5(), payload.openStream()); Files.asByteSink(outputFile).writeFrom(his); - payload.getContentMetadata().setContentMD5(his.hash()); - String eTag = base16().lowerCase().encode(payload.getContentMetadata().getContentMD5()); - return eTag; + HashCode actualHashCode = his.hash(); + HashCode expectedHashCode = payload.getContentMetadata().getContentMD5AsHashCode(); + if (expectedHashCode != null && !actualHashCode.equals(expectedHashCode)) { + throw new IOException("MD5 hash code mismatch, actual: " + actualHashCode + + " expected: " + expectedHashCode); + } + payload.getContentMetadata().setContentMD5(actualHashCode); + // TODO: store metadata in extended attributes when moving to Java 7 + return base16().lowerCase().encode(actualHashCode.asBytes()); } catch (IOException ex) { if (outputFile != null) { if (!outputFile.delete()) { http://git-wip-us.apache.org/repos/asf/jclouds/blob/f4eca042/apis/filesystem/src/test/java/org/jclouds/filesystem/integration/FilesystemBlobIntegrationTest.java ---------------------------------------------------------------------- diff --git a/apis/filesystem/src/test/java/org/jclouds/filesystem/integration/FilesystemBlobIntegrationTest.java b/apis/filesystem/src/test/java/org/jclouds/filesystem/integration/FilesystemBlobIntegrationTest.java index ca2face..bb20fc6 100644 --- a/apis/filesystem/src/test/java/org/jclouds/filesystem/integration/FilesystemBlobIntegrationTest.java +++ b/apis/filesystem/src/test/java/org/jclouds/filesystem/integration/FilesystemBlobIntegrationTest.java @@ -77,9 +77,4 @@ public class FilesystemBlobIntegrationTest extends BaseBlobIntegrationTest { public void testPutObjectStream() throws InterruptedException, IOException, ExecutionException { throw new SkipException("not yet implemented"); } - - @Override - public void testPutIncorrectContentMD5() throws InterruptedException, IOException { - throw new SkipException("not yet implemented"); - } } http://git-wip-us.apache.org/repos/asf/jclouds/blob/f4eca042/blobstore/src/main/java/org/jclouds/blobstore/LocalAsyncBlobStore.java ---------------------------------------------------------------------- diff --git a/blobstore/src/main/java/org/jclouds/blobstore/LocalAsyncBlobStore.java b/blobstore/src/main/java/org/jclouds/blobstore/LocalAsyncBlobStore.java index 7b57c62..ba987ab 100644 --- a/blobstore/src/main/java/org/jclouds/blobstore/LocalAsyncBlobStore.java +++ b/blobstore/src/main/java/org/jclouds/blobstore/LocalAsyncBlobStore.java @@ -392,6 +392,11 @@ public class LocalAsyncBlobStore extends BaseAsyncBlobStore { try { return immediateFuture(storageStrategy.putBlob(containerName, blob)); } catch (IOException e) { + if (e.getMessage().startsWith("MD5 hash code mismatch")) { + HttpResponseException exception = returnResponseException(400); + exception.initCause(e); + throw exception; + } logger.error(e, "An error occurred storing the new blob with name [%s] to container [%s].", blobKey, containerName); throw Throwables.propagate(e); http://git-wip-us.apache.org/repos/asf/jclouds/blob/f4eca042/blobstore/src/main/java/org/jclouds/blobstore/TransientStorageStrategy.java ---------------------------------------------------------------------- diff --git a/blobstore/src/main/java/org/jclouds/blobstore/TransientStorageStrategy.java b/blobstore/src/main/java/org/jclouds/blobstore/TransientStorageStrategy.java index 4871ddf..dfa4365 100644 --- a/blobstore/src/main/java/org/jclouds/blobstore/TransientStorageStrategy.java +++ b/blobstore/src/main/java/org/jclouds/blobstore/TransientStorageStrategy.java @@ -39,14 +39,15 @@ import org.jclouds.io.ContentMetadataCodec; import org.jclouds.io.MutableContentMetadata; import org.jclouds.io.Payload; import org.jclouds.io.Payloads; -import org.jclouds.io.payloads.ByteArrayPayload; -import org.jclouds.io.payloads.DelegatingPayload; +import org.jclouds.util.Closeables2; import com.google.common.base.Supplier; -import com.google.common.base.Throwables; import com.google.common.collect.ImmutableList; import com.google.common.collect.Multimaps; +import com.google.common.hash.HashCode; import com.google.common.hash.Hashing; +import com.google.common.hash.HashingInputStream; +import com.google.common.io.ByteSource; import com.google.common.io.ByteStreams; import com.google.common.net.HttpHeaders; @@ -123,10 +124,25 @@ public class TransientStorageStrategy implements LocalStorageStrategy { @Override public String putBlob(final String containerName, final Blob blob) throws IOException { - Blob newBlob = createUpdatedCopyOfBlobInContainer(containerName, blob); + byte[] payload; + HashCode actualHashCode; + HashingInputStream input = new HashingInputStream(Hashing.md5(), blob.getPayload().openStream()); + try { + payload = ByteStreams.toByteArray(input); + actualHashCode = input.hash(); + HashCode expectedHashCode = blob.getPayload().getContentMetadata().getContentMD5AsHashCode(); + if (expectedHashCode != null && !actualHashCode.equals(expectedHashCode)) { + throw new IOException("MD5 hash code mismatch, actual: " + actualHashCode + + " expected: " + expectedHashCode); + } + } finally { + Closeables2.closeQuietly(input); + } + + Blob newBlob = createUpdatedCopyOfBlobInContainer(containerName, blob, payload, actualHashCode); Map<String, Blob> map = containerToBlobs.get(containerName); map.put(newBlob.getMetadata().getName(), newBlob); - return base16().lowerCase().encode(newBlob.getPayload().getContentMetadata().getContentMD5()); + return base16().lowerCase().encode(actualHashCode.asBytes()); } @Override @@ -146,37 +162,22 @@ public class TransientStorageStrategy implements LocalStorageStrategy { return "/"; } - private Blob createUpdatedCopyOfBlobInContainer(String containerName, Blob in) { + private Blob createUpdatedCopyOfBlobInContainer(String containerName, Blob in, byte[] input, HashCode contentMd5) { + checkNotNull(containerName, "containerName"); checkNotNull(in, "blob"); - checkNotNull(in.getPayload(), "blob.payload"); - ByteArrayPayload payload = (in.getPayload() instanceof ByteArrayPayload) ? ByteArrayPayload.class.cast(in - .getPayload()) : null; - if (payload == null) - payload = (in.getPayload() instanceof DelegatingPayload) ? (DelegatingPayload.class.cast(in.getPayload()) - .getDelegate() instanceof ByteArrayPayload) ? ByteArrayPayload.class.cast(DelegatingPayload.class - .cast(in.getPayload()).getDelegate()) : null : null; - try { - if (payload == null || !(payload instanceof ByteArrayPayload)) { - MutableContentMetadata oldMd = in.getPayload().getContentMetadata(); - byte[] out = ByteStreams.toByteArray(in.getPayload()); - payload = Payloads.newByteArrayPayload(out); - HttpUtils.copy(oldMd, payload.getContentMetadata()); - payload.getContentMetadata().setContentMD5(Hashing.md5().hashBytes(out)); - } else { - if (payload.getContentMetadata().getContentMD5() == null) { - payload.getContentMetadata().setContentMD5(ByteStreams.hash(payload, Hashing.md5())); - } - } - } catch (IOException e) { - Throwables.propagate(e); - } + checkNotNull(input, "input"); + checkNotNull(contentMd5, "contentMd5"); + Payload payload = Payloads.newByteSourcePayload(ByteSource.wrap(input)); + MutableContentMetadata oldMd = in.getPayload().getContentMetadata(); + HttpUtils.copy(oldMd, payload.getContentMetadata()); + payload.getContentMetadata().setContentMD5(contentMd5); Blob blob = blobFactory.create(BlobStoreUtils.copy(in.getMetadata())); blob.setPayload(payload); blob.getMetadata().setContainer(containerName); blob.getMetadata().setUri( uriBuilder(new StringBuilder("mem://").append(containerName)).path(in.getMetadata().getName()).build()); blob.getMetadata().setLastModified(new Date()); - String eTag = base16().lowerCase().encode(payload.getContentMetadata().getContentMD5()); + String eTag = base16().lowerCase().encode(contentMd5.asBytes()); blob.getMetadata().setETag(eTag); // Set HTTP headers to match metadata blob.getAllHeaders().replaceValues(HttpHeaders.LAST_MODIFIED, http://git-wip-us.apache.org/repos/asf/jclouds/blob/f4eca042/blobstore/src/test/java/org/jclouds/blobstore/integration/TransientBlobIntegrationTest.java ---------------------------------------------------------------------- diff --git a/blobstore/src/test/java/org/jclouds/blobstore/integration/TransientBlobIntegrationTest.java b/blobstore/src/test/java/org/jclouds/blobstore/integration/TransientBlobIntegrationTest.java index efd5d34..4e2d496 100644 --- a/blobstore/src/test/java/org/jclouds/blobstore/integration/TransientBlobIntegrationTest.java +++ b/blobstore/src/test/java/org/jclouds/blobstore/integration/TransientBlobIntegrationTest.java @@ -27,9 +27,4 @@ public class TransientBlobIntegrationTest extends BaseBlobIntegrationTest { public TransientBlobIntegrationTest() { provider = "transient"; } - - @Override - public void testPutIncorrectContentMD5() throws InterruptedException, IOException { - throw new SkipException("not yet implemented"); - } }
