Updated Branches: refs/heads/master d05e77b8b -> d60d2681d
JCLOUDS-251: Swift: Delete chunks when deleting a multipart blob Also: - Make SwiftBlobIntegrationLiveTest.testMultipartChunkedFileStream more realistic by uploading a file large enough to be split into parts. - JavaDoc fixes for SwiftBlobStore: don't reference nonexistent methods. Project: http://git-wip-us.apache.org/repos/asf/incubator-jclouds/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-jclouds/commit/d60d2681 Tree: http://git-wip-us.apache.org/repos/asf/incubator-jclouds/tree/d60d2681 Diff: http://git-wip-us.apache.org/repos/asf/incubator-jclouds/diff/d60d2681 Branch: refs/heads/master Commit: d60d2681d1d7d7a38bbb65d5408672ee71b15727 Parents: d05e77b Author: Francis Devereux <[email protected]> Authored: Thu Aug 22 15:27:25 2013 +0100 Committer: Everett Toews <[email protected]> Committed: Sun Aug 25 10:20:22 2013 -0500 ---------------------------------------------------------------------- .../CloudFilesBlobIntegrationLiveTest.java | 23 +++++++ .../swift/blobstore/SwiftBlobStore.java | 56 ++++++++++++++- .../domain/MutableObjectInfoWithMetadata.java | 3 + ...DelegatingMutableObjectInfoWithMetadata.java | 10 +++ .../MutableObjectInfoWithMetadataImpl.java | 21 +++++- .../functions/ParseObjectInfoFromHeaders.java | 2 + .../swift/blobstore/SwiftBlobStoreTest.java | 45 +++++++++++++ .../SwiftBlobIntegrationLiveTest.java | 71 ++++++++++++++++---- .../internal/BaseBlobIntegrationTest.java | 7 ++ 9 files changed, 220 insertions(+), 18 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-jclouds/blob/d60d2681/apis/cloudfiles/src/test/java/org/jclouds/cloudfiles/blobstore/integration/CloudFilesBlobIntegrationLiveTest.java ---------------------------------------------------------------------- diff --git a/apis/cloudfiles/src/test/java/org/jclouds/cloudfiles/blobstore/integration/CloudFilesBlobIntegrationLiveTest.java b/apis/cloudfiles/src/test/java/org/jclouds/cloudfiles/blobstore/integration/CloudFilesBlobIntegrationLiveTest.java index a56e9f8..3d7f42b 100644 --- a/apis/cloudfiles/src/test/java/org/jclouds/cloudfiles/blobstore/integration/CloudFilesBlobIntegrationLiveTest.java +++ b/apis/cloudfiles/src/test/java/org/jclouds/cloudfiles/blobstore/integration/CloudFilesBlobIntegrationLiveTest.java @@ -16,10 +16,15 @@ */ package org.jclouds.cloudfiles.blobstore.integration; +import java.io.IOException; + +import org.jclouds.blobstore.BlobStore; import org.jclouds.blobstore.domain.Blob; import org.jclouds.openstack.swift.blobstore.integration.SwiftBlobIntegrationLiveTest; import org.testng.annotations.Test; +import static org.testng.Assert.assertEquals; + /** * * @author Adrian Cole @@ -38,4 +43,22 @@ public class CloudFilesBlobIntegrationLiveTest extends SwiftBlobIntegrationLiveT .getMetadata().getContentMetadata().getContentDisposition(); } + @Test(groups = { "integration", "live" }) + public void testChunksAreDeletedWhenMultipartBlobIsDeleted() throws IOException, InterruptedException { + String containerName = getContainerName(); + try { + BlobStore blobStore = view.getBlobStore(); + + long countBefore = blobStore.countBlobs(containerName); + String blobName = "deleteme.txt"; + addMultipartBlobToContainer(containerName, blobName); + + blobStore.removeBlob(containerName, blobName); + long countAfter = blobStore.countBlobs(containerName); + + assertEquals(countAfter, countBefore); + } finally { + returnContainer(containerName); + } + } } http://git-wip-us.apache.org/repos/asf/incubator-jclouds/blob/d60d2681/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/SwiftBlobStore.java ---------------------------------------------------------------------- diff --git a/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/SwiftBlobStore.java b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/SwiftBlobStore.java index 8c0ac70..2940d6f 100644 --- a/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/SwiftBlobStore.java +++ b/apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/SwiftBlobStore.java @@ -16,8 +16,10 @@ */ package org.jclouds.openstack.swift.blobstore; +import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; import static org.jclouds.blobstore.util.BlobStoreUtils.createParentIfNeededAsync; +import static org.jclouds.openstack.swift.options.ListContainerOptions.Builder.withPrefix; import java.util.Set; @@ -25,6 +27,7 @@ import javax.inject.Inject; import javax.inject.Provider; import javax.inject.Singleton; +import com.google.common.annotations.VisibleForTesting; import org.jclouds.blobstore.BlobStoreContext; import org.jclouds.blobstore.domain.Blob; import org.jclouds.blobstore.domain.BlobMetadata; @@ -50,8 +53,11 @@ import org.jclouds.openstack.swift.blobstore.functions.ObjectToBlob; import org.jclouds.openstack.swift.blobstore.functions.ObjectToBlobMetadata; import org.jclouds.openstack.swift.blobstore.strategy.internal.MultipartUploadStrategy; import org.jclouds.openstack.swift.domain.ContainerMetadata; +import org.jclouds.openstack.swift.domain.MutableObjectInfoWithMetadata; +import org.jclouds.openstack.swift.domain.ObjectInfo; import com.google.common.base.Function; +import com.google.common.base.Strings; import com.google.common.base.Supplier; import com.google.common.collect.Iterables; @@ -118,7 +124,7 @@ public class SwiftBlobStore extends BaseBlobStore { } /** - * This implementation invokes {@link CommonSwiftClient#putBucketInRegion} + * This implementation invokes {@link CommonSwiftClient#createContainer} * * @param location * currently ignored @@ -145,7 +151,7 @@ public class SwiftBlobStore extends BaseBlobStore { } /** - * This implementation invokes {@link CommonSwiftClient#blobExists} + * This implementation invokes {@link CommonSwiftClient#objectExists} * * @param container * container name @@ -225,7 +231,53 @@ public class SwiftBlobStore extends BaseBlobStore { */ @Override public void removeBlob(String container, String key) { + String objectManifest = getObjectManifestOrNull(container, key); + sync.removeObject(container, key); + + if (!Strings.isNullOrEmpty(objectManifest)) { + removeObjectsWithPrefix(objectManifest); + } + } + + private String getObjectManifestOrNull(String container, String key) { + MutableObjectInfoWithMetadata objectInfo = sync.getObjectInfo(container, key); + return objectInfo == null ? null : objectInfo.getObjectManifest(); + } + + private void removeObjectsWithPrefix(String containerAndPrefix) { + String[] parts = splitContainerAndKey(containerAndPrefix); + + String container = parts[0]; + String prefix = parts[1]; + + removeObjectsWithPrefix(container, prefix); + } + + @VisibleForTesting + static String[] splitContainerAndKey(String containerAndKey) { + String[] parts = containerAndKey.split("/", 2); + checkArgument(parts.length == 2, + "No / separator found in \"%s\"", + containerAndKey); + return parts; + } + + private void removeObjectsWithPrefix(String container, String prefix) { + String nextMarker = null; + do { + org.jclouds.openstack.swift.options.ListContainerOptions listContainerOptions = + withPrefix(prefix); + if (nextMarker != null) { + listContainerOptions = listContainerOptions.afterMarker(nextMarker); + } + + PageSet<ObjectInfo> chunks = sync.listObjects(container, listContainerOptions); + for (ObjectInfo chunk : chunks) { + sync.removeObject(container, chunk.getName()); + } + nextMarker = chunks.getNextMarker(); + } while (nextMarker != null); } @Override http://git-wip-us.apache.org/repos/asf/incubator-jclouds/blob/d60d2681/apis/swift/src/main/java/org/jclouds/openstack/swift/domain/MutableObjectInfoWithMetadata.java ---------------------------------------------------------------------- diff --git a/apis/swift/src/main/java/org/jclouds/openstack/swift/domain/MutableObjectInfoWithMetadata.java b/apis/swift/src/main/java/org/jclouds/openstack/swift/domain/MutableObjectInfoWithMetadata.java index ee326a4..103292f 100644 --- a/apis/swift/src/main/java/org/jclouds/openstack/swift/domain/MutableObjectInfoWithMetadata.java +++ b/apis/swift/src/main/java/org/jclouds/openstack/swift/domain/MutableObjectInfoWithMetadata.java @@ -48,4 +48,7 @@ public interface MutableObjectInfoWithMetadata extends ObjectInfo { Map<String, String> getMetadata(); + String getObjectManifest(); + + void setObjectManifest(String objectManifest); } http://git-wip-us.apache.org/repos/asf/incubator-jclouds/blob/d60d2681/apis/swift/src/main/java/org/jclouds/openstack/swift/domain/internal/DelegatingMutableObjectInfoWithMetadata.java ---------------------------------------------------------------------- diff --git a/apis/swift/src/main/java/org/jclouds/openstack/swift/domain/internal/DelegatingMutableObjectInfoWithMetadata.java b/apis/swift/src/main/java/org/jclouds/openstack/swift/domain/internal/DelegatingMutableObjectInfoWithMetadata.java index 57d3444..9b57686 100644 --- a/apis/swift/src/main/java/org/jclouds/openstack/swift/domain/internal/DelegatingMutableObjectInfoWithMetadata.java +++ b/apis/swift/src/main/java/org/jclouds/openstack/swift/domain/internal/DelegatingMutableObjectInfoWithMetadata.java @@ -147,4 +147,14 @@ public class DelegatingMutableObjectInfoWithMetadata extends BaseMutableContentM public URI getUri() { return delegate.getUri(); } + + @Override + public void setObjectManifest(String objectManifest) { + delegate.setObjectManifest(objectManifest); + } + + @Override + public String getObjectManifest() { + return delegate.getObjectManifest(); + } } http://git-wip-us.apache.org/repos/asf/incubator-jclouds/blob/d60d2681/apis/swift/src/main/java/org/jclouds/openstack/swift/domain/internal/MutableObjectInfoWithMetadataImpl.java ---------------------------------------------------------------------- diff --git a/apis/swift/src/main/java/org/jclouds/openstack/swift/domain/internal/MutableObjectInfoWithMetadataImpl.java b/apis/swift/src/main/java/org/jclouds/openstack/swift/domain/internal/MutableObjectInfoWithMetadataImpl.java index dc09819..484888d 100644 --- a/apis/swift/src/main/java/org/jclouds/openstack/swift/domain/internal/MutableObjectInfoWithMetadataImpl.java +++ b/apis/swift/src/main/java/org/jclouds/openstack/swift/domain/internal/MutableObjectInfoWithMetadataImpl.java @@ -41,6 +41,7 @@ public class MutableObjectInfoWithMetadataImpl implements MutableObjectInfoWithM private byte[] hash; private String contentType = MediaType.APPLICATION_OCTET_STREAM; private Date lastModified; + private String objectManifest; private final Map<String, String> metadata = Maps.newLinkedHashMap(); /** @@ -121,6 +122,7 @@ public class MutableObjectInfoWithMetadataImpl implements MutableObjectInfoWithM int result = 1; result = prime * result + ((container == null) ? 0 : container.hashCode()); result = prime * result + ((name == null) ? 0 : name.hashCode()); + result = prime * result + ((objectManifest == null) ? 0 : objectManifest.hashCode()); return result; } @@ -143,6 +145,11 @@ public class MutableObjectInfoWithMetadataImpl implements MutableObjectInfoWithM return false; } else if (!name.equals(other.name)) return false; + if (objectManifest == null) { + if (other.objectManifest != null) + return false; + } else if (!objectManifest.equals(other.objectManifest)) + return false; return true; } @@ -197,9 +204,19 @@ public class MutableObjectInfoWithMetadataImpl implements MutableObjectInfoWithM } @Override + public String getObjectManifest() { + return objectManifest; + } + + @Override + public void setObjectManifest(String objectManifest) { + this.objectManifest = objectManifest; + } + + @Override public String toString() { - return String.format("[name=%s, container=%s, uri=%s, bytes=%s, contentType=%s, lastModified=%s, hash=%s]", name, - container, uri, bytes, contentType, lastModified, Arrays.toString(hash)); + return String.format("[name=%s, container=%s, uri=%s, bytes=%s, contentType=%s, lastModified=%s, hash=%s, objectManifest=%s]", + name, container, uri, bytes, contentType, lastModified, Arrays.toString(hash), objectManifest); } } http://git-wip-us.apache.org/repos/asf/incubator-jclouds/blob/d60d2681/apis/swift/src/main/java/org/jclouds/openstack/swift/functions/ParseObjectInfoFromHeaders.java ---------------------------------------------------------------------- diff --git a/apis/swift/src/main/java/org/jclouds/openstack/swift/functions/ParseObjectInfoFromHeaders.java b/apis/swift/src/main/java/org/jclouds/openstack/swift/functions/ParseObjectInfoFromHeaders.java index bd082fe..d9f0297 100644 --- a/apis/swift/src/main/java/org/jclouds/openstack/swift/functions/ParseObjectInfoFromHeaders.java +++ b/apis/swift/src/main/java/org/jclouds/openstack/swift/functions/ParseObjectInfoFromHeaders.java @@ -65,6 +65,8 @@ public class ParseObjectInfoFromHeaders implements Function<HttpResponse, Mutabl if (eTagHeader != null) { to.setHash(ETagUtils.convertHexETagToByteArray(eTagHeader)); } + to.setObjectManifest(from.getFirstHeaderOrNull("X-Object-Manifest")); + return to; } http://git-wip-us.apache.org/repos/asf/incubator-jclouds/blob/d60d2681/apis/swift/src/test/java/org/jclouds/openstack/swift/blobstore/SwiftBlobStoreTest.java ---------------------------------------------------------------------- diff --git a/apis/swift/src/test/java/org/jclouds/openstack/swift/blobstore/SwiftBlobStoreTest.java b/apis/swift/src/test/java/org/jclouds/openstack/swift/blobstore/SwiftBlobStoreTest.java new file mode 100644 index 0000000..439d7e0 --- /dev/null +++ b/apis/swift/src/test/java/org/jclouds/openstack/swift/blobstore/SwiftBlobStoreTest.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jclouds.openstack.swift.blobstore; + +import org.testng.annotations.Test; + +import static org.testng.Assert.assertEquals; + +@Test(groups = "unit") +public class SwiftBlobStoreTest { + @Test + public void testSplitContainerAndKey() { + String container = "test-container"; + String key = "key/with/some/slashes/in/it/and/a/trailing/slash/"; + + String containerAndKey = container + "/" + key; + + String[] split = SwiftBlobStore.splitContainerAndKey(containerAndKey); + String actualContainer = split[0]; + String actualKey = split[1]; + + assertEquals(actualContainer, container); + assertEquals(actualKey, key); + } + + @Test(expectedExceptions = IllegalArgumentException.class, + expectedExceptionsMessageRegExp = "No / separator found in \"not-a-container-and-key\"") + public void testSplitContainerAndKeyWithNoSeparator() { + SwiftBlobStore.splitContainerAndKey("not-a-container-and-key"); + } +} http://git-wip-us.apache.org/repos/asf/incubator-jclouds/blob/d60d2681/apis/swift/src/test/java/org/jclouds/openstack/swift/blobstore/integration/SwiftBlobIntegrationLiveTest.java ---------------------------------------------------------------------- diff --git a/apis/swift/src/test/java/org/jclouds/openstack/swift/blobstore/integration/SwiftBlobIntegrationLiveTest.java b/apis/swift/src/test/java/org/jclouds/openstack/swift/blobstore/integration/SwiftBlobIntegrationLiveTest.java index 57e381a..365dc57 100644 --- a/apis/swift/src/test/java/org/jclouds/openstack/swift/blobstore/integration/SwiftBlobIntegrationLiveTest.java +++ b/apis/swift/src/test/java/org/jclouds/openstack/swift/blobstore/integration/SwiftBlobIntegrationLiveTest.java @@ -21,11 +21,13 @@ import java.io.IOException; import java.io.InputStream; import java.util.Properties; +import com.google.common.io.ByteStreams; import org.jclouds.blobstore.BlobStore; import org.jclouds.blobstore.domain.Blob; import org.jclouds.blobstore.integration.internal.BaseBlobIntegrationTest; import org.jclouds.blobstore.options.PutOptions; import org.jclouds.openstack.keystone.v2_0.config.KeystoneProperties; +import org.jclouds.openstack.swift.blobstore.strategy.MultipartUpload; import org.testng.ITestContext; import org.testng.annotations.BeforeClass; import org.testng.annotations.DataProvider; @@ -34,6 +36,9 @@ import org.testng.annotations.Test; import com.google.common.io.Files; import com.google.common.io.InputSupplier; +import static org.testng.Assert.assertNotEquals; +import static org.testng.Assert.assertTrue; + /** * * @author James Murty @@ -41,15 +46,21 @@ import com.google.common.io.InputSupplier; */ @Test(groups = "live") public class SwiftBlobIntegrationLiveTest extends BaseBlobIntegrationTest { + /** + * Use the minimum part size to minimise the file size that we have to + * upload to get a multipart blob thereby make the test run faster + */ + private static final long PART_SIZE = MultipartUpload.MIN_PART_SIZE; + @Override protected Properties setupProperties() { Properties props = super.setupProperties(); setIfTestSystemPropertyPresent(props, KeystoneProperties.CREDENTIAL_TYPE); + props.setProperty("jclouds.mpu.parts.size", String.valueOf(PART_SIZE)); return props; } private InputSupplier<InputStream> oneHundredOneConstitutions; - private byte[] oneHundredOneConstitutionsMD5; public SwiftBlobIntegrationLiveTest() { provider = System.getProperty("test.swift.provider", "swift"); @@ -66,7 +77,6 @@ public class SwiftBlobIntegrationLiveTest extends BaseBlobIntegrationTest { public void setUpResourcesOnThisThread(ITestContext testContext) throws Exception { super.setUpResourcesOnThisThread(testContext); oneHundredOneConstitutions = getTestDataSupplier(); - oneHundredOneConstitutionsMD5 = md5Supplier(oneHundredOneConstitutions); } @Override @@ -91,18 +101,51 @@ public class SwiftBlobIntegrationLiveTest extends BaseBlobIntegrationTest { { "asteri*k" }, { "{great<r}" }, { "lesst>en" }, { "p|pe" } }; } + @Test(groups = { "integration", "live" }) public void testMultipartChunkedFileStream() throws IOException, InterruptedException { - Files.copy(oneHundredOneConstitutions, new File("target/const.txt")); - String containerName = getContainerName(); - - try { - BlobStore blobStore = view.getBlobStore(); - blobStore.createContainerInLocation(null, containerName); - Blob blob = blobStore.blobBuilder("const.txt") - .payload(new File("target/const.txt")).contentMD5(oneHundredOneConstitutionsMD5).build(); - blobStore.putBlob(containerName, blob, PutOptions.Builder.multipart()); - } finally { - returnContainer(containerName); - } + String containerName = getContainerName(); + try { + BlobStore blobStore = view.getBlobStore(); + long countBefore = blobStore.countBlobs(containerName); + + addMultipartBlobToContainer(containerName, "const.txt"); + + long countAfter = blobStore.countBlobs(containerName); + assertNotEquals(countBefore, countAfter, + "No blob was created"); + assertTrue(countAfter - countBefore > 1, + "A multipart blob wasn't actually created - " + + "there was only 1 extra blob but there should be one manifest blob and multiple chunk blobs"); + } finally { + returnContainer(containerName); + } + } + + protected void addMultipartBlobToContainer(String containerName, String key) throws IOException { + File fileToUpload = createFileBiggerThan(PART_SIZE); + + BlobStore blobStore = view.getBlobStore(); + blobStore.createContainerInLocation(null, containerName); + Blob blob = blobStore.blobBuilder(key) + .payload(fileToUpload) + .build(); + blobStore.putBlob(containerName, blob, PutOptions.Builder.multipart()); + } + + @SuppressWarnings("unchecked") + private File createFileBiggerThan(long partSize) throws IOException { + long copiesNeeded = (partSize / getOneHundredOneConstitutionsLength()) + 1; + + InputSupplier<InputStream> temp = ByteStreams.join(oneHundredOneConstitutions); + + for (int i = 0; i < copiesNeeded; i++) { + temp = ByteStreams.join(temp, oneHundredOneConstitutions); + } + + File fileToUpload = new File("target/lots-of-const.txt"); + Files.copy(temp, fileToUpload); + + assertTrue(fileToUpload.length() > partSize); + return fileToUpload; } } http://git-wip-us.apache.org/repos/asf/incubator-jclouds/blob/d60d2681/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseBlobIntegrationTest.java ---------------------------------------------------------------------- diff --git a/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseBlobIntegrationTest.java b/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseBlobIntegrationTest.java index fceeb85..19ec3f5 100644 --- a/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseBlobIntegrationTest.java +++ b/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseBlobIntegrationTest.java @@ -119,6 +119,13 @@ public class BaseBlobIntegrationTest extends BaseBlobStoreIntegrationTest { return temp; } + public static long getOneHundredOneConstitutionsLength() throws IOException { + if (oneHundredOneConstitutionsLength == 0) { + getTestDataSupplier(); + } + return oneHundredOneConstitutionsLength; + } + /** * Attempt to capture the issue detailed in * http://groups.google.com/group/jclouds/browse_thread/thread/4a7c8d58530b287f
