JCLOUDS-1337: S3 putBlob portable storage tiers Also promote hacky and limited storage class support from aws-s3 provider to s3 api.
Project: http://git-wip-us.apache.org/repos/asf/jclouds/repo Commit: http://git-wip-us.apache.org/repos/asf/jclouds/commit/89053d9a Tree: http://git-wip-us.apache.org/repos/asf/jclouds/tree/89053d9a Diff: http://git-wip-us.apache.org/repos/asf/jclouds/diff/89053d9a Branch: refs/heads/master Commit: 89053d9a8b43ccc54c99e552eb5a868b236d1057 Parents: 14c41ea Author: Andrew Gaul <[email protected]> Authored: Tue Oct 10 18:53:52 2017 -0700 Committer: Andrew Gaul <[email protected]> Committed: Thu Oct 12 11:16:48 2017 -0700 ---------------------------------------------------------------------- .../s3/binders/BindObjectMetadataToRequest.java | 5 ++++ .../binders/BindS3ObjectMetadataToRequest.java | 10 +++++++- .../org/jclouds/s3/blobstore/S3BlobStore.java | 2 ++ .../functions/BlobToObjectMetadata.java | 2 ++ .../functions/ObjectToBlobMetadata.java | 1 + .../s3/domain/MutableObjectMetadata.java | 1 + .../org/jclouds/s3/domain/ObjectMetadata.java | 27 +++++++++++++++++++- .../ParseObjectMetadataFromHeaders.java | 5 ++++ .../integration/S3BlobIntegrationLiveTest.java | 23 +++++++++++++++++ .../aws/s3/blobstore/AWSS3BlobStore.java | 26 ------------------- .../options/AWSS3PutObjectOptions.java | 3 +++ 11 files changed, 77 insertions(+), 28 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/jclouds/blob/89053d9a/apis/s3/src/main/java/org/jclouds/s3/binders/BindObjectMetadataToRequest.java ---------------------------------------------------------------------- diff --git a/apis/s3/src/main/java/org/jclouds/s3/binders/BindObjectMetadataToRequest.java b/apis/s3/src/main/java/org/jclouds/s3/binders/BindObjectMetadataToRequest.java index 84948e1..836f20b 100644 --- a/apis/s3/src/main/java/org/jclouds/s3/binders/BindObjectMetadataToRequest.java +++ b/apis/s3/src/main/java/org/jclouds/s3/binders/BindObjectMetadataToRequest.java @@ -81,6 +81,11 @@ public class BindObjectMetadataToRequest implements Binder { headers.put("Content-MD5", base64().encode(md.getContentMetadata().getContentMD5())); } + ObjectMetadata.StorageClass storageClass = md.getStorageClass(); + if (storageClass != ObjectMetadata.StorageClass.STANDARD) { + headers.put("x-amz-storage-class", storageClass.toString()); + } + return (R) request.toBuilder().replaceHeaders(headers.build()).build(); } } http://git-wip-us.apache.org/repos/asf/jclouds/blob/89053d9a/apis/s3/src/main/java/org/jclouds/s3/binders/BindS3ObjectMetadataToRequest.java ---------------------------------------------------------------------- diff --git a/apis/s3/src/main/java/org/jclouds/s3/binders/BindS3ObjectMetadataToRequest.java b/apis/s3/src/main/java/org/jclouds/s3/binders/BindS3ObjectMetadataToRequest.java index c50405c..e611bd9 100644 --- a/apis/s3/src/main/java/org/jclouds/s3/binders/BindS3ObjectMetadataToRequest.java +++ b/apis/s3/src/main/java/org/jclouds/s3/binders/BindS3ObjectMetadataToRequest.java @@ -25,6 +25,7 @@ import javax.inject.Singleton; import org.jclouds.blobstore.binders.BindMapToHeadersWithPrefix; import org.jclouds.http.HttpRequest; import org.jclouds.rest.Binder; +import org.jclouds.s3.domain.ObjectMetadata.StorageClass; import org.jclouds.s3.domain.S3Object; @Singleton @@ -48,7 +49,14 @@ public class BindS3ObjectMetadataToRequest implements Binder { "contentLength must be set, streaming not supported"); checkArgument(s3Object.getPayload().getContentMetadata().getContentLength() <= 5L * 1024 * 1024 * 1024, "maximum size for put object is 5GB"); - + + StorageClass storageClass = s3Object.getMetadata().getStorageClass(); + if (storageClass != StorageClass.STANDARD) { + request = (R) request.toBuilder() + .replaceHeader("x-amz-storage-class", storageClass.toString()) + .build(); + } + request = metadataPrefixer.bindToRequest(request, s3Object.getMetadata().getUserMetadata()); return request; http://git-wip-us.apache.org/repos/asf/jclouds/blob/89053d9a/apis/s3/src/main/java/org/jclouds/s3/blobstore/S3BlobStore.java ---------------------------------------------------------------------- diff --git a/apis/s3/src/main/java/org/jclouds/s3/blobstore/S3BlobStore.java b/apis/s3/src/main/java/org/jclouds/s3/blobstore/S3BlobStore.java index 195b637..9b91048 100644 --- a/apis/s3/src/main/java/org/jclouds/s3/blobstore/S3BlobStore.java +++ b/apis/s3/src/main/java/org/jclouds/s3/blobstore/S3BlobStore.java @@ -266,6 +266,8 @@ public class S3BlobStore extends BaseBlobStore { if (overrides.getBlobAccess() == BlobAccess.PUBLIC_READ) { options = options.withAcl(CannedAccessPolicy.PUBLIC_READ); } + // TODO: S3 does not allow putObject if Tier.ARCHIVE. Instead, copyBlob + // after putBlob when the former supports tiers. return sync.putObject(container, blob2Object.apply(blob), options); } http://git-wip-us.apache.org/repos/asf/jclouds/blob/89053d9a/apis/s3/src/main/java/org/jclouds/s3/blobstore/functions/BlobToObjectMetadata.java ---------------------------------------------------------------------- diff --git a/apis/s3/src/main/java/org/jclouds/s3/blobstore/functions/BlobToObjectMetadata.java b/apis/s3/src/main/java/org/jclouds/s3/blobstore/functions/BlobToObjectMetadata.java index cfa97ae..48bae93 100644 --- a/apis/s3/src/main/java/org/jclouds/s3/blobstore/functions/BlobToObjectMetadata.java +++ b/apis/s3/src/main/java/org/jclouds/s3/blobstore/functions/BlobToObjectMetadata.java @@ -26,6 +26,7 @@ import org.jclouds.http.HttpUtils; import org.jclouds.rest.InvocationContext; import org.jclouds.rest.internal.GeneratedHttpRequest; import org.jclouds.s3.domain.MutableObjectMetadata; +import org.jclouds.s3.domain.ObjectMetadata.StorageClass; import org.jclouds.s3.domain.internal.MutableObjectMetadataImpl; import com.google.common.base.Function; @@ -48,6 +49,7 @@ public class BlobToObjectMetadata implements Function<BlobMetadata, MutableObjec for (Entry<String, String> entry : from.getUserMetadata().entrySet()) to.getUserMetadata().put(entry.getKey().toLowerCase(), entry.getValue()); } + to.setStorageClass(StorageClass.fromTier(from.getTier())); return to; } http://git-wip-us.apache.org/repos/asf/jclouds/blob/89053d9a/apis/s3/src/main/java/org/jclouds/s3/blobstore/functions/ObjectToBlobMetadata.java ---------------------------------------------------------------------- diff --git a/apis/s3/src/main/java/org/jclouds/s3/blobstore/functions/ObjectToBlobMetadata.java b/apis/s3/src/main/java/org/jclouds/s3/blobstore/functions/ObjectToBlobMetadata.java index f8bf9bf..8af059a 100644 --- a/apis/s3/src/main/java/org/jclouds/s3/blobstore/functions/ObjectToBlobMetadata.java +++ b/apis/s3/src/main/java/org/jclouds/s3/blobstore/functions/ObjectToBlobMetadata.java @@ -51,6 +51,7 @@ public class ObjectToBlobMetadata implements Function<ObjectMetadata, MutableBlo to.setLocation(locationOfBucket.apply(from.getBucket())); to.setType(StorageType.BLOB); to.setSize(from.getContentMetadata().getContentLength()); + to.setTier((from.getStorageClass() == null ? ObjectMetadata.StorageClass.STANDARD : from.getStorageClass()).toTier()); return to; } } http://git-wip-us.apache.org/repos/asf/jclouds/blob/89053d9a/apis/s3/src/main/java/org/jclouds/s3/domain/MutableObjectMetadata.java ---------------------------------------------------------------------- diff --git a/apis/s3/src/main/java/org/jclouds/s3/domain/MutableObjectMetadata.java b/apis/s3/src/main/java/org/jclouds/s3/domain/MutableObjectMetadata.java index 8ec9a71..945bb07 100644 --- a/apis/s3/src/main/java/org/jclouds/s3/domain/MutableObjectMetadata.java +++ b/apis/s3/src/main/java/org/jclouds/s3/domain/MutableObjectMetadata.java @@ -21,6 +21,7 @@ import java.util.Date; import java.util.Map; import org.jclouds.io.MutableContentMetadata; +import org.jclouds.s3.domain.ObjectMetadata.StorageClass; import org.jclouds.s3.domain.internal.MutableObjectMetadataImpl; import com.google.inject.ImplementedBy; http://git-wip-us.apache.org/repos/asf/jclouds/blob/89053d9a/apis/s3/src/main/java/org/jclouds/s3/domain/ObjectMetadata.java ---------------------------------------------------------------------- diff --git a/apis/s3/src/main/java/org/jclouds/s3/domain/ObjectMetadata.java b/apis/s3/src/main/java/org/jclouds/s3/domain/ObjectMetadata.java index 00b8508..c439a35 100644 --- a/apis/s3/src/main/java/org/jclouds/s3/domain/ObjectMetadata.java +++ b/apis/s3/src/main/java/org/jclouds/s3/domain/ObjectMetadata.java @@ -16,10 +16,13 @@ */ package org.jclouds.s3.domain; +import static com.google.common.base.Preconditions.checkNotNull; + import java.net.URI; import java.util.Date; import java.util.Map; +import org.jclouds.blobstore.domain.Tier; import org.jclouds.io.ContentMetadata; /** @@ -31,7 +34,29 @@ import org.jclouds.io.ContentMetadata; public interface ObjectMetadata extends Comparable<ObjectMetadata> { public enum StorageClass { - STANDARD, STANDARD_IA, REDUCED_REDUNDANCY + STANDARD(Tier.STANDARD), + STANDARD_IA(Tier.INFREQUENT), + REDUCED_REDUNDANCY(Tier.STANDARD), + GLACIER(Tier.ARCHIVE); + + private final Tier tier; + + private StorageClass(Tier tier) { + this.tier = checkNotNull(tier, "tier"); + } + + public static StorageClass fromTier(Tier tier) { + switch (tier) { + case STANDARD: return StorageClass.STANDARD; + case INFREQUENT: return StorageClass.STANDARD_IA; + case ARCHIVE: return StorageClass.GLACIER; + } + throw new IllegalArgumentException("invalid tier: " + tier); + } + + public Tier toTier() { + return tier; + } } /** http://git-wip-us.apache.org/repos/asf/jclouds/blob/89053d9a/apis/s3/src/main/java/org/jclouds/s3/functions/ParseObjectMetadataFromHeaders.java ---------------------------------------------------------------------- diff --git a/apis/s3/src/main/java/org/jclouds/s3/functions/ParseObjectMetadataFromHeaders.java b/apis/s3/src/main/java/org/jclouds/s3/functions/ParseObjectMetadataFromHeaders.java index 71d81f0..aadb313 100644 --- a/apis/s3/src/main/java/org/jclouds/s3/functions/ParseObjectMetadataFromHeaders.java +++ b/apis/s3/src/main/java/org/jclouds/s3/functions/ParseObjectMetadataFromHeaders.java @@ -32,6 +32,7 @@ import org.jclouds.http.HttpResponse; import org.jclouds.rest.InvocationContext; import org.jclouds.s3.blobstore.functions.BlobToObjectMetadata; import org.jclouds.s3.domain.MutableObjectMetadata; +import org.jclouds.s3.domain.ObjectMetadata.StorageClass; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Function; @@ -75,6 +76,10 @@ public class ParseObjectMetadataFromHeaders implements Function<HttpResponse, Mu // amz has an etag, but matches syntax for usermetadata to.getUserMetadata().remove("object-etag"); to.setCacheControl(from.getFirstHeaderOrNull(HttpHeaders.CACHE_CONTROL)); + String storageClass = from.getFirstHeaderOrNull("x-amz-storage-class"); + if (storageClass != null) { + to.setStorageClass(StorageClass.valueOf(storageClass)); + } return to; } http://git-wip-us.apache.org/repos/asf/jclouds/blob/89053d9a/apis/s3/src/test/java/org/jclouds/s3/blobstore/integration/S3BlobIntegrationLiveTest.java ---------------------------------------------------------------------- diff --git a/apis/s3/src/test/java/org/jclouds/s3/blobstore/integration/S3BlobIntegrationLiveTest.java b/apis/s3/src/test/java/org/jclouds/s3/blobstore/integration/S3BlobIntegrationLiveTest.java index 54890e9..09ab6b3 100644 --- a/apis/s3/src/test/java/org/jclouds/s3/blobstore/integration/S3BlobIntegrationLiveTest.java +++ b/apis/s3/src/test/java/org/jclouds/s3/blobstore/integration/S3BlobIntegrationLiveTest.java @@ -16,13 +16,17 @@ */ package org.jclouds.s3.blobstore.integration; +import static org.assertj.core.api.Fail.failBecauseExceptionWasNotThrown; + import java.io.IOException; import java.util.Properties; import java.util.concurrent.ExecutionException; import org.jclouds.blobstore.integration.internal.BaseBlobIntegrationTest; import org.jclouds.blobstore.integration.internal.BaseBlobStoreIntegrationTest; +import org.jclouds.http.HttpResponseException; import org.jclouds.s3.blobstore.strategy.MultipartUpload; +import org.testng.SkipException; import org.testng.annotations.Test; @Test(groups = "live", testName = "S3BlobIntegrationLiveTest") @@ -51,4 +55,23 @@ public class S3BlobIntegrationLiveTest extends BaseBlobIntegrationTest { super.testPutObjectStream(); } + @Override + public void testPutBlobTierArchive() throws Exception { + try { + super.testPutBlobTierArchive(); + failBecauseExceptionWasNotThrown(HttpResponseException.class); + } catch (HttpResponseException hre) { + throw new SkipException("S3 does not allow setting Glacier storage class on putBlob", hre); + } + } + + @Override + public void testPutBlobTierArchiveMultipart() throws Exception { + try { + super.testPutBlobTierArchiveMultipart(); + failBecauseExceptionWasNotThrown(HttpResponseException.class); + } catch (HttpResponseException hre) { + throw new SkipException("S3 does not allow setting Glacier storage class on putBlob", hre); + } + } } http://git-wip-us.apache.org/repos/asf/jclouds/blob/89053d9a/providers/aws-s3/src/main/java/org/jclouds/aws/s3/blobstore/AWSS3BlobStore.java ---------------------------------------------------------------------- diff --git a/providers/aws-s3/src/main/java/org/jclouds/aws/s3/blobstore/AWSS3BlobStore.java b/providers/aws-s3/src/main/java/org/jclouds/aws/s3/blobstore/AWSS3BlobStore.java index 3d9f17a..90ba59d 100644 --- a/providers/aws-s3/src/main/java/org/jclouds/aws/s3/blobstore/AWSS3BlobStore.java +++ b/providers/aws-s3/src/main/java/org/jclouds/aws/s3/blobstore/AWSS3BlobStore.java @@ -16,8 +16,6 @@ */ package org.jclouds.aws.s3.blobstore; -import static org.jclouds.s3.domain.ObjectMetadata.StorageClass.REDUCED_REDUNDANCY; - import java.util.Set; import javax.inject.Inject; @@ -25,15 +23,11 @@ import javax.inject.Provider; import org.jclouds.aws.domain.Region; import org.jclouds.aws.s3.AWSS3Client; -import org.jclouds.aws.s3.blobstore.options.AWSS3PutObjectOptions; -import org.jclouds.aws.s3.blobstore.options.AWSS3PutOptions; import org.jclouds.blobstore.BlobStoreContext; -import org.jclouds.blobstore.domain.Blob; import org.jclouds.blobstore.domain.PageSet; import org.jclouds.blobstore.domain.StorageMetadata; import org.jclouds.blobstore.functions.BlobToHttpGetOptions; import org.jclouds.blobstore.options.CreateContainerOptions; -import org.jclouds.blobstore.options.PutOptions; import org.jclouds.blobstore.strategy.internal.FetchBlobMetadata; import org.jclouds.blobstore.util.BlobUtils; import org.jclouds.collect.Memoized; @@ -47,7 +41,6 @@ import org.jclouds.s3.blobstore.functions.ContainerToBucketListOptions; import org.jclouds.s3.blobstore.functions.ObjectToBlob; import org.jclouds.s3.blobstore.functions.ObjectToBlobMetadata; import org.jclouds.s3.domain.BucketMetadata; -import org.jclouds.s3.domain.ObjectMetadata; import com.google.common.base.Function; import com.google.common.base.Supplier; @@ -74,25 +67,6 @@ public class AWSS3BlobStore extends S3BlobStore { } @Override - public String putBlob(String container, Blob blob, PutOptions options) { - if (options.isMultipart()) { - return putMultipartBlob(container, blob, options); - } else if ((options instanceof AWSS3PutOptions) && - (((AWSS3PutOptions) options).getStorageClass() == REDUCED_REDUNDANCY)) { - return putBlobWithReducedRedundancy(container, blob); - - } else { - return super.putBlob(container, blob, options); - } - } - - private String putBlobWithReducedRedundancy(String container, Blob blob) { - AWSS3PutObjectOptions options = new AWSS3PutObjectOptions(); - options.storageClass(ObjectMetadata.StorageClass.REDUCED_REDUNDANCY); - return getContext().unwrapApi(AWSS3Client.class).putObject(container, blob2Object.apply(blob), options); - } - - @Override public boolean createContainerInLocation(Location location, String container, CreateContainerOptions options) { if ((location == null || location.getId().equals(Region.US_STANDARD)) && http://git-wip-us.apache.org/repos/asf/jclouds/blob/89053d9a/providers/aws-s3/src/main/java/org/jclouds/aws/s3/blobstore/options/AWSS3PutObjectOptions.java ---------------------------------------------------------------------- diff --git a/providers/aws-s3/src/main/java/org/jclouds/aws/s3/blobstore/options/AWSS3PutObjectOptions.java b/providers/aws-s3/src/main/java/org/jclouds/aws/s3/blobstore/options/AWSS3PutObjectOptions.java index 66ab815..216f496 100644 --- a/providers/aws-s3/src/main/java/org/jclouds/aws/s3/blobstore/options/AWSS3PutObjectOptions.java +++ b/providers/aws-s3/src/main/java/org/jclouds/aws/s3/blobstore/options/AWSS3PutObjectOptions.java @@ -33,6 +33,7 @@ public class AWSS3PutObjectOptions extends PutObjectOptions { /** * @see AWSS3PutObjectOptions#storageClass */ + @Deprecated public static AWSS3PutObjectOptions storageClass(ObjectMetadata.StorageClass storageClass) { AWSS3PutObjectOptions options = new AWSS3PutObjectOptions(); return options.storageClass(storageClass); @@ -49,6 +50,7 @@ public class AWSS3PutObjectOptions extends PutObjectOptions { private ObjectMetadata.StorageClass storageClass = ObjectMetadata.StorageClass.STANDARD; + @Deprecated public AWSS3PutObjectOptions storageClass(ObjectMetadata.StorageClass storageClass) { this.storageClass = storageClass; if (storageClass != ObjectMetadata.StorageClass.STANDARD) { @@ -57,6 +59,7 @@ public class AWSS3PutObjectOptions extends PutObjectOptions { return this; } + @Deprecated public ObjectMetadata.StorageClass getStorageClass() { return storageClass; }
