This is an automated email from the ASF dual-hosted git repository. btellier pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/james-project.git
commit d5e93159b9f72aa9e90636efcd9811344fb9489d Author: Benoit Tellier <[email protected]> AuthorDate: Fri Jul 31 14:27:19 2020 +0700 JAMES-3028 Expose DeDuplication/PassThrough choice for S3 DumbBlobStore --- .../CassandraRabbitMQAwsS3SmtpTestRuleFactory.java | 8 +- server/blob/blob-s3/pom.xml | 5 + .../james/blob/objectstorage/aws/S3BlobStore.java | 130 --------------------- ...Test.java => S3DeDuplicationBlobStoreTest.java} | 12 +- ...reTest.java => S3PassThroughBlobStoreTest.java} | 12 +- .../modules/blobstore/BlobStoreModulesChooser.java | 18 +-- 6 files changed, 30 insertions(+), 155 deletions(-) diff --git a/mpt/impl/smtp/cassandra-rabbitmq-object-storage/src/test/java/org/apache/james/mpt/smtp/CassandraRabbitMQAwsS3SmtpTestRuleFactory.java b/mpt/impl/smtp/cassandra-rabbitmq-object-storage/src/test/java/org/apache/james/mpt/smtp/CassandraRabbitMQAwsS3SmtpTestRuleFactory.java index d97bbd7..a09f56e 100644 --- a/mpt/impl/smtp/cassandra-rabbitmq-object-storage/src/test/java/org/apache/james/mpt/smtp/CassandraRabbitMQAwsS3SmtpTestRuleFactory.java +++ b/mpt/impl/smtp/cassandra-rabbitmq-object-storage/src/test/java/org/apache/james/mpt/smtp/CassandraRabbitMQAwsS3SmtpTestRuleFactory.java @@ -26,9 +26,9 @@ import org.apache.james.backends.cassandra.DockerCassandra; import org.apache.james.backends.cassandra.init.configuration.ClusterConfiguration; import org.apache.james.backends.rabbitmq.DockerRabbitMQSingleton; import org.apache.james.blob.api.BlobStore; -import org.apache.james.blob.api.BucketName; +import org.apache.james.blob.api.DumbBlobStore; import org.apache.james.blob.api.MetricableBlobStore; -import org.apache.james.blob.objectstorage.aws.S3BlobStore; +import org.apache.james.blob.objectstorage.aws.S3DumbBlobStore; import org.apache.james.dnsservice.api.DNSService; import org.apache.james.modules.TestRabbitMQModule; import org.apache.james.modules.mailbox.KeyspacesConfiguration; @@ -38,6 +38,7 @@ import org.apache.james.modules.rabbitmq.RabbitMQModule; import org.apache.james.modules.server.CamelMailetContainerModule; import org.apache.james.queue.api.MailQueueItemDecoratorFactory; import org.apache.james.queue.api.RawMailQueueItemDecoratorFactory; +import org.apache.james.server.blob.deduplication.DeDuplicationBlobStore; import org.apache.james.server.core.configuration.Configuration; import org.apache.james.util.Host; import org.junit.rules.TemporaryFolder; @@ -58,7 +59,8 @@ public final class CassandraRabbitMQAwsS3SmtpTestRuleFactory { protected void configure() { bind(BlobStore.class) .annotatedWith(Names.named(MetricableBlobStore.BLOB_STORE_IMPLEMENTATION)) - .to(S3BlobStore.class); + .to(DeDuplicationBlobStore.class); + bind(DumbBlobStore.class).to(S3DumbBlobStore.class); } }; diff --git a/server/blob/blob-s3/pom.xml b/server/blob/blob-s3/pom.xml index b57a33b..31cf300 100644 --- a/server/blob/blob-s3/pom.xml +++ b/server/blob/blob-s3/pom.xml @@ -49,6 +49,11 @@ </dependency> <dependency> <groupId>${james.groupId}</groupId> + <artifactId>blob-storage-strategy</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>${james.groupId}</groupId> <artifactId>james-core</artifactId> </dependency> <dependency> diff --git a/server/blob/blob-s3/src/main/java/org/apache/james/blob/objectstorage/aws/S3BlobStore.java b/server/blob/blob-s3/src/main/java/org/apache/james/blob/objectstorage/aws/S3BlobStore.java deleted file mode 100644 index 1eba115..0000000 --- a/server/blob/blob-s3/src/main/java/org/apache/james/blob/objectstorage/aws/S3BlobStore.java +++ /dev/null @@ -1,130 +0,0 @@ -/**************************************************************** - * Licensed to the Apache Software Foundation (ASF) under one * - * or more contributor license agreements. See the NOTICE file * - * distributed with this work for additional information * - * regarding copyright ownership. The ASF licenses this file * - * to you under the Apache License, Version 2.0 (the * - * "License"); you may not use this file except in compliance * - * with the License. You may obtain a copy of the License at * - * * - * http://www.apache.org/licenses/LICENSE-2.0 * - * * - * Unless required by applicable law or agreed to in writing, * - * software distributed under the License is distributed on an * - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * - * KIND, either express or implied. See the License for the * - * specific language governing permissions and limitations * - * under the License. * - ****************************************************************/ - -package org.apache.james.blob.objectstorage.aws; - -import java.io.IOException; -import java.io.InputStream; - -import javax.inject.Inject; - -import org.apache.commons.io.IOUtils; -import org.apache.james.blob.api.BlobId; -import org.apache.james.blob.api.BlobStore; -import org.apache.james.blob.api.BucketName; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import com.google.common.hash.Hashing; -import com.google.common.hash.HashingInputStream; -import com.google.common.io.FileBackedOutputStream; - -import reactor.core.publisher.Mono; -import reactor.util.function.Tuples; - -public class S3BlobStore implements BlobStore { - - private static final boolean LAZY_RESOURCE_CLEANUP = false; - private static final int FILE_THRESHOLD = 1024 * 100; - private final S3DumbBlobStore dumbBlobStore; - private final BlobId.Factory blobIdFactory; - private final BucketName defaultBucket; - - @Inject - @VisibleForTesting - S3BlobStore(S3DumbBlobStore dumbBlobStore, BlobId.Factory blobIdFactory, BucketName defaultBucket) { - this.dumbBlobStore = dumbBlobStore; - this.blobIdFactory = blobIdFactory; - this.defaultBucket = defaultBucket; - } - - @Override - public Mono<BlobId> save(BucketName bucketName, byte[] data, StoragePolicy storagePolicy) { - Preconditions.checkNotNull(bucketName); - Preconditions.checkNotNull(data); - - BlobId blobId = blobIdFactory.forPayload(data); - - return dumbBlobStore.save(bucketName, blobId, data) - .then(Mono.just(blobId)); - } - - @Override - public Mono<BlobId> save(BucketName bucketName, InputStream data, StoragePolicy storagePolicy) { - Preconditions.checkNotNull(bucketName); - Preconditions.checkNotNull(data); - HashingInputStream hashingInputStream = new HashingInputStream(Hashing.sha256(), data); - return Mono.using( - () -> new FileBackedOutputStream(FILE_THRESHOLD), - fileBackedOutputStream -> saveAndGenerateBlobId(bucketName, hashingInputStream, fileBackedOutputStream), - fileBackedOutputStream -> { - try { - fileBackedOutputStream.reset(); - } catch (IOException e) { - throw new RuntimeException(e); - } - }, - LAZY_RESOURCE_CLEANUP); - } - - private Mono<BlobId> saveAndGenerateBlobId(BucketName bucketName, HashingInputStream hashingInputStream, FileBackedOutputStream fileBackedOutputStream) { - return Mono - .fromCallable(() -> { - IOUtils.copy(hashingInputStream, fileBackedOutputStream); - return Tuples.of(blobIdFactory.from(hashingInputStream.hash().toString()), fileBackedOutputStream.asByteSource()); - }) - .flatMap(tuple -> dumbBlobStore.save(bucketName, tuple.getT1(), tuple.getT2()).thenReturn(tuple.getT1())); - } - - - @Override - public Mono<byte[]> readBytes(BucketName bucketName, BlobId blobId) { - Preconditions.checkNotNull(bucketName); - - return dumbBlobStore.readBytes(bucketName, blobId); - } - - @Override - public InputStream read(BucketName bucketName, BlobId blobId) { - Preconditions.checkNotNull(bucketName); - - return dumbBlobStore.read(bucketName, blobId); - } - - @Override - public Mono<Void> deleteBucket(BucketName bucketName) { - Preconditions.checkNotNull(bucketName); - - return dumbBlobStore.deleteBucket(bucketName); - } - - @Override - public BucketName getDefaultBucketName() { - return defaultBucket; - } - - @Override - public Mono<Void> delete(BucketName bucketName, BlobId blobId) { - Preconditions.checkNotNull(bucketName); - Preconditions.checkNotNull(blobId); - - return dumbBlobStore.delete(bucketName, blobId); - } - -} diff --git a/server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/S3BlobStoreTest.java b/server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/S3DeDuplicationBlobStoreTest.java similarity index 87% copy from server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/S3BlobStoreTest.java copy to server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/S3DeDuplicationBlobStoreTest.java index 286c300..6bc1d34 100644 --- a/server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/S3BlobStoreTest.java +++ b/server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/S3DeDuplicationBlobStoreTest.java @@ -22,17 +22,17 @@ package org.apache.james.blob.objectstorage.aws; import org.apache.james.blob.api.BlobId; import org.apache.james.blob.api.BlobStore; import org.apache.james.blob.api.BlobStoreContract; -import org.apache.james.blob.api.BucketName; import org.apache.james.blob.api.HashBlobId; +import org.apache.james.server.blob.deduplication.BlobStoreFactory; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.extension.ExtendWith; @ExtendWith(DockerAwsS3Extension.class) -class S3BlobStoreTest implements BlobStoreContract { +class S3DeDuplicationBlobStoreTest implements BlobStoreContract { - private static S3BlobStore testee; + private static BlobStore testee; private static S3DumbBlobStore s3DumbBlobStore; @BeforeAll @@ -44,7 +44,11 @@ class S3BlobStoreTest implements BlobStoreContract { .build(); s3DumbBlobStore = new S3DumbBlobStore(configuration, dockerAwsS3.dockerAwsS3().region()); - testee = new S3BlobStore(s3DumbBlobStore, new HashBlobId.Factory(), BucketName.DEFAULT); + testee = BlobStoreFactory.builder() + .dumbBlobStore(s3DumbBlobStore) + .blobIdFactory(new HashBlobId.Factory()) + .defaultBucketName() + .passthrough(); } @AfterEach diff --git a/server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/S3BlobStoreTest.java b/server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/S3PassThroughBlobStoreTest.java similarity index 87% rename from server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/S3BlobStoreTest.java rename to server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/S3PassThroughBlobStoreTest.java index 286c300..71027d9 100644 --- a/server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/S3BlobStoreTest.java +++ b/server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/S3PassThroughBlobStoreTest.java @@ -22,17 +22,17 @@ package org.apache.james.blob.objectstorage.aws; import org.apache.james.blob.api.BlobId; import org.apache.james.blob.api.BlobStore; import org.apache.james.blob.api.BlobStoreContract; -import org.apache.james.blob.api.BucketName; import org.apache.james.blob.api.HashBlobId; +import org.apache.james.server.blob.deduplication.BlobStoreFactory; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.extension.ExtendWith; @ExtendWith(DockerAwsS3Extension.class) -class S3BlobStoreTest implements BlobStoreContract { +class S3PassThroughBlobStoreTest implements BlobStoreContract { - private static S3BlobStore testee; + private static BlobStore testee; private static S3DumbBlobStore s3DumbBlobStore; @BeforeAll @@ -44,7 +44,11 @@ class S3BlobStoreTest implements BlobStoreContract { .build(); s3DumbBlobStore = new S3DumbBlobStore(configuration, dockerAwsS3.dockerAwsS3().region()); - testee = new S3BlobStore(s3DumbBlobStore, new HashBlobId.Factory(), BucketName.DEFAULT); + testee = BlobStoreFactory.builder() + .dumbBlobStore(s3DumbBlobStore) + .blobIdFactory(new HashBlobId.Factory()) + .defaultBucketName() + .deduplication(); } @AfterEach diff --git a/server/container/guice/cassandra-rabbitmq-guice/src/main/java/org/apache/james/modules/blobstore/BlobStoreModulesChooser.java b/server/container/guice/cassandra-rabbitmq-guice/src/main/java/org/apache/james/modules/blobstore/BlobStoreModulesChooser.java index 770cffa..0aab073 100644 --- a/server/container/guice/cassandra-rabbitmq-guice/src/main/java/org/apache/james/modules/blobstore/BlobStoreModulesChooser.java +++ b/server/container/guice/cassandra-rabbitmq-guice/src/main/java/org/apache/james/modules/blobstore/BlobStoreModulesChooser.java @@ -25,7 +25,6 @@ import org.apache.james.blob.api.BlobStore; import org.apache.james.blob.api.DumbBlobStore; import org.apache.james.blob.cassandra.CassandraDumbBlobStore; import org.apache.james.blob.cassandra.cache.CachedBlobStore; -import org.apache.james.blob.objectstorage.aws.S3BlobStore; import org.apache.james.blob.objectstorage.aws.S3DumbBlobStore; import org.apache.james.eventsourcing.Event; import org.apache.james.eventsourcing.eventstore.cassandra.dto.EventDTO; @@ -64,10 +63,6 @@ public class BlobStoreModulesChooser { install(new S3BlobStoreModule()); bind(DumbBlobStore.class).to(S3DumbBlobStore.class); - - bind(BlobStore.class) - .annotatedWith(Names.named(CachedBlobStore.BACKEND)) - .to(S3BlobStore.class); } } @@ -94,16 +89,11 @@ public class BlobStoreModulesChooser { @VisibleForTesting public static List<Module> chooseModules(BlobStoreConfiguration choosingConfiguration) { - ImmutableList.Builder<Module> moduleBuilder = ImmutableList.<Module>builder() + return ImmutableList.<Module>builder() .add(chooseDumBlobStoreModule(choosingConfiguration.getImplementation())) - .add(new StoragePolicyConfigurationSanityEnforcementModule(choosingConfiguration)); - - //TODO JAMES-3028 add the storage policy module for all implementation and unbind the ObjectStorageBlobStore - if (choosingConfiguration.getImplementation() == BlobStoreConfiguration.BlobStoreImplName.CASSANDRA) { - moduleBuilder.add( - chooseStoragePolicyModule(choosingConfiguration.storageStrategy())); - } - return moduleBuilder.build(); + .add(chooseStoragePolicyModule(choosingConfiguration.storageStrategy())) + .add(new StoragePolicyConfigurationSanityEnforcementModule(choosingConfiguration)) + .build(); } public static Module chooseDumBlobStoreModule(BlobStoreConfiguration.BlobStoreImplName implementation) { --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
