This is an automated email from the ASF dual-hosted git repository. rcordier pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/james-project.git
commit 71187b503a5b5f012b590289363d0992ed067ca8 Author: Benoit Tellier <[email protected]> AuthorDate: Fri Jul 24 09:26:11 2020 +0700 JAMES-3318 BlobStoreDeletedMessageVault should use DumbBlobStore for effective deletion --- .../vault/blob/BlobStoreDeletedMessageVault.java | 7 +- .../james/vault/DeletedMessageVaultHookTest.java | 9 ++- .../blob/BlobStoreDeletedMessageVaultTest.java | 9 ++- .../blob/objectstorage/ObjectStorageBlobStore.java | 7 ++ .../modules/blobstore/BlobStoreModulesChooser.java | 1 + .../james/modules/blobstore/NoopDumbBlobStore.java | 83 ++++++++++++++++++++++ .../routes/DeletedMessagesVaultRoutesTest.java | 9 ++- 7 files changed, 114 insertions(+), 11 deletions(-) diff --git a/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/blob/BlobStoreDeletedMessageVault.java b/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/blob/BlobStoreDeletedMessageVault.java index 8febe09..2bc66b4 100644 --- a/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/blob/BlobStoreDeletedMessageVault.java +++ b/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/blob/BlobStoreDeletedMessageVault.java @@ -30,6 +30,7 @@ import javax.inject.Inject; import org.apache.james.blob.api.BlobStore; import org.apache.james.blob.api.BucketName; +import org.apache.james.blob.api.DumbBlobStore; import org.apache.james.blob.api.ObjectNotFoundException; import org.apache.james.core.Username; import org.apache.james.mailbox.model.MessageId; @@ -67,6 +68,7 @@ public class BlobStoreDeletedMessageVault implements DeletedMessageVault { private final MetricFactory metricFactory; private final DeletedMessageMetadataVault messageMetadataVault; private final BlobStore blobStore; + private final DumbBlobStore dumbBlobStore; private final BucketNameGenerator nameGenerator; private final Clock clock; private final RetentionConfiguration retentionConfiguration; @@ -74,12 +76,13 @@ public class BlobStoreDeletedMessageVault implements DeletedMessageVault { @Inject public BlobStoreDeletedMessageVault(MetricFactory metricFactory, DeletedMessageMetadataVault messageMetadataVault, - BlobStore blobStore, BucketNameGenerator nameGenerator, + BlobStore blobStore, DumbBlobStore dumbBlobStore, BucketNameGenerator nameGenerator, Clock clock, RetentionConfiguration retentionConfiguration) { this.metricFactory = metricFactory; this.messageMetadataVault = messageMetadataVault; this.blobStore = blobStore; + this.dumbBlobStore = dumbBlobStore; this.nameGenerator = nameGenerator; this.clock = clock; this.retentionConfiguration = retentionConfiguration; @@ -156,7 +159,7 @@ public class BlobStoreDeletedMessageVault implements DeletedMessageVault { return Mono.from(messageMetadataVault.retrieveStorageInformation(username, messageId)) .flatMap(storageInformation -> Mono.from(messageMetadataVault.remove(storageInformation.getBucketName(), username, messageId)) .thenReturn(storageInformation)) - .flatMap(storageInformation -> Mono.from(blobStore.delete(storageInformation.getBucketName(), storageInformation.getBlobId()))) + .flatMap(storageInformation -> Mono.from(dumbBlobStore.delete(storageInformation.getBucketName(), storageInformation.getBlobId()))) .subscribeOn(Schedulers.elastic()); } diff --git a/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/DeletedMessageVaultHookTest.java b/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/DeletedMessageVaultHookTest.java index 0fa94af..688574f 100644 --- a/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/DeletedMessageVaultHookTest.java +++ b/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/DeletedMessageVaultHookTest.java @@ -30,7 +30,7 @@ import java.time.ZoneOffset; import java.util.List; import org.apache.james.blob.api.HashBlobId; -import org.apache.james.blob.memory.MemoryBlobStoreFactory; +import org.apache.james.blob.memory.MemoryDumbBlobStore; import org.apache.james.core.MailAddress; import org.apache.james.core.MaybeSender; import org.apache.james.core.Username; @@ -50,6 +50,7 @@ import org.apache.james.mailbox.model.MessageRange; import org.apache.james.mailbox.model.SearchQuery; import org.apache.james.metrics.tests.RecordingMetricFactory; import org.apache.james.mime4j.dom.Message; +import org.apache.james.server.blob.deduplication.BlobStoreFactory; import org.apache.james.vault.blob.BlobStoreDeletedMessageVault; import org.apache.james.vault.blob.BucketNameGenerator; import org.apache.james.vault.memory.metadata.MemoryDeletedMessageMetadataVault; @@ -108,11 +109,13 @@ class DeletedMessageVaultHookTest { @BeforeEach void setUp() throws Exception { clock = Clock.fixed(DELETION_DATE.toInstant(), ZoneOffset.UTC); + MemoryDumbBlobStore dumbBlobStore = new MemoryDumbBlobStore(); messageVault = new BlobStoreDeletedMessageVault(new RecordingMetricFactory(), new MemoryDeletedMessageMetadataVault(), - MemoryBlobStoreFactory.builder() + BlobStoreFactory.builder() + .dumbBlobStore(dumbBlobStore) .blobIdFactory(new HashBlobId.Factory()) .defaultBucketName() - .passthrough(), new BucketNameGenerator(clock), clock, + .passthrough(), dumbBlobStore, new BucketNameGenerator(clock), clock, RetentionConfiguration.DEFAULT); DeletedMessageConverter deletedMessageConverter = new DeletedMessageConverter(); diff --git a/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/blob/BlobStoreDeletedMessageVaultTest.java b/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/blob/BlobStoreDeletedMessageVaultTest.java index 28b6506..982fefd 100644 --- a/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/blob/BlobStoreDeletedMessageVaultTest.java +++ b/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/blob/BlobStoreDeletedMessageVaultTest.java @@ -40,8 +40,9 @@ import java.time.ZonedDateTime; import org.apache.james.blob.api.BucketName; import org.apache.james.blob.api.HashBlobId; -import org.apache.james.blob.memory.MemoryBlobStoreFactory; +import org.apache.james.blob.memory.MemoryDumbBlobStore; import org.apache.james.metrics.tests.RecordingMetricFactory; +import org.apache.james.server.blob.deduplication.BlobStoreFactory; import org.apache.james.utils.UpdatableTickingClock; import org.apache.james.vault.DeletedMessageVault; import org.apache.james.vault.DeletedMessageVaultContract; @@ -63,12 +64,14 @@ class BlobStoreDeletedMessageVaultTest implements DeletedMessageVaultContract, D void setUp() { clock = new UpdatableTickingClock(NOW.toInstant()); metricFactory = new RecordingMetricFactory(); + MemoryDumbBlobStore dumbBlobStore = new MemoryDumbBlobStore(); messageVault = new BlobStoreDeletedMessageVault(metricFactory, new MemoryDeletedMessageMetadataVault(), - MemoryBlobStoreFactory.builder() + BlobStoreFactory.builder() + .dumbBlobStore(dumbBlobStore) .blobIdFactory(new HashBlobId.Factory()) .defaultBucketName() .passthrough(), - new BucketNameGenerator(clock), clock, RetentionConfiguration.DEFAULT); + dumbBlobStore, new BucketNameGenerator(clock), clock, RetentionConfiguration.DEFAULT); } @Override diff --git a/server/blob/blob-objectstorage/src/main/java/org/apache/james/blob/objectstorage/ObjectStorageBlobStore.java b/server/blob/blob-objectstorage/src/main/java/org/apache/james/blob/objectstorage/ObjectStorageBlobStore.java index 3c45fe8..cd7c7b3 100644 --- a/server/blob/blob-objectstorage/src/main/java/org/apache/james/blob/objectstorage/ObjectStorageBlobStore.java +++ b/server/blob/blob-objectstorage/src/main/java/org/apache/james/blob/objectstorage/ObjectStorageBlobStore.java @@ -214,4 +214,11 @@ public class ObjectStorageBlobStore implements BlobStore { return Mono.<Void>fromRunnable(() -> blobStore.removeBlob(resolvedBucketName.asString(), blobId.asString())) .subscribeOn(Schedulers.elastic()); } + + // Workaround while waiting for DumbBlobStore extraction + public Mono<Void> deleteEffectively(BucketName bucketName, BlobId blobId) { + ObjectStorageBucketName resolvedBucketName = bucketNameResolver.resolve(bucketName); + return Mono.<Void>fromRunnable(() -> blobStore.removeBlob(resolvedBucketName.asString(), blobId.asString())) + .subscribeOn(Schedulers.elastic()); + } } diff --git a/server/container/guice/cassandra-rabbitmq-guice/src/main/java/org/apache/james/modules/blobstore/BlobStoreModulesChooser.java b/server/container/guice/cassandra-rabbitmq-guice/src/main/java/org/apache/james/modules/blobstore/BlobStoreModulesChooser.java index d790968..c0cdcc2 100644 --- a/server/container/guice/cassandra-rabbitmq-guice/src/main/java/org/apache/james/modules/blobstore/BlobStoreModulesChooser.java +++ b/server/container/guice/cassandra-rabbitmq-guice/src/main/java/org/apache/james/modules/blobstore/BlobStoreModulesChooser.java @@ -54,6 +54,7 @@ public class BlobStoreModulesChooser { @Override protected void configure() { install(new ObjectStorageDependenciesModule()); + bind(DumbBlobStore.class).to(NoopDumbBlobStore.class); bind(BlobStore.class) .annotatedWith(Names.named(CachedBlobStore.BACKEND)) .to(ObjectStorageBlobStore.class); diff --git a/server/container/guice/cassandra-rabbitmq-guice/src/main/java/org/apache/james/modules/blobstore/NoopDumbBlobStore.java b/server/container/guice/cassandra-rabbitmq-guice/src/main/java/org/apache/james/modules/blobstore/NoopDumbBlobStore.java new file mode 100644 index 0000000..9ebc084 --- /dev/null +++ b/server/container/guice/cassandra-rabbitmq-guice/src/main/java/org/apache/james/modules/blobstore/NoopDumbBlobStore.java @@ -0,0 +1,83 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ + +package org.apache.james.modules.blobstore; + +import java.io.InputStream; + +import javax.inject.Inject; + +import org.apache.commons.lang3.NotImplementedException; +import org.apache.james.blob.api.BlobId; +import org.apache.james.blob.api.BucketName; +import org.apache.james.blob.api.DumbBlobStore; +import org.apache.james.blob.api.ObjectNotFoundException; +import org.apache.james.blob.api.ObjectStoreIOException; +import org.apache.james.blob.objectstorage.ObjectStorageBlobStore; +import org.reactivestreams.Publisher; + +import com.google.common.io.ByteSource; + + +/** + * This class is a workaround while waiting for a real DumbBlobStore to be extracted from ObjectStorageBlobStore + */ +public class NoopDumbBlobStore implements DumbBlobStore { + private final ObjectStorageBlobStore objectStorageBlobStore; + + @Inject + public NoopDumbBlobStore(ObjectStorageBlobStore objectStorageBlobStore) { + this.objectStorageBlobStore = objectStorageBlobStore; + } + + @Override + public InputStream read(BucketName bucketName, BlobId blobId) throws ObjectStoreIOException, ObjectNotFoundException { + throw new NotImplementedException("Not implemented"); + } + + @Override + public Publisher<byte[]> readBytes(BucketName bucketName, BlobId blobId) { + throw new NotImplementedException("Not implemented"); + } + + @Override + public Publisher<Void> save(BucketName bucketName, BlobId blobId, byte[] data) { + throw new NotImplementedException("Not implemented"); + } + + @Override + public Publisher<Void> save(BucketName bucketName, BlobId blobId, InputStream inputStream) { + throw new NotImplementedException("Not implemented"); + } + + @Override + public Publisher<Void> save(BucketName bucketName, BlobId blobId, ByteSource content) { + throw new NotImplementedException("Not implemented"); + } + + @Override + public Publisher<Void> delete(BucketName bucketName, BlobId blobId) { + return objectStorageBlobStore.deleteEffectively(bucketName, blobId); + } + + @Override + public Publisher<Void> deleteBucket(BucketName bucketName) { + throw new NotImplementedException("Not implemented"); + } +} diff --git a/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/src/test/java/org/apache/james/webadmin/vault/routes/DeletedMessagesVaultRoutesTest.java b/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/src/test/java/org/apache/james/webadmin/vault/routes/DeletedMessagesVaultRoutesTest.java index 1c8d463..cfdb6b3 100644 --- a/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/src/test/java/org/apache/james/webadmin/vault/routes/DeletedMessagesVaultRoutesTest.java +++ b/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/src/test/java/org/apache/james/webadmin/vault/routes/DeletedMessagesVaultRoutesTest.java @@ -76,7 +76,7 @@ import org.apache.james.blob.api.BlobStore; import org.apache.james.blob.api.BucketName; import org.apache.james.blob.api.HashBlobId; import org.apache.james.blob.export.api.BlobExportMechanism; -import org.apache.james.blob.memory.MemoryBlobStoreFactory; +import org.apache.james.blob.memory.MemoryDumbBlobStore; import org.apache.james.core.Domain; import org.apache.james.core.MailAddress; import org.apache.james.core.MaybeSender; @@ -102,6 +102,7 @@ import org.apache.james.mailbox.model.MessageResult; import org.apache.james.mailbox.model.MultimailboxesSearchQuery; import org.apache.james.mailbox.model.SearchQuery; import org.apache.james.metrics.tests.RecordingMetricFactory; +import org.apache.james.server.blob.deduplication.BlobStoreFactory; import org.apache.james.task.Hostname; import org.apache.james.task.MemoryTaskManager; import org.apache.james.user.memory.MemoryUsersRepository; @@ -181,13 +182,15 @@ class DeletedMessagesVaultRoutesTest { @BeforeEach void beforeEach() throws Exception { blobIdFactory = new HashBlobId.Factory(); - blobStore = spy(MemoryBlobStoreFactory.builder() + MemoryDumbBlobStore dumbBlobStore = new MemoryDumbBlobStore(); + blobStore = spy(BlobStoreFactory.builder() + .dumbBlobStore(dumbBlobStore) .blobIdFactory(blobIdFactory) .defaultBucketName() .passthrough()); clock = new UpdatableTickingClock(OLD_DELETION_DATE.toInstant()); vault = spy(new BlobStoreDeletedMessageVault(new RecordingMetricFactory(), new MemoryDeletedMessageMetadataVault(), - blobStore, new BucketNameGenerator(clock), clock, + blobStore, dumbBlobStore, new BucketNameGenerator(clock), clock, RetentionConfiguration.DEFAULT)); InMemoryIntegrationResources inMemoryResource = InMemoryIntegrationResources.defaultResources(); mailboxManager = spy(inMemoryResource.getMailboxManager()); --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
