This is an automated email from the ASF dual-hosted git repository. btellier pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/james-project.git
commit f52c3b63e6dd31b686445a3ac6644d5af23f3a95 Author: Tran Tien Duc <dt...@linagora.com> AuthorDate: Thu Mar 21 11:39:11 2019 +0700 JAMES-2685 LocalFileBlobExportModule for blob sharing to be used in integration tests --- pom.xml | 10 ++++ .../export/file/LocalFileBlobExportMechanism.java | 7 +++ .../file/LocalFileBlobExportMechanismTest.java | 5 +- .../pom.xml | 20 ++------ .../LocalFileBlobExportMechanismModule.java | 37 +++++++++++++++ server/container/guice/blob-memory-guice/pom.xml | 11 ----- server/container/guice/cassandra-guice/pom.xml | 4 ++ .../org/apache/james/CassandraJamesServerMain.java | 4 +- server/container/guice/memory-guice/pom.xml | 4 ++ .../org/apache/james/MemoryJamesServerMain.java | 2 + server/container/guice/pom.xml | 1 + .../memory-jmap-integration-testing/pom.xml | 5 ++ .../vault/routes/DeletedMessagesVaultRoutes.java | 5 +- .../james/webadmin/vault/routes/ExportService.java | 53 ++++++++++++---------- 14 files changed, 109 insertions(+), 59 deletions(-) diff --git a/pom.xml b/pom.xml index f478bff..0a66da5 100644 --- a/pom.xml +++ b/pom.xml @@ -1103,6 +1103,16 @@ </dependency> <dependency> <groupId>${james.groupId}</groupId> + <artifactId>blob-export-guice</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>${james.groupId}</groupId> + <artifactId>blob-export-file</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>${james.groupId}</groupId> <artifactId>blob-memory</artifactId> <version>${project.version}</version> </dependency> diff --git a/server/blob/blob-export-file/src/main/java/org/apache/james/blob/export/file/LocalFileBlobExportMechanism.java b/server/blob/blob-export-file/src/main/java/org/apache/james/blob/export/file/LocalFileBlobExportMechanism.java index f731c9a..656dc45 100644 --- a/server/blob/blob-export-file/src/main/java/org/apache/james/blob/export/file/LocalFileBlobExportMechanism.java +++ b/server/blob/blob-export-file/src/main/java/org/apache/james/blob/export/file/LocalFileBlobExportMechanism.java @@ -23,6 +23,8 @@ import java.io.File; import java.io.IOException; import java.net.UnknownHostException; +import javax.inject.Inject; + import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.RandomStringUtils; import org.apache.james.blob.api.BlobId; @@ -43,6 +45,10 @@ public class LocalFileBlobExportMechanism implements BlobExportMechanism { static final String CORRESPONDING_FILE_HEADER = "corresponding-file"; public static class Configuration { + + private static final String DEFAULT_DIRECTORY_LOCATION = "file://var/blobExporting"; + public static final Configuration DEFAULT_CONFIGURATION = new Configuration(DEFAULT_DIRECTORY_LOCATION); + private final String exportDirectory; public Configuration(String exportDirectory) { @@ -56,6 +62,7 @@ public class LocalFileBlobExportMechanism implements BlobExportMechanism { private final DNSService dnsService; private final Configuration configuration; + @Inject LocalFileBlobExportMechanism(MailetContext mailetContext, BlobStore blobStore, FileSystem fileSystem, DNSService dnsService, Configuration configuration) { this.mailetContext = mailetContext; this.blobStore = blobStore; diff --git a/server/blob/blob-export-file/src/test/java/org/apache/james/blob/export/file/LocalFileBlobExportMechanismTest.java b/server/blob/blob-export-file/src/test/java/org/apache/james/blob/export/file/LocalFileBlobExportMechanismTest.java index 35983a0..83cd0e6 100644 --- a/server/blob/blob-export-file/src/test/java/org/apache/james/blob/export/file/LocalFileBlobExportMechanismTest.java +++ b/server/blob/blob-export-file/src/test/java/org/apache/james/blob/export/file/LocalFileBlobExportMechanismTest.java @@ -64,9 +64,8 @@ class LocalFileBlobExportMechanismTest { DNSService dnsService = mock(DNSService.class); when(dnsService.getLocalHost()).thenReturn(localHost); - LocalFileBlobExportMechanism.Configuration blobExportConfiguration = new LocalFileBlobExportMechanism.Configuration("file://var/blobExporting"); - - testee = new LocalFileBlobExportMechanism(mailetContext, blobStore, fileSystem, dnsService, blobExportConfiguration); + testee = new LocalFileBlobExportMechanism(mailetContext, blobStore, fileSystem, dnsService, + LocalFileBlobExportMechanism.Configuration.DEFAULT_CONFIGURATION); } @Test diff --git a/server/container/guice/blob-memory-guice/pom.xml b/server/container/guice/blob-export-guice/pom.xml similarity index 73% copy from server/container/guice/blob-memory-guice/pom.xml copy to server/container/guice/blob-export-guice/pom.xml index fbe487e..29a358b 100644 --- a/server/container/guice/blob-memory-guice/pom.xml +++ b/server/container/guice/blob-export-guice/pom.xml @@ -28,35 +28,23 @@ <relativePath>../pom.xml</relativePath> </parent> - <artifactId>blob-memory-guice</artifactId> + <artifactId>blob-export-guice</artifactId> <packaging>jar</packaging> - <name>Apache James :: Server :: Blob Memory - guice injection</name> - <description>Blob modules on memory storage</description> + <name>Apache James :: Server :: Blob Exporting Mechanisms - guice injection</name> <dependencies> <dependency> <groupId>${james.groupId}</groupId> - <artifactId>blob-api</artifactId> + <artifactId>blob-export-api</artifactId> </dependency> <dependency> <groupId>${james.groupId}</groupId> - <artifactId>blob-memory</artifactId> + <artifactId>blob-export-file</artifactId> </dependency> <dependency> <groupId>${james.groupId}</groupId> <artifactId>james-server-guice-common</artifactId> </dependency> </dependencies> - <build> - <plugins> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-surefire-plugin</artifactId> - <configuration> - <reuseForks>true</reuseForks> - </configuration> - </plugin> - </plugins> - </build> </project> \ No newline at end of file diff --git a/server/container/guice/blob-export-guice/src/main/java/org/apache/james/modules/LocalFileBlobExportMechanismModule.java b/server/container/guice/blob-export-guice/src/main/java/org/apache/james/modules/LocalFileBlobExportMechanismModule.java new file mode 100644 index 0000000..7d38201 --- /dev/null +++ b/server/container/guice/blob-export-guice/src/main/java/org/apache/james/modules/LocalFileBlobExportMechanismModule.java @@ -0,0 +1,37 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ + +package org.apache.james.modules; + +import org.apache.james.blob.export.api.BlobExportMechanism; +import org.apache.james.blob.export.file.LocalFileBlobExportMechanism; + +import com.google.inject.AbstractModule; +import com.google.inject.Scopes; + +public class LocalFileBlobExportMechanismModule extends AbstractModule { + + @Override + protected void configure() { + bind(LocalFileBlobExportMechanism.Configuration.class).toInstance(LocalFileBlobExportMechanism.Configuration.DEFAULT_CONFIGURATION); + + bind(LocalFileBlobExportMechanism.class).in(Scopes.SINGLETON); + bind(BlobExportMechanism.class).to(LocalFileBlobExportMechanism.class); + } +} diff --git a/server/container/guice/blob-memory-guice/pom.xml b/server/container/guice/blob-memory-guice/pom.xml index fbe487e..e409945 100644 --- a/server/container/guice/blob-memory-guice/pom.xml +++ b/server/container/guice/blob-memory-guice/pom.xml @@ -48,15 +48,4 @@ <artifactId>james-server-guice-common</artifactId> </dependency> </dependencies> - <build> - <plugins> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-surefire-plugin</artifactId> - <configuration> - <reuseForks>true</reuseForks> - </configuration> - </plugin> - </plugins> - </build> </project> \ No newline at end of file diff --git a/server/container/guice/cassandra-guice/pom.xml b/server/container/guice/cassandra-guice/pom.xml index 01127fa..206e9bd 100644 --- a/server/container/guice/cassandra-guice/pom.xml +++ b/server/container/guice/cassandra-guice/pom.xml @@ -112,6 +112,10 @@ </dependency> <dependency> <groupId>${james.groupId}</groupId> + <artifactId>blob-export-guice</artifactId> + </dependency> + <dependency> + <groupId>${james.groupId}</groupId> <artifactId>event-sourcing-event-store-cassandra</artifactId> </dependency> <dependency> diff --git a/server/container/guice/cassandra-guice/src/main/java/org/apache/james/CassandraJamesServerMain.java b/server/container/guice/cassandra-guice/src/main/java/org/apache/james/CassandraJamesServerMain.java index a25e71d..f5f74d5 100644 --- a/server/container/guice/cassandra-guice/src/main/java/org/apache/james/CassandraJamesServerMain.java +++ b/server/container/guice/cassandra-guice/src/main/java/org/apache/james/CassandraJamesServerMain.java @@ -19,6 +19,7 @@ package org.apache.james; +import org.apache.james.modules.LocalFileBlobExportMechanismModule; import org.apache.james.modules.MailboxModule; import org.apache.james.modules.activemq.ActiveMQQueueModule; import org.apache.james.modules.data.CassandraDLPConfigurationStoreModule; @@ -124,7 +125,8 @@ public class CassandraJamesServerMain { CASSANDRA_SERVER_CORE_MODULE, CASSANDRA_MAILBOX_MODULE, PROTOCOLS, - PLUGINS); + PLUGINS, + new LocalFileBlobExportMechanismModule()); public static void main(String[] args) throws Exception { Configuration configuration = Configuration.builder() diff --git a/server/container/guice/memory-guice/pom.xml b/server/container/guice/memory-guice/pom.xml index 792c017..6e3ae2f 100644 --- a/server/container/guice/memory-guice/pom.xml +++ b/server/container/guice/memory-guice/pom.xml @@ -65,6 +65,10 @@ </dependency> <dependency> <groupId>${james.groupId}</groupId> + <artifactId>blob-export-guice</artifactId> + </dependency> + <dependency> + <groupId>${james.groupId}</groupId> <artifactId>blob-memory-guice</artifactId> </dependency> <dependency> diff --git a/server/container/guice/memory-guice/src/main/java/org/apache/james/MemoryJamesServerMain.java b/server/container/guice/memory-guice/src/main/java/org/apache/james/MemoryJamesServerMain.java index 6e0f235..c885f49 100644 --- a/server/container/guice/memory-guice/src/main/java/org/apache/james/MemoryJamesServerMain.java +++ b/server/container/guice/memory-guice/src/main/java/org/apache/james/MemoryJamesServerMain.java @@ -21,6 +21,7 @@ package org.apache.james; import org.apache.commons.configuration.DefaultConfigurationBuilder; import org.apache.james.modules.BlobMemoryModule; +import org.apache.james.modules.LocalFileBlobExportMechanismModule; import org.apache.james.modules.MailboxModule; import org.apache.james.modules.data.MemoryDataJmapModule; import org.apache.james.modules.data.MemoryDataModule; @@ -82,6 +83,7 @@ public class MemoryJamesServerMain { public static final Module IN_MEMORY_SERVER_MODULE = Modules.combine( new BlobMemoryModule(), new DeletedMessageVaultModule(), + new LocalFileBlobExportMechanismModule(), new MailboxModule(), new MemoryDataModule(), new MemoryEventStoreModule(), diff --git a/server/container/guice/pom.xml b/server/container/guice/pom.xml index 5948b80..f0f0ef0 100644 --- a/server/container/guice/pom.xml +++ b/server/container/guice/pom.xml @@ -34,6 +34,7 @@ <modules> <module>blob-api-guice</module> + <module>blob-export-guice</module> <module>blob-memory-guice</module> <module>blob-objectstorage-guice</module> <module>cassandra-guice</module> diff --git a/server/protocols/jmap-integration-testing/memory-jmap-integration-testing/pom.xml b/server/protocols/jmap-integration-testing/memory-jmap-integration-testing/pom.xml index cf4901c..02531d3 100644 --- a/server/protocols/jmap-integration-testing/memory-jmap-integration-testing/pom.xml +++ b/server/protocols/jmap-integration-testing/memory-jmap-integration-testing/pom.xml @@ -52,6 +52,11 @@ </dependency> <dependency> <groupId>${james.groupId}</groupId> + <artifactId>blob-export-guice</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>${james.groupId}</groupId> <artifactId>james-server-dnsservice-test</artifactId> </dependency> <dependency> diff --git a/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/src/main/java/org/apache/james/webadmin/vault/routes/DeletedMessagesVaultRoutes.java b/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/src/main/java/org/apache/james/webadmin/vault/routes/DeletedMessagesVaultRoutes.java index 940e3a8..7c1fa72 100644 --- a/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/src/main/java/org/apache/james/webadmin/vault/routes/DeletedMessagesVaultRoutes.java +++ b/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/src/main/java/org/apache/james/webadmin/vault/routes/DeletedMessagesVaultRoutes.java @@ -164,13 +164,12 @@ public class DeletedMessagesVaultRoutes implements Routes { name = "exportTo", paramType = "query", example = "?exportTo=u...@james.org", - value = "Compulsory if action is export. Needs to be a valid mail address to represent for the destination " + - "where deleted messages content is export to") + value = "Compulsory if action is export. Needs to be a valid mail address. The content of the vault matching the query will be sent to that address") }) @ApiResponses(value = { @ApiResponse(code = HttpStatus.CREATED_201, message = "Task is created", response = TaskIdDto.class), @ApiResponse(code = HttpStatus.BAD_REQUEST_400, message = "Bad request - user param is invalid"), - @ApiResponse(code = HttpStatus.NOT_FOUND_404, message = "Not found - requested user is not existed in the system"), + @ApiResponse(code = HttpStatus.NOT_FOUND_404, message = "Not found - requested user does not exist"), @ApiResponse(code = HttpStatus.INTERNAL_SERVER_ERROR_500, message = "Internal server error - Something went bad on the server side.") }) private TaskIdDto userActions(Request request, Response response) throws JsonExtractException { diff --git a/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/src/main/java/org/apache/james/webadmin/vault/routes/ExportService.java b/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/src/main/java/org/apache/james/webadmin/vault/routes/ExportService.java index bab02b8..2e3c014 100644 --- a/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/src/main/java/org/apache/james/webadmin/vault/routes/ExportService.java +++ b/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/src/main/java/org/apache/james/webadmin/vault/routes/ExportService.java @@ -19,11 +19,14 @@ package org.apache.james.webadmin.vault.routes; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; -import java.io.PipedInputStream; -import java.io.PipedOutputStream; +import java.nio.file.Files; import java.util.Collection; +import java.util.UUID; import java.util.function.Function; import java.util.stream.Stream; @@ -38,8 +41,6 @@ import org.apache.james.vault.DeletedMessage; import org.apache.james.vault.DeletedMessageVault; import org.apache.james.vault.DeletedMessageZipper; import org.apache.james.vault.search.Query; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import com.github.fge.lambdas.Throwing; import com.github.fge.lambdas.functions.ThrowingFunction; @@ -52,7 +53,17 @@ import reactor.core.scheduler.Schedulers; class ExportService { - private static final Logger LOGGER = LoggerFactory.getLogger(ExportService.class); + private class ZippedData { + private final long contentLength; + private final InputStream inputStream; + + private ZippedData(long contentLength, InputStream content) { + this.contentLength = contentLength; + this.inputStream = content; + } + } + + private static final String TEMPORARY_FILE_EXTENSION = ".temp"; private final BlobExportMechanism blobExport; private final BlobStore blobStore; @@ -75,8 +86,8 @@ class ExportService { .doOnNext(any -> messageToExportCallback.run()) .collect(Guavate.toImmutableList()) .map(Collection::stream) - .map(sneakyThrow(messages -> zipData(user, messages))) - .flatMap(sneakyThrow(zippedStream -> blobStore.save(zippedStream, zippedStream.available()))) + .flatMap(messages -> Mono.fromCallable(() -> zipData(user, messages))) + .flatMap(sneakyThrow(zippedData -> blobStore.save(zippedData.inputStream, zippedData.contentLength))) .flatMap(blobId -> exportTo(user, exportToAddress, blobId)) .then(); } @@ -86,27 +97,19 @@ class ExportService { .publishOn(Schedulers.elastic()); } - private PipedInputStream zipData(User user, Stream<DeletedMessage> messages) throws IOException { - PipedOutputStream outputStream = new PipedOutputStream(); - PipedInputStream inputStream = new PipedInputStream(); - inputStream.connect(outputStream); + private ZippedData zipData(User user, Stream<DeletedMessage> messages) throws IOException { + File tempFile = temporaryFile(); + FileOutputStream fileOutputStream = new FileOutputStream(tempFile); - asyncZipData(user, messages, outputStream).subscribe(); - - return inputStream; + zipper.zip(message -> loadMessageContent(user, message), messages, fileOutputStream); + return new ZippedData(tempFile.length(), new FileInputStream(tempFile)); } - private Mono<Void> asyncZipData(User user, Stream<DeletedMessage> messages, PipedOutputStream outputStream) { - return Mono.fromRunnable(Throwing.runnable(() -> zipper.zip(message -> loadMessageContent(user, message), messages, outputStream)).sneakyThrow()) - .doOnSuccessOrError(Throwing.biConsumer((result, throwable) -> { - if (throwable != null) { - LOGGER.error("Error happens when zipping deleted messages", throwable); - } - outputStream.flush(); - outputStream.close(); - })) - .subscribeOn(Schedulers.elastic()) - .then(); + private File temporaryFile() throws IOException { + String tempFileName = UUID.randomUUID().toString(); + File tempFile = Files.createTempFile(tempFileName, TEMPORARY_FILE_EXTENSION).toFile(); + tempFile.deleteOnExit(); + return tempFile; } private InputStream loadMessageContent(User user, DeletedMessage message) { --------------------------------------------------------------------- To unsubscribe, e-mail: server-dev-unsubscr...@james.apache.org For additional commands, e-mail: server-dev-h...@james.apache.org