This is an automated email from the ASF dual-hosted git repository. btellier pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/james-project.git
commit 9d8b47050eafc39f489d10ccc6dca805a5a5b3ef Author: Tran Tien Duc <dt...@linagora.com> AuthorDate: Thu Mar 14 17:10:36 2019 +0700 JAMES-2685 DMV Route export API --- .../webadmin-mailbox-deleted-message-vault/pom.xml | 27 + .../routes/DeletedMessagesVaultExportTask.java | 100 +++ .../vault/routes/DeletedMessagesVaultRoutes.java | 99 ++- .../james/webadmin/vault/routes/ExportService.java | 128 ++++ .../routes/DeletedMessagesVaultRoutesTest.java | 759 ++++++++++++++------- 5 files changed, 841 insertions(+), 272 deletions(-) diff --git a/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/pom.xml b/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/pom.xml index 2c06be8..50adb89 100644 --- a/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/pom.xml +++ b/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/pom.xml @@ -71,12 +71,34 @@ </dependency> <dependency> <groupId>${james.groupId}</groupId> + <artifactId>blob-api</artifactId> + </dependency> + <dependency> + <groupId>${james.groupId}</groupId> + <artifactId>blob-memory</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>${james.groupId}</groupId> + <artifactId>blob-export-api</artifactId> + </dependency> + <dependency> + <groupId>${james.groupId}</groupId> <artifactId>james-core</artifactId> <type>test-jar</type> <scope>test</scope> </dependency> <dependency> <groupId>${james.groupId}</groupId> + <artifactId>james-server-data-api</artifactId> + </dependency> + <dependency> + <groupId>${james.groupId}</groupId> + <artifactId>james-server-data-memory</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>${james.groupId}</groupId> <artifactId>james-server-webadmin-core</artifactId> </dependency> <dependency> @@ -130,6 +152,11 @@ <scope>test</scope> </dependency> <dependency> + <groupId>org.junit.jupiter</groupId> + <artifactId>junit-jupiter-params</artifactId> + <scope>test</scope> + </dependency> + <dependency> <groupId>org.hamcrest</groupId> <artifactId>java-hamcrest</artifactId> <scope>test</scope> diff --git a/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/src/main/java/org/apache/james/webadmin/vault/routes/DeletedMessagesVaultExportTask.java b/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/src/main/java/org/apache/james/webadmin/vault/routes/DeletedMessagesVaultExportTask.java new file mode 100644 index 0000000..de4204b --- /dev/null +++ b/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/src/main/java/org/apache/james/webadmin/vault/routes/DeletedMessagesVaultExportTask.java @@ -0,0 +1,100 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ + +package org.apache.james.webadmin.vault.routes; + +import java.util.Optional; +import java.util.concurrent.atomic.AtomicLong; + +import org.apache.james.core.MailAddress; +import org.apache.james.core.User; +import org.apache.james.task.Task; +import org.apache.james.task.TaskExecutionDetails; +import org.apache.james.vault.search.Query; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +class DeletedMessagesVaultExportTask implements Task { + + public class AdditionalInformation implements TaskExecutionDetails.AdditionalInformation { + + private final User userExportFrom; + private final MailAddress exportTo; + private final long totalExportedMessages; + + public AdditionalInformation(User userExportFrom, MailAddress exportTo, long totalExportedMessages) { + this.userExportFrom = userExportFrom; + this.exportTo = exportTo; + this.totalExportedMessages = totalExportedMessages; + } + + public String getUserExportFrom() { + return userExportFrom.asString(); + } + + public String getExportTo() { + return exportTo.asString(); + } + + public long getTotalExportedMessages() { + return totalExportedMessages; + } + } + + private static final Logger LOGGER = LoggerFactory.getLogger(DeletedMessagesVaultExportTask.class); + + static final String TYPE = "deletedMessages/export"; + + private final ExportService exportService; + private final User userExportFrom; + private final Query exportQuery; + private final MailAddress exportTo; + private final AtomicLong totalExportedMessages; + + DeletedMessagesVaultExportTask(ExportService exportService, User userExportFrom, Query exportQuery, MailAddress exportTo) { + this.exportService = exportService; + this.userExportFrom = userExportFrom; + this.exportQuery = exportQuery; + this.exportTo = exportTo; + this.totalExportedMessages = new AtomicLong(); + } + + @Override + public Result run() { + try { + Runnable messageToShareCallback = totalExportedMessages::incrementAndGet; + exportService.export(userExportFrom, exportQuery, exportTo, messageToShareCallback) + .block(); + return Result.COMPLETED; + } catch (Exception e) { + LOGGER.error("Error happens when exporting deleted messages from {} to {}", userExportFrom.asString(), exportTo.asString()); + return Result.PARTIAL; + } + } + + @Override + public String type() { + return TYPE; + } + + @Override + public Optional<TaskExecutionDetails.AdditionalInformation> details() { + return Optional.of(new AdditionalInformation(userExportFrom, exportTo, totalExportedMessages.get())); + } +} diff --git a/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/src/main/java/org/apache/james/webadmin/vault/routes/DeletedMessagesVaultRoutes.java b/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/src/main/java/org/apache/james/webadmin/vault/routes/DeletedMessagesVaultRoutes.java index 527b8d9..ba8864c 100644 --- a/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/src/main/java/org/apache/james/webadmin/vault/routes/DeletedMessagesVaultRoutes.java +++ b/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/src/main/java/org/apache/james/webadmin/vault/routes/DeletedMessagesVaultRoutes.java @@ -26,16 +26,20 @@ import java.util.Optional; import java.util.stream.Stream; import javax.inject.Inject; +import javax.mail.internet.AddressException; import javax.ws.rs.POST; import javax.ws.rs.Path; import javax.ws.rs.Produces; import org.apache.commons.lang3.NotImplementedException; import org.apache.commons.lang3.StringUtils; +import org.apache.james.core.MailAddress; import org.apache.james.core.User; import org.apache.james.task.Task; import org.apache.james.task.TaskId; import org.apache.james.task.TaskManager; +import org.apache.james.user.api.UsersRepository; +import org.apache.james.user.api.UsersRepositoryException; import org.apache.james.vault.search.Query; import org.apache.james.webadmin.Constants; import org.apache.james.webadmin.Routes; @@ -70,7 +74,8 @@ import spark.Service; public class DeletedMessagesVaultRoutes implements Routes { enum UserVaultAction { - RESTORE("restore"); + RESTORE("restore"), + EXPORT("export"); static Optional<UserVaultAction> getAction(String value) { Preconditions.checkNotNull(value, "action cannot be null"); @@ -102,21 +107,26 @@ public class DeletedMessagesVaultRoutes implements Routes { private static final String USER_PATH_PARAM = "user"; private static final String RESTORE_PATH = ROOT_PATH + SEPARATOR + ":" + USER_PATH_PARAM; private static final String ACTION_QUERY_PARAM = "action"; + private static final String EXPORT_TO_QUERY_PARAM = "exportTo"; private final RestoreService vaultRestore; + private final ExportService vaultExport; private final JsonTransformer jsonTransformer; private final TaskManager taskManager; private final JsonExtractor<QueryElement> jsonExtractor; private final QueryTranslator queryTranslator; + private final UsersRepository usersRepository; @Inject @VisibleForTesting - DeletedMessagesVaultRoutes(RestoreService vaultRestore, JsonTransformer jsonTransformer, - TaskManager taskManager, QueryTranslator queryTranslator) { + DeletedMessagesVaultRoutes(RestoreService vaultRestore, ExportService vaultExport, JsonTransformer jsonTransformer, + TaskManager taskManager, QueryTranslator queryTranslator, UsersRepository usersRepository) { this.vaultRestore = vaultRestore; + this.vaultExport = vaultExport; this.jsonTransformer = jsonTransformer; this.taskManager = taskManager; this.queryTranslator = queryTranslator; + this.usersRepository = usersRepository; this.jsonExtractor = new JsonExtractor<>(QueryElement.class); } @@ -149,11 +159,19 @@ public class DeletedMessagesVaultRoutes implements Routes { paramType = "query", example = "?action=restore", value = "Compulsory. Needs to be a valid action represent for an operation to perform on the Deleted Message Vault, " + - "valid action should be in the list (restore)") + "valid action should be in the list (restore, export)"), + @ApiImplicitParam( + dataType = "String", + name = "exportTo", + paramType = "query", + example = "?exportTo=u...@james.org", + value = "Compulsory if action is export. Needs to be a valid mail address to represent for the destination " + + "where deleted messages content is export to") }) @ApiResponses(value = { @ApiResponse(code = HttpStatus.CREATED_201, message = "Task is created", response = TaskIdDto.class), @ApiResponse(code = HttpStatus.BAD_REQUEST_400, message = "Bad request - user param is invalid"), + @ApiResponse(code = HttpStatus.NOT_FOUND_404, message = "Not found - requested user is not existed in the system"), @ApiResponse(code = HttpStatus.INTERNAL_SERVER_ERROR_500, message = "Internal server error - Something went bad on the server side.") }) private TaskIdDto userActions(Request request, Response response) throws JsonExtractException { @@ -165,22 +183,73 @@ public class DeletedMessagesVaultRoutes implements Routes { } private Task generateTask(UserVaultAction requestedAction, Request request) throws JsonExtractException { - User userToRestore = extractUser(request); + User user = extractUser(request); + validateUserExist(user); Query query = translate(jsonExtractor.parse(request.body())); switch (requestedAction) { case RESTORE: - return new DeletedMessagesVaultRestoreTask(vaultRestore, userToRestore, query); + return new DeletedMessagesVaultRestoreTask(vaultRestore, user, query); + case EXPORT: + return new DeletedMessagesVaultExportTask(vaultExport, user, query, extractMailAddress(request)); default: throw new NotImplementedException(requestedAction + " is not yet handled."); } } + private void validateUserExist(User user) { + try { + if (!usersRepository.contains(user.asString())) { + throw ErrorResponder.builder() + .statusCode(HttpStatus.NOT_FOUND_404) + .type(ErrorResponder.ErrorType.NOT_FOUND) + .message("User '" + user.asString() + "' does not exist in the system") + .haltError(); + } + } catch (UsersRepositoryException e) { + throw ErrorResponder.builder() + .statusCode(HttpStatus.INTERNAL_SERVER_ERROR_500) + .type(ErrorResponder.ErrorType.SERVER_ERROR) + .message("Unable to validate 'user' parameter") + .cause(e) + .haltError(); + } + } + + private MailAddress extractMailAddress(Request request) { + return Optional.ofNullable(request.queryParams(EXPORT_TO_QUERY_PARAM)) + .filter(StringUtils::isNotBlank) + .map(this::parseToMailAddress) + .orElseThrow(() -> ErrorResponder.builder() + .statusCode(HttpStatus.BAD_REQUEST_400) + .type(ErrorResponder.ErrorType.INVALID_ARGUMENT) + .message("Invalid 'exportTo' parameter, null or blank value is not accepted") + .haltError()); + } + + private MailAddress parseToMailAddress(String addressString) { + try { + return new MailAddress(addressString); + } catch (AddressException e) { + throw ErrorResponder.builder() + .statusCode(HttpStatus.BAD_REQUEST_400) + .type(ErrorResponder.ErrorType.INVALID_ARGUMENT) + .message("Invalid 'exportTo' parameter") + .cause(e) + .haltError(); + } + } + private Query translate(QueryElement queryElement) { try { return queryTranslator.translate(queryElement); } catch (QueryTranslator.QueryTranslatorException e) { - throw badRequest("Invalid payload passing to the route", e); + throw ErrorResponder.builder() + .statusCode(HttpStatus.BAD_REQUEST_400) + .type(ErrorResponder.ErrorType.INVALID_ARGUMENT) + .message("Invalid payload passing to the route") + .cause(e) + .haltError(); } } @@ -188,19 +257,15 @@ public class DeletedMessagesVaultRoutes implements Routes { try { return User.fromUsername(request.params(USER_PATH_PARAM)); } catch (IllegalArgumentException e) { - throw badRequest("Invalid 'user' parameter", e); + throw ErrorResponder.builder() + .statusCode(HttpStatus.BAD_REQUEST_400) + .type(ErrorResponder.ErrorType.INVALID_ARGUMENT) + .message("Invalid 'user' parameter") + .cause(e) + .haltError(); } } - private HaltException badRequest(String message, Exception e) { - return ErrorResponder.builder() - .statusCode(HttpStatus.BAD_REQUEST_400) - .type(ErrorResponder.ErrorType.INVALID_ARGUMENT) - .message(message) - .cause(e) - .haltError(); - } - private UserVaultAction extractUserVaultAction(Request request) { String actionParam = request.queryParams(ACTION_QUERY_PARAM); return Optional.ofNullable(actionParam) diff --git a/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/src/main/java/org/apache/james/webadmin/vault/routes/ExportService.java b/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/src/main/java/org/apache/james/webadmin/vault/routes/ExportService.java new file mode 100644 index 0000000..bab02b8 --- /dev/null +++ b/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/src/main/java/org/apache/james/webadmin/vault/routes/ExportService.java @@ -0,0 +1,128 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ + +package org.apache.james.webadmin.vault.routes; + +import java.io.IOException; +import java.io.InputStream; +import java.io.PipedInputStream; +import java.io.PipedOutputStream; +import java.util.Collection; +import java.util.function.Function; +import java.util.stream.Stream; + +import javax.inject.Inject; + +import org.apache.james.blob.api.BlobId; +import org.apache.james.blob.api.BlobStore; +import org.apache.james.blob.export.api.BlobExportMechanism; +import org.apache.james.core.MailAddress; +import org.apache.james.core.User; +import org.apache.james.vault.DeletedMessage; +import org.apache.james.vault.DeletedMessageVault; +import org.apache.james.vault.DeletedMessageZipper; +import org.apache.james.vault.search.Query; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.github.fge.lambdas.Throwing; +import com.github.fge.lambdas.functions.ThrowingFunction; +import com.github.steveash.guavate.Guavate; +import com.google.common.annotations.VisibleForTesting; + +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; +import reactor.core.scheduler.Schedulers; + +class ExportService { + + private static final Logger LOGGER = LoggerFactory.getLogger(ExportService.class); + + private final BlobExportMechanism blobExport; + private final BlobStore blobStore; + private final DeletedMessageZipper zipper; + private final DeletedMessageVault vault; + + @Inject + @VisibleForTesting + ExportService(BlobExportMechanism blobExport, BlobStore blobStore, DeletedMessageZipper zipper, DeletedMessageVault vault) { + this.blobExport = blobExport; + this.blobStore = blobStore; + this.zipper = zipper; + this.vault = vault; + } + + Mono<Void> export(User user, Query exportQuery, MailAddress exportToAddress, + Runnable messageToExportCallback) { + + return matchingMessages(user, exportQuery) + .doOnNext(any -> messageToExportCallback.run()) + .collect(Guavate.toImmutableList()) + .map(Collection::stream) + .map(sneakyThrow(messages -> zipData(user, messages))) + .flatMap(sneakyThrow(zippedStream -> blobStore.save(zippedStream, zippedStream.available()))) + .flatMap(blobId -> exportTo(user, exportToAddress, blobId)) + .then(); + } + + private Flux<DeletedMessage> matchingMessages(User user, Query exportQuery) { + return Flux.from(vault.search(user, exportQuery)) + .publishOn(Schedulers.elastic()); + } + + private PipedInputStream zipData(User user, Stream<DeletedMessage> messages) throws IOException { + PipedOutputStream outputStream = new PipedOutputStream(); + PipedInputStream inputStream = new PipedInputStream(); + inputStream.connect(outputStream); + + asyncZipData(user, messages, outputStream).subscribe(); + + return inputStream; + } + + private Mono<Void> asyncZipData(User user, Stream<DeletedMessage> messages, PipedOutputStream outputStream) { + return Mono.fromRunnable(Throwing.runnable(() -> zipper.zip(message -> loadMessageContent(user, message), messages, outputStream)).sneakyThrow()) + .doOnSuccessOrError(Throwing.biConsumer((result, throwable) -> { + if (throwable != null) { + LOGGER.error("Error happens when zipping deleted messages", throwable); + } + outputStream.flush(); + outputStream.close(); + })) + .subscribeOn(Schedulers.elastic()) + .then(); + } + + private InputStream loadMessageContent(User user, DeletedMessage message) { + return Mono.from(vault.loadMimeMessage(user, message.getMessageId())) + .block(); + } + + private Mono<Void> exportTo(User user, MailAddress exportToAddress, BlobId blobId) { + return Mono.fromRunnable(() -> blobExport + .blobId(blobId) + .with(exportToAddress) + .explanation(String.format("Some deleted messages from user %s has been shared to you", user.asString())) + .export()); + } + + private <T, R> Function<T, R> sneakyThrow(ThrowingFunction<T, R> function) { + return Throwing.function(function).sneakyThrow(); + } +} diff --git a/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/src/test/java/org/apache/james/webadmin/vault/routes/DeletedMessagesVaultRoutesTest.java b/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/src/test/java/org/apache/james/webadmin/vault/routes/DeletedMessagesVaultRoutesTest.java index 3ed17dd..fcc563f 100644 --- a/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/src/test/java/org/apache/james/webadmin/vault/routes/DeletedMessagesVaultRoutesTest.java +++ b/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/src/test/java/org/apache/james/webadmin/vault/routes/DeletedMessagesVaultRoutesTest.java @@ -21,6 +21,7 @@ package org.apache.james.webadmin.vault.routes; import static io.restassured.RestAssured.given; import static io.restassured.RestAssured.when; +import static io.restassured.RestAssured.with; import static org.apache.james.vault.DeletedMessageFixture.CONTENT; import static org.apache.james.vault.DeletedMessageFixture.DELETED_MESSAGE; import static org.apache.james.vault.DeletedMessageFixture.DELETED_MESSAGE_2; @@ -44,17 +45,33 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; import java.io.InputStream; import java.util.List; import java.util.stream.Stream; +import org.apache.commons.configuration.DefaultConfigurationBuilder; +import org.apache.james.blob.api.BlobId; +import org.apache.james.blob.api.HashBlobId; +import org.apache.james.blob.export.api.BlobExportMechanism; +import org.apache.james.blob.memory.MemoryBlobStore; +import org.apache.james.core.Domain; +import org.apache.james.core.MailAddress; import org.apache.james.core.MaybeSender; import org.apache.james.core.User; +import org.apache.james.dnsservice.api.DNSService; +import org.apache.james.domainlist.lib.DomainListConfiguration; +import org.apache.james.domainlist.memory.MemoryDomainList; import org.apache.james.mailbox.MailboxSession; import org.apache.james.mailbox.MessageManager; import org.apache.james.mailbox.exception.MailboxException; @@ -71,7 +88,9 @@ import org.apache.james.mailbox.model.MultimailboxesSearchQuery; import org.apache.james.mailbox.model.SearchQuery; import org.apache.james.metrics.logger.DefaultMetricFactory; import org.apache.james.task.MemoryTaskManager; +import org.apache.james.user.memory.MemoryUsersRepository; import org.apache.james.vault.DeletedMessage; +import org.apache.james.vault.DeletedMessageZipper; import org.apache.james.vault.memory.MemoryDeletedMessagesVault; import org.apache.james.vault.search.Query; import org.apache.james.webadmin.WebAdminServer; @@ -85,7 +104,8 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; -import org.mockito.Mockito; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import com.google.common.collect.ImmutableList; @@ -96,15 +116,33 @@ import reactor.core.publisher.Mono; class DeletedMessagesVaultRoutesTest { + private class NoopBlobExporting implements BlobExportMechanism { + @Override + public ShareeStage blobId(BlobId blobId) { + return exportTo -> explanation -> () -> export(exportTo, explanation); + } + + void export(MailAddress exportTo, String explanation) { + // do nothing + } + } + private static final String MATCH_ALL_QUERY = "{" + "\"combinator\": \"and\"," + "\"criteria\": []" + "}"; + private static final Domain DOMAIN = Domain.of("apache.org"); private WebAdminServer webAdminServer; private MemoryDeletedMessagesVault vault; private InMemoryMailboxManager mailboxManager; private MemoryTaskManager taskManager; + private NoopBlobExporting blobExporting; + private MemoryBlobStore blobStore; + private DeletedMessageZipper zipper; + private MemoryUsersRepository usersRepository; + private ExportService exportService; + private HashBlobId.Factory blobIdFactory; @BeforeEach void beforeEach() throws Exception { @@ -116,11 +154,17 @@ class DeletedMessagesVaultRoutesTest { JsonTransformer jsonTransformer = new JsonTransformer(); RestoreService vaultRestore = new RestoreService(vault, mailboxManager); + blobExporting = spy(new NoopBlobExporting()); + blobIdFactory = new HashBlobId.Factory(); + blobStore = new MemoryBlobStore(blobIdFactory); + zipper = new DeletedMessageZipper(); + exportService = new ExportService(blobExporting, blobStore, zipper, vault); QueryTranslator queryTranslator = new QueryTranslator(new InMemoryId.Factory()); + usersRepository = createUsersRepository(); webAdminServer = WebAdminUtils.createWebAdminServer( new DefaultMetricFactory(), new TasksRoutes(taskManager, jsonTransformer), - new DeletedMessagesVaultRoutes(vaultRestore, jsonTransformer, taskManager, queryTranslator)); + new DeletedMessagesVaultRoutes(vaultRestore, exportService, jsonTransformer, taskManager, queryTranslator, usersRepository)); webAdminServer.configure(NO_CONFIGURATION); webAdminServer.await(); @@ -130,6 +174,23 @@ class DeletedMessagesVaultRoutesTest { .build(); } + private MemoryUsersRepository createUsersRepository() throws Exception { + DNSService dnsService = mock(DNSService.class); + MemoryDomainList domainList = new MemoryDomainList(dnsService); + domainList.configure(DomainListConfiguration.builder() + .autoDetect(false) + .autoDetectIp(false)); + domainList.addDomain(DOMAIN); + + MemoryUsersRepository usersRepository = MemoryUsersRepository.withVirtualHosting(); + usersRepository.setDomainList(domainList); + usersRepository.configure(new DefaultConfigurationBuilder()); + + usersRepository.addUser(USER.asString(), "userPassword"); + + return usersRepository; + } + @AfterEach void afterEach() { webAdminServer.destroy(); @@ -137,6 +198,286 @@ class DeletedMessagesVaultRoutesTest { } @Nested + class UserVaultActionsValidationTest { + + @Test + void userVaultAPIShouldReturnInvalidWhenActionIsMissing() { + when() + .post(USER.asString()) + .then() + .statusCode(HttpStatus.BAD_REQUEST_400) + .body("statusCode", is(400)) + .body("type", is(ErrorResponder.ErrorType.INVALID_ARGUMENT.getType())) + .body("message", is(notNullValue())) + .body("details", is(notNullValue())); + } + + @Test + void userVaultAPIShouldReturnInvalidWhenPassingEmptyAction() { + given() + .queryParam("action", "") + .when() + .post(USER.asString()) + .then() + .statusCode(HttpStatus.BAD_REQUEST_400) + .body("statusCode", is(400)) + .body("type", is(ErrorResponder.ErrorType.INVALID_ARGUMENT.getType())) + .body("message", is(notNullValue())) + .body("details", is(notNullValue())); + } + + @Test + void userVaultAPIShouldReturnInvalidWhenActionIsInValid() { + given() + .queryParam("action", "invalid action") + .when() + .post(USER.asString()) + .then() + .statusCode(HttpStatus.BAD_REQUEST_400) + .body("statusCode", is(400)) + .body("type", is(ErrorResponder.ErrorType.INVALID_ARGUMENT.getType())) + .body("message", is(notNullValue())) + .body("details", is(notNullValue())); + } + + @Test + void userVaultAPIShouldReturnInvalidWhenPassingCaseInsensitiveAction() { + given() + .queryParam("action", "RESTORE") + .when() + .post(USER.asString()) + .then() + .statusCode(HttpStatus.BAD_REQUEST_400) + .body("statusCode", is(400)) + .body("type", is(ErrorResponder.ErrorType.INVALID_ARGUMENT.getType())) + .body("message", is(notNullValue())) + .body("details", is(notNullValue())); + } + + @ParameterizedTest + @ValueSource(strings = {"restore", "export"}) + void userVaultAPIShouldReturnInvalidWhenUserIsInvalid(String action) { + given() + .queryParam("action", action) + .when() + .post("not@va...@user.com") + .then() + .statusCode(HttpStatus.BAD_REQUEST_400) + .body("statusCode", is(400)) + .body("type", is(ErrorResponder.ErrorType.INVALID_ARGUMENT.getType())) + .body("message", is(notNullValue())) + .body("details", is(notNullValue())); + } + + @ParameterizedTest + @ValueSource(strings = {"restore", "export"}) + void userVaultAPIShouldReturnNotFoundWhenUserIsNotFoundInSystem(String action) { + given() + .queryParam("action", action) + .when() + .post(USER_2.asString()) + .then() + .statusCode(HttpStatus.NOT_FOUND_404) + .body("statusCode", is(404)) + .body("type", is(ErrorResponder.ErrorType.NOT_FOUND.getType())) + .body("message", is(notNullValue())); + } + + @ParameterizedTest + @ValueSource(strings = {"restore", "export"}) + void userVaultAPIShouldReturnNotFoundWhenNoUserPathParameter(String action) { + given() + .queryParam("action", action) + .when() + .post() + .then() + .statusCode(HttpStatus.NOT_FOUND_404) + .body("statusCode", is(404)) + .body("type", is(notNullValue())) + .body("message", is(notNullValue())); + } + + @ParameterizedTest + @ValueSource(strings = {"restore", "export"}) + void userVaultAPIShouldReturnBadRequestWhenPassingUnsupportedField(String action) throws Exception { + vault.append(USER, DELETED_MESSAGE, new ByteArrayInputStream(CONTENT)).block(); + + String query = + "{" + + " \"criteria\": [" + + " {" + + " \"fieldName\": \"unsupported\"," + + " \"operator\": \"contains\"," + + " \"value\": \"" + MAILBOX_ID_1.serialize() + "\"" + + " }" + + " ]" + + "}"; + + given() + .queryParam("action", action) + .body(query) + .when() + .post(USER.asString()) + .then() + .statusCode(HttpStatus.BAD_REQUEST_400) + .body("statusCode", is(400)) + .body("type", is(ErrorResponder.ErrorType.INVALID_ARGUMENT.getType())) + .body("message", is(notNullValue())) + .body("details", is(notNullValue())); + } + + @ParameterizedTest + @ValueSource(strings = {"restore", "export"}) + void userVaultAPIShouldReturnBadRequestWhenPassingUnsupportedOperator(String action) throws Exception { + vault.append(USER, DELETED_MESSAGE, new ByteArrayInputStream(CONTENT)).block(); + + String query = + "{" + + " \"criteria\": [" + + " {" + + " \"fieldName\": \"subject\"," + + " \"operator\": \"isLongerThan\"," + + " \"value\": \"" + SUBJECT + "\"" + + " }" + + " ]" + + "}"; + + given() + .queryParam("action", action) + .body(query) + .when() + .post(USER.asString()) + .then() + .statusCode(HttpStatus.BAD_REQUEST_400) + .body("statusCode", is(400)) + .body("type", is(ErrorResponder.ErrorType.INVALID_ARGUMENT.getType())) + .body("message", is(notNullValue())) + .body("details", is(notNullValue())); + } + + @ParameterizedTest + @ValueSource(strings = {"restore", "export"}) + void userVaultAPIShouldReturnBadRequestWhenPassingUnsupportedPairOfFieldNameAndOperator(String action) throws Exception { + vault.append(USER, DELETED_MESSAGE, new ByteArrayInputStream(CONTENT)).block(); + + String query = + "{" + + " \"criteria\": [" + + " {" + + " \"fieldName\": \"sender\"," + + " \"operator\": \"contains\"," + + " \"value\": \"" + SENDER.asString() + "\"" + + " }" + + " ]" + + "}"; + + given() + .queryParam("action", action) + .body(query) + .when() + .post(USER.asString()) + .then() + .statusCode(HttpStatus.BAD_REQUEST_400) + .body("statusCode", is(400)) + .body("type", is(ErrorResponder.ErrorType.INVALID_ARGUMENT.getType())) + .body("message", is(notNullValue())) + .body("details", is(notNullValue())); + } + + @ParameterizedTest + @ValueSource(strings = {"restore", "export"}) + void userVaultAPIShouldReturnBadRequestWhenPassingInvalidMailAddress(String action) throws Exception { + vault.append(USER, DELETED_MESSAGE, new ByteArrayInputStream(CONTENT)).block(); + + String query = + "{" + + " \"criteria\": [" + + " {" + + " \"fieldName\": \"sender\"," + + " \"operator\": \"contains\"," + + " \"value\": \"invalid@m...@domain.tld\"" + + " }" + + " ]" + + "}"; + + given() + .queryParam("action", action) + .body(query) + .when() + .post(USER.asString()) + .then() + .statusCode(HttpStatus.BAD_REQUEST_400) + .body("statusCode", is(400)) + .body("type", is(ErrorResponder.ErrorType.INVALID_ARGUMENT.getType())) + .body("message", is(notNullValue())) + .body("details", is(notNullValue())); + } + + @ParameterizedTest + @ValueSource(strings = {"restore", "export"}) + void userVaultAPIShouldReturnBadRequestWhenPassingOrCombinator(String action) throws Exception { + vault.append(USER, DELETED_MESSAGE, new ByteArrayInputStream(CONTENT)).block(); + + String query = + "{" + + " \"combinator\": \"or\"," + + " \"criteria\": [" + + " {" + + " \"fieldName\": \"sender\"," + + " \"operator\": \"contains\"," + + " \"value\": \"" + SENDER.asString() + "\"" + + " }" + + " ]" + + "}"; + + given() + .queryParam("action", action) + .body(query) + .when() + .post(USER.asString()) + .then() + .statusCode(HttpStatus.BAD_REQUEST_400) + .body("statusCode", is(400)) + .body("type", is(ErrorResponder.ErrorType.INVALID_ARGUMENT.getType())) + .body("message", is(notNullValue())) + .body("details", is(notNullValue())); + } + + @ParameterizedTest + @ValueSource(strings = {"restore", "export"}) + void userVaultAPIShouldReturnBadRequestWhenPassingNestedStructuredQuery(String action) throws Exception { + vault.append(USER, DELETED_MESSAGE, new ByteArrayInputStream(CONTENT)).block(); + + String query = + "{" + + " \"combinator\": \"and\"," + + " \"criteria\": [" + + " {" + + " \"combinator\": \"or\"," + + " \"criteria\": [" + + " {\"fieldName\": \"subject\", \"operator\": \"containsIgnoreCase\", \"value\": \"Apache James\"}," + + " {\"fieldName\": \"subject\", \"operator\": \"containsIgnoreCase\", \"value\": \"Apache James\"}" + + " ]" + + " }," + + " {\"fieldName\": \"subject\", \"operator\": \"containsIgnoreCase\", \"value\": \"Apache James\"}" + + " ]" + + "}"; + + given() + .queryParam("action", action) + .body(query) + .when() + .post(USER.asString()) + .then() + .statusCode(HttpStatus.BAD_REQUEST_400) + .body("statusCode", is(400)) + .body("type", is(ErrorResponder.ErrorType.INVALID_ARGUMENT.getType())) + .body("message", is(notNullValue())) + .body("details", is(notNullValue())); + } + } + + @Nested class RestoreTest { @Nested @@ -1083,258 +1424,6 @@ class DeletedMessagesVaultRoutesTest { } @Nested - class ValidationTest { - - @Test - void restoreShouldReturnInvalidWhenActionIsMissing() { - when() - .post(USER.asString()) - .then() - .statusCode(HttpStatus.BAD_REQUEST_400) - .body("statusCode", is(400)) - .body("type", is(ErrorResponder.ErrorType.INVALID_ARGUMENT.getType())) - .body("message", is(notNullValue())) - .body("details", is(notNullValue())); - } - - @Test - void restoreShouldReturnInvalidWhenPassingEmptyAction() { - given() - .queryParam("action", "") - .when() - .post(USER.asString()) - .then() - .statusCode(HttpStatus.BAD_REQUEST_400) - .body("statusCode", is(400)) - .body("type", is(ErrorResponder.ErrorType.INVALID_ARGUMENT.getType())) - .body("message", is(notNullValue())) - .body("details", is(notNullValue())); - } - - @Test - void restoreShouldReturnInvalidWhenActionIsInValid() { - given() - .queryParam("action", "invalid action") - .when() - .post(USER.asString()) - .then() - .statusCode(HttpStatus.BAD_REQUEST_400) - .body("statusCode", is(400)) - .body("type", is(ErrorResponder.ErrorType.INVALID_ARGUMENT.getType())) - .body("message", is(notNullValue())) - .body("details", is(notNullValue())); - } - - @Test - void restoreShouldReturnInvalidWhenPassingCaseInsensitiveAction() { - given() - .queryParam("action", "RESTORE") - .when() - .post(USER.asString()) - .then() - .statusCode(HttpStatus.BAD_REQUEST_400) - .body("statusCode", is(400)) - .body("type", is(ErrorResponder.ErrorType.INVALID_ARGUMENT.getType())) - .body("message", is(notNullValue())) - .body("details", is(notNullValue())); - } - - @Test - void restoreShouldReturnInvalidWhenUserIsInvalid() { - given() - .queryParam("action", "restore") - .when() - .post("not@va...@user.com") - .then() - .statusCode(HttpStatus.BAD_REQUEST_400) - .body("statusCode", is(400)) - .body("type", is(ErrorResponder.ErrorType.INVALID_ARGUMENT.getType())) - .body("message", is(notNullValue())) - .body("details", is(notNullValue())); - } - - @Test - void postShouldReturnNotFoundWhenNoUserPathParameter() { - given() - .queryParam("action", "restore") - .when() - .post() - .then() - .statusCode(HttpStatus.NOT_FOUND_404) - .body("statusCode", is(404)) - .body("type", is(notNullValue())) - .body("message", is(notNullValue())); - } - - @Test - void restoreShouldReturnBadRequestWhenPassingUnsupportedField() throws Exception { - vault.append(USER, DELETED_MESSAGE, new ByteArrayInputStream(CONTENT)).block(); - - String query = - "{" + - " \"criteria\": [" + - " {" + - " \"fieldName\": \"unsupported\"," + - " \"operator\": \"contains\"," + - " \"value\": \"" + MAILBOX_ID_1.serialize() + "\"" + - " }" + - " ]" + - "}"; - - given() - .body(query) - .when() - .post(USER.asString()) - .then() - .statusCode(HttpStatus.BAD_REQUEST_400) - .body("statusCode", is(400)) - .body("type", is(ErrorResponder.ErrorType.INVALID_ARGUMENT.getType())) - .body("message", is(notNullValue())) - .body("details", is(notNullValue())); - } - - @Test - void restoreShouldReturnBadRequestWhenPassingUnsupportedOperator() throws Exception { - vault.append(USER, DELETED_MESSAGE, new ByteArrayInputStream(CONTENT)).block(); - - String query = - "{" + - " \"criteria\": [" + - " {" + - " \"fieldName\": \"subject\"," + - " \"operator\": \"isLongerThan\"," + - " \"value\": \"" + SUBJECT + "\"" + - " }" + - " ]" + - "}"; - - given() - .body(query) - .when() - .post(USER.asString()) - .then() - .statusCode(HttpStatus.BAD_REQUEST_400) - .body("statusCode", is(400)) - .body("type", is(ErrorResponder.ErrorType.INVALID_ARGUMENT.getType())) - .body("message", is(notNullValue())) - .body("details", is(notNullValue())); - } - - @Test - void restoreShouldReturnBadRequestWhenPassingUnsupportedPairOfFieldNameAndOperator() throws Exception { - vault.append(USER, DELETED_MESSAGE, new ByteArrayInputStream(CONTENT)).block(); - - String query = - "{" + - " \"criteria\": [" + - " {" + - " \"fieldName\": \"sender\"," + - " \"operator\": \"contains\"," + - " \"value\": \"" + SENDER.asString() + "\"" + - " }" + - " ]" + - "}"; - - given() - .body(query) - .when() - .post(USER.asString()) - .then() - .statusCode(HttpStatus.BAD_REQUEST_400) - .body("statusCode", is(400)) - .body("type", is(ErrorResponder.ErrorType.INVALID_ARGUMENT.getType())) - .body("message", is(notNullValue())) - .body("details", is(notNullValue())); - } - - @Test - void restoreShouldReturnBadRequestWhenPassingInvalidMailAddress() throws Exception { - vault.append(USER, DELETED_MESSAGE, new ByteArrayInputStream(CONTENT)).block(); - - String query = - "{" + - " \"criteria\": [" + - " {" + - " \"fieldName\": \"sender\"," + - " \"operator\": \"contains\"," + - " \"value\": \"invalid@m...@domain.tld\"" + - " }" + - " ]" + - "}"; - - given() - .body(query) - .when() - .post(USER.asString()) - .then() - .statusCode(HttpStatus.BAD_REQUEST_400) - .body("statusCode", is(400)) - .body("type", is(ErrorResponder.ErrorType.INVALID_ARGUMENT.getType())) - .body("message", is(notNullValue())) - .body("details", is(notNullValue())); - } - - @Test - void restoreShouldReturnBadRequestWhenPassingOrCombinator() throws Exception { - vault.append(USER, DELETED_MESSAGE, new ByteArrayInputStream(CONTENT)).block(); - - String query = - "{" + - " \"combinator\": \"or\"," + - " \"criteria\": [" + - " {" + - " \"fieldName\": \"sender\"," + - " \"operator\": \"contains\"," + - " \"value\": \"" + SENDER.asString() + "\"" + - " }" + - " ]" + - "}"; - - given() - .body(query) - .when() - .post(USER.asString()) - .then() - .statusCode(HttpStatus.BAD_REQUEST_400) - .body("statusCode", is(400)) - .body("type", is(ErrorResponder.ErrorType.INVALID_ARGUMENT.getType())) - .body("message", is(notNullValue())) - .body("details", is(notNullValue())); - } - - @Test - void restoreShouldReturnBadRequestWhenPassingNestedStructuredQuery() throws Exception { - vault.append(USER, DELETED_MESSAGE, new ByteArrayInputStream(CONTENT)).block(); - - String query = - "{" + - " \"combinator\": \"and\"," + - " \"criteria\": [" + - " {" + - " \"combinator\": \"or\"," + - " \"criteria\": [" + - " {\"fieldName\": \"subject\", \"operator\": \"containsIgnoreCase\", \"value\": \"Apache James\"}," + - " {\"fieldName\": \"subject\", \"operator\": \"containsIgnoreCase\", \"value\": \"Apache James\"}" + - " ]" + - " }," + - " {\"fieldName\": \"subject\", \"operator\": \"containsIgnoreCase\", \"value\": \"Apache James\"}" + - " ]" + - "}"; - - given() - .body(query) - .when() - .post(USER.asString()) - .then() - .statusCode(HttpStatus.BAD_REQUEST_400) - .body("statusCode", is(400)) - .body("type", is(ErrorResponder.ErrorType.INVALID_ARGUMENT.getType())) - .body("message", is(notNullValue())) - .body("details", is(notNullValue())); - } - } - - @Nested class FailingRestoreTest { @Test @@ -1376,7 +1465,7 @@ class DeletedMessagesVaultRoutesTest { vault.append(USER, DELETED_MESSAGE, new ByteArrayInputStream(CONTENT)).block(); vault.append(USER, DELETED_MESSAGE_2, new ByteArrayInputStream(CONTENT)).block(); - MessageManager mockMessageManager = Mockito.mock(MessageManager.class); + MessageManager mockMessageManager = mock(MessageManager.class); doReturn(mockMessageManager) .when(mailboxManager) .getMailbox(any(MailboxId.class), any(MailboxSession.class)); @@ -1597,6 +1686,166 @@ class DeletedMessagesVaultRoutesTest { } + @Nested + class ExportTest { + + @Nested + class ValidationTest { + + @Test + void exportShouldReturnBadRequestWhenExportToIsMissing() { + given() + .queryParam("action", "export") + .body(MATCH_ALL_QUERY) + .when() + .post(USER.asString()) + .then() + .statusCode(HttpStatus.BAD_REQUEST_400) + .body("statusCode", is(400)) + .body("type", is(ErrorResponder.ErrorType.INVALID_ARGUMENT.getType())) + .body("message", is(notNullValue())); + } + + @Test + void exportShouldReturnBadRequestWhenExportToIsInvalid() { + given() + .queryParam("action", "export") + .queryParam("exportTo", "export@to#me@") + .body(MATCH_ALL_QUERY) + .when() + .post(USER.asString()) + .then() + .statusCode(HttpStatus.BAD_REQUEST_400) + .body("statusCode", is(400)) + .body("type", is(ErrorResponder.ErrorType.INVALID_ARGUMENT.getType())) + .body("message", is(notNullValue())) + .body("details", is(notNullValue())); + } + } + + @Nested + class TaskGeneratingTest { + + @Test + void exportShouldReturnATaskCreated() { + given() + .queryParam("action", "export") + .queryParam("exportTo", "expor...@james.org") + .body(MATCH_ALL_QUERY) + .when() + .post(USER.asString()) + .then() + .statusCode(HttpStatus.CREATED_201) + .body("taskId", notNullValue()); + } + + @Test + void exportShouldProduceASuccessfulTaskWithInformation() { + vault.append(USER, DELETED_MESSAGE, new ByteArrayInputStream(CONTENT)).block(); + vault.append(USER, DELETED_MESSAGE_2, new ByteArrayInputStream(CONTENT)).block(); + + String taskId = + with() + .queryParam("action", "export") + .queryParam("exportTo", USER_2.asString()) + .body(MATCH_ALL_QUERY) + .post(USER.asString()) + .jsonPath() + .get("taskId"); + + given() + .basePath(TasksRoutes.BASE) + .when() + .get(taskId + "/await") + .then() + .body("status", is("completed")) + .body("taskId", is(taskId)) + .body("type", is(DeletedMessagesVaultExportTask.TYPE)) + .body("additionalInformation.userExportFrom", is(USER.asString())) + .body("additionalInformation.exportTo", is(USER_2.asString())) + .body("additionalInformation.totalExportedMessages", is(2)) + .body("startedDate", is(notNullValue())) + .body("submitDate", is(notNullValue())) + .body("completedDate", is(notNullValue())); + } + } + + @Test + void exportShouldCallBlobExportingTargetToExportAddress() throws Exception { + vault.append(USER, DELETED_MESSAGE, new ByteArrayInputStream(CONTENT)).block(); + vault.append(USER, DELETED_MESSAGE_2, new ByteArrayInputStream(CONTENT)).block(); + + String taskId = + with() + .queryParam("action", "export") + .queryParam("exportTo", USER_2.asString()) + .body(MATCH_ALL_QUERY) + .post(USER.asString()) + .jsonPath() + .get("taskId"); + + with() + .basePath(TasksRoutes.BASE) + .get(taskId + "/await"); + + verify(blobExporting, times(1)) + .export(eq(USER_2.asMailAddress()), any()); + } + + @Test + void exportShouldNotDeleteMessagesInTheVault() { + vault.append(USER, DELETED_MESSAGE, new ByteArrayInputStream(CONTENT)).block(); + vault.append(USER, DELETED_MESSAGE_2, new ByteArrayInputStream(CONTENT)).block(); + + String taskId = + with() + .queryParam("action", "restore") + .body(MATCH_ALL_QUERY) + .post(USER.asString()) + .jsonPath() + .get("taskId"); + + with() + .basePath(TasksRoutes.BASE) + .get(taskId + "/await"); + + assertThat(Flux.from(vault.search(USER, Query.ALL)).toStream()) + .containsOnly(DELETED_MESSAGE, DELETED_MESSAGE_2); + } + + @Test + void exportShouldSaveDeletedMessagesDataToBlobStore() throws Exception { + vault.append(USER, DELETED_MESSAGE, new ByteArrayInputStream(CONTENT)).block(); + vault.append(USER, DELETED_MESSAGE_2, new ByteArrayInputStream(CONTENT)).block(); + + String taskId = + with() + .queryParam("action", "export") + .queryParam("exportTo", USER_2.asString()) + .body(MATCH_ALL_QUERY) + .post(USER.asString()) + .jsonPath() + .get("taskId"); + + with() + .basePath(TasksRoutes.BASE) + .get(taskId + "/await"); + + byte[] expectedZippedData = zippedMessagesData(); + + assertThat(blobStore.read(blobIdFactory.forPayload(expectedZippedData))) + .hasSameContentAs(new ByteArrayInputStream(expectedZippedData)); + } + + private byte[] zippedMessagesData() throws IOException { + ByteArrayOutputStream expectedZippedData = new ByteArrayOutputStream(); + zipper.zip(message -> new ByteArrayInputStream(CONTENT), + Stream.of(DELETED_MESSAGE, DELETED_MESSAGE_2), + expectedZippedData); + return expectedZippedData.toByteArray(); + } + } + private boolean hasAnyMail(User user) throws MailboxException { MailboxSession session = mailboxManager.createSystemSession(user.asString()); int limitToOneMessage = 1; --------------------------------------------------------------------- To unsubscribe, e-mail: server-dev-unsubscr...@james.apache.org For additional commands, e-mail: server-dev-h...@james.apache.org