This is an automated email from the ASF dual-hosted git repository. aduprat pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/james-project.git
commit 9a97b6a5f4f0498495431f83815b5c65d2bf9d85 Author: datph <dphamho...@linagora.com> AuthorDate: Fri Apr 5 11:55:03 2019 +0700 JAMES-2710 Add Purge API to DeletedMessagesVaultRoutes --- .../vault/utils/VaultGarbageCollectionTask.java | 2 +- .../vault/routes/DeletedMessagesVaultRoutes.java | 130 +++++++++-- .../routes/DeletedMessagesVaultRoutesTest.java | 240 ++++++++++++++++++++- 3 files changed, 357 insertions(+), 15 deletions(-) diff --git a/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/utils/VaultGarbageCollectionTask.java b/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/utils/VaultGarbageCollectionTask.java index 36170f3..5a55c80 100644 --- a/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/utils/VaultGarbageCollectionTask.java +++ b/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/utils/VaultGarbageCollectionTask.java @@ -65,7 +65,7 @@ public class VaultGarbageCollectionTask implements Task { } } - private static final String TYPE = "deletedMessages/garbageCollection"; + public static final String TYPE = "deletedMessages/garbageCollection"; private final DeleteByQueryExecutor deleteByQueryExecutor; private final DeleteByQueryExecutor.Notifiers notifiers; diff --git a/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/src/main/java/org/apache/james/webadmin/vault/routes/DeletedMessagesVaultRoutes.java b/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/src/main/java/org/apache/james/webadmin/vault/routes/DeletedMessagesVaultRoutes.java index 32f5d2b..50f483e 100644 --- a/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/src/main/java/org/apache/james/webadmin/vault/routes/DeletedMessagesVaultRoutes.java +++ b/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/src/main/java/org/apache/james/webadmin/vault/routes/DeletedMessagesVaultRoutes.java @@ -23,10 +23,12 @@ import static org.apache.james.webadmin.Constants.SEPARATOR; import java.util.List; import java.util.Optional; +import java.util.function.Function; import java.util.stream.Stream; import javax.inject.Inject; import javax.mail.internet.AddressException; +import javax.ws.rs.DELETE; import javax.ws.rs.POST; import javax.ws.rs.Path; import javax.ws.rs.Produces; @@ -40,6 +42,7 @@ import org.apache.james.task.TaskId; import org.apache.james.task.TaskManager; import org.apache.james.user.api.UsersRepository; import org.apache.james.user.api.UsersRepositoryException; +import org.apache.james.vault.DeletedMessageVault; import org.apache.james.vault.search.Query; import org.apache.james.webadmin.Constants; import org.apache.james.webadmin.Routes; @@ -63,6 +66,7 @@ import io.swagger.annotations.ApiImplicitParams; import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiResponse; import io.swagger.annotations.ApiResponses; +import spark.HaltException; import spark.Request; import spark.Response; import spark.Service; @@ -85,7 +89,8 @@ public class DeletedMessagesVaultRoutes implements Routes { .findFirst(); } - private static List<String> plainValues() { + @VisibleForTesting + static List<String> plainValues() { return Stream.of(values()) .map(VaultAction::getValue) .collect(Guavate.toImmutableList()); @@ -102,16 +107,51 @@ public class DeletedMessagesVaultRoutes implements Routes { } } + enum VaultScope { + EXPIRED("expired"); + + static Optional<VaultScope> getScope(String value) { + Preconditions.checkNotNull(value, "scope cannot be null"); + Preconditions.checkArgument(StringUtils.isNotBlank(value), "scope cannot be empty or blank"); + + return Stream.of(values()) + .filter(action -> action.value.equals(value)) + .findFirst(); + } + + @VisibleForTesting + static List<String> plainValues() { + return Stream.of(values()) + .map(VaultScope::getValue) + .collect(Guavate.toImmutableList()); + } + + private final String value; + + VaultScope(String value) { + this.value = value; + } + + public String getValue() { + return value; + } + } + public static final String ROOT_PATH = "deletedMessages"; public static final String USERS = "users"; public static final String USER_PATH = ROOT_PATH + SEPARATOR + USERS; - private static final String USER_PATH_PARAM = "user"; - private static final String RESTORE_PATH = USER_PATH + SEPARATOR + ":" + USER_PATH_PARAM; + private static final String USER_PATH_PARAM = ":user"; + static final String MESSAGE_PATH_PARAM = "messages"; + private static final String MESSAGE_ID_PARAM = ":messageId"; + private static final String RESTORE_PATH = USER_PATH + SEPARATOR + USER_PATH_PARAM; + private static final String DELETE_PATH = USER_PATH + SEPARATOR + USER_PATH_PARAM + SEPARATOR + MESSAGE_PATH_PARAM + SEPARATOR + MESSAGE_ID_PARAM; + private static final String SCOPE_QUERY_PARAM = "scope"; private static final String ACTION_QUERY_PARAM = "action"; private static final String EXPORT_TO_QUERY_PARAM = "exportTo"; private final RestoreService vaultRestore; private final ExportService vaultExport; + private final DeletedMessageVault deletedMessageVault; private final JsonTransformer jsonTransformer; private final TaskManager taskManager; private final JsonExtractor<QueryElement> jsonExtractor; @@ -120,8 +160,9 @@ public class DeletedMessagesVaultRoutes implements Routes { @Inject @VisibleForTesting - DeletedMessagesVaultRoutes(RestoreService vaultRestore, ExportService vaultExport, JsonTransformer jsonTransformer, + DeletedMessagesVaultRoutes(DeletedMessageVault deletedMessageVault, RestoreService vaultRestore, ExportService vaultExport, JsonTransformer jsonTransformer, TaskManager taskManager, QueryTranslator queryTranslator, UsersRepository usersRepository) { + this.deletedMessageVault = deletedMessageVault; this.vaultRestore = vaultRestore; this.vaultExport = vaultExport; this.jsonTransformer = jsonTransformer; @@ -139,6 +180,7 @@ public class DeletedMessagesVaultRoutes implements Routes { @Override public void define(Service service) { service.post(RESTORE_PATH, this::userActions, jsonTransformer); + service.delete(ROOT_PATH, this::deleteWithScope, jsonTransformer); } @POST @@ -175,14 +217,52 @@ public class DeletedMessagesVaultRoutes implements Routes { @ApiResponse(code = HttpStatus.INTERNAL_SERVER_ERROR_500, message = "Internal server error - Something went bad on the server side.") }) private TaskIdDto userActions(Request request, Response response) throws JsonExtractException { - VaultAction requestedAction = extractVaultAction(request); + VaultAction requestedAction = extractParam(request, ACTION_QUERY_PARAM, this::getVaultAction); + validateAction(requestedAction, VaultAction.RESTORE, VaultAction.EXPORT); - Task requestedTask = generateTask(requestedAction, request); + Task requestedTask = generateUserTask(requestedAction, request); TaskId taskId = taskManager.submit(requestedTask); return TaskIdDto.respond(response, taskId); } - private Task generateTask(VaultAction requestedAction, Request request) throws JsonExtractException { + @DELETE + @Path(ROOT_PATH) + @ApiOperation(value = "Purge all expired messages based on retentionPeriod of deletedMessageVault configuration") + @ApiImplicitParams({ + @ApiImplicitParam( + required = true, + name = "action", + dataType = "String", + paramType = "query", + example = "?scope=expired", + value = "Compulsory. Needs to be a purge action") + }) + @ApiResponses(value = { + @ApiResponse(code = HttpStatus.CREATED_201, message = "Task is created", response = TaskIdDto.class), + @ApiResponse(code = HttpStatus.BAD_REQUEST_400, message = "Bad request - action is invalid"), + @ApiResponse(code = HttpStatus.INTERNAL_SERVER_ERROR_500, message = "Internal server error - Something went bad on the server side.") + }) + private TaskIdDto deleteWithScope(Request request, Response response) { + Task vaultTask = generateVaultScopeTask(request); + TaskId taskId = taskManager.submit(vaultTask); + return TaskIdDto.respond(response, taskId); + } + + private Task generateVaultScopeTask(Request request) { + VaultScope scope = extractParam(request, SCOPE_QUERY_PARAM, this::getVaultScope); + if (!scope.equals(VaultScope.EXPIRED)) { + throw ErrorResponder.builder() + .statusCode(HttpStatus.BAD_REQUEST_400) + .type(ErrorResponder.ErrorType.INVALID_ARGUMENT) + .message(String.format("'%s' is not a valid scope. Supported values are: %s", + scope.value, + VaultScope.plainValues())) + .haltError(); + } + return deletedMessageVault.deleteExpiredMessagesTask(); + } + + private Task generateUserTask(VaultAction requestedAction, Request request) throws JsonExtractException { User user = extractUser(request); validateUserExist(user); Query query = translate(jsonExtractor.parse(request.body())); @@ -266,11 +346,11 @@ public class DeletedMessagesVaultRoutes implements Routes { } } - private VaultAction extractVaultAction(Request request) { - String actionParam = request.queryParams(ACTION_QUERY_PARAM); - return Optional.ofNullable(actionParam) - .map(this::getVaultAction) - .orElseThrow(() -> new IllegalArgumentException("action parameter is missing")); + private <T> T extractParam(Request request, String queryParam, Function<String, T> mapper) { + String param = request.queryParams(queryParam); + return Optional.ofNullable(param) + .map(mapper) + .orElseThrow(() -> new IllegalArgumentException("parameter is missing")); } private VaultAction getVaultAction(String actionString) { @@ -279,4 +359,30 @@ public class DeletedMessagesVaultRoutes implements Routes { actionString, Joiner.on(",").join(VaultAction.plainValues())))); } + + private VaultScope getVaultScope(String scopeString) { + return VaultScope.getScope(scopeString) + .orElseThrow(() -> new IllegalArgumentException(String.format("'%s' is not a valid scope. Supported values are: (%s)", + scopeString, + Joiner.on(",").join(VaultScope.plainValues())))); + } + + private void validateAction(VaultAction requestedAction, VaultAction... validActions) { + Stream.of(validActions) + .filter(action -> action.equals(requestedAction)) + .findFirst() + .orElseThrow(() -> throwNotSupportedAction(requestedAction, validActions)); + } + + private HaltException throwNotSupportedAction(VaultAction requestAction, VaultAction... validActions) { + return ErrorResponder.builder() + .statusCode(HttpStatus.BAD_REQUEST_400) + .type(ErrorResponder.ErrorType.INVALID_ARGUMENT) + .message(String.format("'%s' is not a valid action. Supported values are: %s", + requestAction.getValue(), + Joiner.on(", ").join(Stream.of(validActions) + .map(action -> String.format("'%s'", action.getValue())) + .collect(Guavate.toImmutableList())))) + .haltError(); + } } diff --git a/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/src/test/java/org/apache/james/webadmin/vault/routes/DeletedMessagesVaultRoutesTest.java b/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/src/test/java/org/apache/james/webadmin/vault/routes/DeletedMessagesVaultRoutesTest.java index db166b0..74c3167 100644 --- a/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/src/test/java/org/apache/james/webadmin/vault/routes/DeletedMessagesVaultRoutesTest.java +++ b/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/src/test/java/org/apache/james/webadmin/vault/routes/DeletedMessagesVaultRoutesTest.java @@ -29,6 +29,7 @@ import static org.apache.james.vault.DeletedMessageFixture.DELETION_DATE; import static org.apache.james.vault.DeletedMessageFixture.DELIVERY_DATE; import static org.apache.james.vault.DeletedMessageFixture.FINAL_STAGE; import static org.apache.james.vault.DeletedMessageFixture.MAILBOX_ID_1; +import static org.apache.james.vault.DeletedMessageFixture.MAILBOX_ID_2; import static org.apache.james.vault.DeletedMessageFixture.MAILBOX_ID_3; import static org.apache.james.vault.DeletedMessageFixture.SUBJECT; import static org.apache.james.vault.DeletedMessageFixture.USER; @@ -47,6 +48,8 @@ import static org.apache.mailet.base.MailAddressFixture.SENDER2; import static org.assertj.core.api.Assertions.assertThat; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; +import static org.hamcrest.Matchers.startsWith; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doReturn; @@ -61,6 +64,8 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.time.Clock; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; import java.util.List; import java.util.Optional; import java.util.stream.Stream; @@ -99,6 +104,8 @@ import org.apache.james.vault.DeletedMessageZipper; import org.apache.james.vault.RetentionConfiguration; import org.apache.james.vault.memory.MemoryDeletedMessagesVault; import org.apache.james.vault.search.Query; +import org.apache.james.vault.utils.DeleteByQueryExecutor; +import org.apache.james.vault.utils.VaultGarbageCollectionTask; import org.apache.james.webadmin.WebAdminServer; import org.apache.james.webadmin.WebAdminUtils; import org.apache.james.webadmin.routes.TasksRoutes; @@ -113,6 +120,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; +import com.google.common.base.Joiner; import com.google.common.collect.ImmutableList; import io.restassured.RestAssured; @@ -150,10 +158,12 @@ class DeletedMessagesVaultRoutesTest { private MemoryUsersRepository usersRepository; private ExportService exportService; private HashBlobId.Factory blobIdFactory; + private Clock clock; @BeforeEach void beforeEach() throws Exception { - vault = spy(new MemoryDeletedMessagesVault(RetentionConfiguration.DEFAULT, Clock.systemUTC())); + clock = Clock.systemUTC(); + vault = spy(new MemoryDeletedMessagesVault(RetentionConfiguration.DEFAULT, clock)); InMemoryIntegrationResources inMemoryResource = InMemoryIntegrationResources.defaultResources(); mailboxManager = spy(inMemoryResource.getMailboxManager()); @@ -171,7 +181,7 @@ class DeletedMessagesVaultRoutesTest { webAdminServer = WebAdminUtils.createWebAdminServer( new DefaultMetricFactory(), new TasksRoutes(taskManager, jsonTransformer), - new DeletedMessagesVaultRoutes(vaultRestore, exportService, jsonTransformer, taskManager, queryTranslator, usersRepository)); + new DeletedMessagesVaultRoutes(vault, vaultRestore, exportService, jsonTransformer, taskManager, queryTranslator, usersRepository)); webAdminServer.configure(NO_CONFIGURATION); webAdminServer.await(); @@ -261,6 +271,60 @@ class DeletedMessagesVaultRoutesTest { .body("details", is(notNullValue())); } + @Test + void purgeAPIShouldReturnInvalidWhenScopeIsMissing() { + when() + .delete() + .then() + .statusCode(HttpStatus.BAD_REQUEST_400) + .body("statusCode", is(400)) + .body("type", is(ErrorResponder.ErrorType.INVALID_ARGUMENT.getType())) + .body("message", is("Invalid arguments supplied in the user request")) + .body("details", is("parameter is missing")); + } + + @Test + void purgeAPIShouldReturnInvalidWhenPassingEmptyScope() { + given() + .queryParam("scope", "") + .when() + .delete() + .then() + .statusCode(HttpStatus.BAD_REQUEST_400) + .body("statusCode", is(400)) + .body("type", is(ErrorResponder.ErrorType.INVALID_ARGUMENT.getType())) + .body("message", is("Invalid arguments supplied in the user request")) + .body("details", is("scope cannot be empty or blank")); + } + + @Test + void purgeAPIShouldReturnInvalidWhenScopeIsInValid() { + given() + .queryParam("scope", "invalid action") + .when() + .delete() + .then() + .statusCode(HttpStatus.BAD_REQUEST_400) + .body("statusCode", is(400)) + .body("type", is(ErrorResponder.ErrorType.INVALID_ARGUMENT.getType())) + .body("message", is("Invalid arguments supplied in the user request")) + .body("details", startsWith("'invalid action' is not a valid scope.")); + } + + @Test + void purgeAPIShouldReturnInvalidWhenPassingCaseInsensitiveScope() { + given() + .queryParam("scope", "EXPIRED") + .when() + .delete() + .then() + .statusCode(HttpStatus.BAD_REQUEST_400) + .body("statusCode", is(400)) + .body("type", is(ErrorResponder.ErrorType.INVALID_ARGUMENT.getType())) + .body("message", is("Invalid arguments supplied in the user request")) + .body("details", startsWith("'EXPIRED' is not a valid scope.")); + } + @ParameterizedTest @ValueSource(strings = {"restore", "export"}) void userVaultAPIShouldReturnInvalidWhenUserIsInvalid(String action) { @@ -1853,6 +1917,178 @@ class DeletedMessagesVaultRoutesTest { } } + @Nested + class PurgeTest { + + @Test + void purgeShouldReturnATaskCreated() { + given() + .queryParam("scope", "expired") + .when() + .delete() + .then() + .statusCode(HttpStatus.CREATED_201) + .body("taskId", notNullValue()); + } + + @Test + void purgeShouldProduceASuccessfulTaskWithAdditionalInformation() { + vault.append(USER, DELETED_MESSAGE, new ByteArrayInputStream(CONTENT)).block(); + vault.append(USER, DELETED_MESSAGE_2, new ByteArrayInputStream(CONTENT)).block(); + + String taskId = + with() + .queryParam("scope", "expired") + .delete() + .jsonPath() + .get("taskId"); + + given() + .basePath(TasksRoutes.BASE) + .when() + .get(taskId + "/await") + .then() + .body("status", is("completed")) + .body("taskId", is(taskId)) + .body("type", is(VaultGarbageCollectionTask.TYPE)) + .body("additionalInformation.beginningOfRetentionPeriod", is(notNullValue())) + .body("additionalInformation.handledUserCount", is(1)) + .body("additionalInformation.permanantlyDeletedMessages", is(2)) + .body("additionalInformation.vaultSearchErrorCount", is(0)) + .body("additionalInformation.deletionErrorCount", is(0)) + .body("startedDate", is(notNullValue())) + .body("submitDate", is(notNullValue())) + .body("completedDate", is(notNullValue())); + } + + @Test + void purgeShouldNotDeleteNotExpiredMessagesInTheVault() { + DeletedMessage notExpiredMessage = DeletedMessage.builder() + .messageId(InMemoryMessageId.of(46)) + .originMailboxes(MAILBOX_ID_1, MAILBOX_ID_2) + .user(USER) + .deliveryDate(DELIVERY_DATE) + .deletionDate(ZonedDateTime.ofInstant(clock.instant(), ZoneOffset.UTC)) + .sender(MaybeSender.of(SENDER)) + .recipients(RECIPIENT1, RECIPIENT3) + .hasAttachment(false) + .size(CONTENT.length) + .build(); + + vault.append(USER, DELETED_MESSAGE, new ByteArrayInputStream(CONTENT)).block(); + vault.append(USER, DELETED_MESSAGE_2, new ByteArrayInputStream(CONTENT)).block(); + vault.append(USER, notExpiredMessage, new ByteArrayInputStream(CONTENT)).block(); + + String taskId = + with() + .queryParam("scope", "expired") + .delete() + .jsonPath() + .get("taskId"); + + with() + .basePath(TasksRoutes.BASE) + .get(taskId + "/await"); + + assertThat(Flux.from(vault.search(USER, Query.ALL)).toStream()) + .containsOnly(notExpiredMessage); + } + + @Test + void purgeShouldNotAppendMessagesToUserMailbox() throws Exception { + vault.append(USER, DELETED_MESSAGE, new ByteArrayInputStream(CONTENT)).block(); + vault.append(USER, DELETED_MESSAGE_2, new ByteArrayInputStream(CONTENT)).block(); + + String taskId = + with() + .queryParam("scope", "expired") + .delete() + .jsonPath() + .get("taskId"); + + with() + .basePath(TasksRoutes.BASE) + .get(taskId + "/await"); + + assertThat(hasAnyMail(USER)) + .isFalse(); + } + + @Nested + class FailingPurgeTest { + + @Test + void purgeShouldProduceAFailedTaskWithVaultSearchError() { + vault.append(USER, DELETED_MESSAGE, new ByteArrayInputStream(CONTENT)).block(); + vault.append(USER, DELETED_MESSAGE_2, new ByteArrayInputStream(CONTENT)).block(); + + doReturn(new DeleteByQueryExecutor(vault)).when(vault).getDeleteByQueryExecutor(); + doReturn(Flux.error(new RuntimeException("mock exception"))) + .when(vault) + .search(any(), any()); + + String taskId = + with() + .queryParam("scope", "expired") + .delete() + .jsonPath() + .get("taskId"); + + given() + .basePath(TasksRoutes.BASE) + .when() + .get(taskId + "/await") + .then() + .body("status", is("failed")) + .body("taskId", is(taskId)) + .body("type", is(VaultGarbageCollectionTask.TYPE)) + .body("additionalInformation.beginningOfRetentionPeriod", is(notNullValue())) + .body("additionalInformation.handledUserCount", is(1)) + .body("additionalInformation.permanantlyDeletedMessages", is(0)) + .body("additionalInformation.vaultSearchErrorCount", is(1)) + .body("additionalInformation.deletionErrorCount", is(0)) + .body("startedDate", is(notNullValue())) + .body("submitDate", is(notNullValue())) + .body("completedDate", is(nullValue())); + } + + @Test + void purgeShouldProduceAFailedTaskWithVaultDeleteError() { + vault.append(USER, DELETED_MESSAGE, new ByteArrayInputStream(CONTENT)).block(); + vault.append(USER, DELETED_MESSAGE_2, new ByteArrayInputStream(CONTENT)).block(); + + doReturn(new DeleteByQueryExecutor(vault)).when(vault).getDeleteByQueryExecutor(); + doReturn(Mono.error(new RuntimeException("mock exception"))) + .when(vault) + .delete(any(), any()); + + String taskId = + with() + .queryParam("scope", "expired") + .delete() + .jsonPath() + .get("taskId"); + + given() + .basePath(TasksRoutes.BASE) + .when() + .get(taskId + "/await") + .then() + .body("status", is("failed")) + .body("taskId", is(taskId)) + .body("type", is(VaultGarbageCollectionTask.TYPE)) + .body("additionalInformation.beginningOfRetentionPeriod", is(notNullValue())) + .body("additionalInformation.handledUserCount", is(1)) + .body("additionalInformation.permanantlyDeletedMessages", is(0)) + .body("additionalInformation.vaultSearchErrorCount", is(0)) + .body("additionalInformation.deletionErrorCount", is(2)) + .body("startedDate", is(notNullValue())) + .body("submitDate", is(notNullValue())) + .body("completedDate", is(nullValue())); + } + } + } + private boolean hasAnyMail(User user) throws MailboxException { MailboxSession session = mailboxManager.createSystemSession(user.asString()); int limitToOneMessage = 1; --------------------------------------------------------------------- To unsubscribe, e-mail: server-dev-unsubscr...@james.apache.org For additional commands, e-mail: server-dev-h...@james.apache.org