This is an automated email from the ASF dual-hosted git repository. btellier pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/james-project.git
commit aff1d11fcd53e6f19a32c270e8db496a4bfa745d Author: LanKhuat <[email protected]> AuthorDate: Thu Mar 12 09:35:00 2020 +0700 JAMES-3072 Add Progress Tracking for MailboxesExport Task --- .../james/webadmin/service/ExportService.java | 42 +++++++++++++--- .../webadmin/service/MailboxesExportTask.java | 18 +++++-- ...ailboxesExportTaskAdditionalInformationDTO.java | 13 ++++- .../james/webadmin/service/ExportServiceTest.java | 57 ++++++++++++++++------ .../service/MailboxesExportRequestToTaskTest.java | 3 ++ ...oxesExportTaskAdditionalInformationDTOTest.java | 2 +- .../mailboxesExport.additionalInformation.json | 3 +- 7 files changed, 108 insertions(+), 30 deletions(-) diff --git a/server/protocols/webadmin/webadmin-mailbox/src/main/java/org/apache/james/webadmin/service/ExportService.java b/server/protocols/webadmin/webadmin-mailbox/src/main/java/org/apache/james/webadmin/service/ExportService.java index e2f96ee..a982fd9 100644 --- a/server/protocols/webadmin/webadmin-mailbox/src/main/java/org/apache/james/webadmin/service/ExportService.java +++ b/server/protocols/webadmin/webadmin-mailbox/src/main/java/org/apache/james/webadmin/service/ExportService.java @@ -24,6 +24,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.PipedInputStream; import java.io.PipedOutputStream; +import java.util.concurrent.ConcurrentLinkedDeque; import javax.inject.Inject; @@ -39,12 +40,36 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.github.fge.lambdas.Throwing; +import com.google.common.collect.Lists; import reactor.core.publisher.Mono; import reactor.core.scheduler.Schedulers; public class ExportService { + public enum Stage { + STARTING, + ZIPPING, + EXPORTING, + COMPLETED, + } + + public static class Progress { + private final ConcurrentLinkedDeque<Stage> stages; + + public Progress() { + this.stages = new ConcurrentLinkedDeque<>(Lists.newArrayList(Stage.STARTING)); + } + + public Stage getStage() { + return stages.getLast(); + } + + public void setStage(Stage stage) { + this.stages.add(stage); + } + } + private static final Logger LOGGER = LoggerFactory.getLogger(ExportService.class); private static final String EXPLANATION = "The backup of your mailboxes has been exported to you"; private static final String FILE_PREFIX = "mailbox-backup-"; @@ -62,17 +87,18 @@ public class ExportService { this.usersRepository = usersRepository; } - public Mono<Task.Result> export(Username username) { + public Mono<Task.Result> export(Progress progress, Username username) { return Mono.usingWhen( - Mono.fromCallable(() -> zipMailboxesContent(username)), - inputStream -> export(username, inputStream), + Mono.fromCallable(() -> zipMailboxesContent(progress, username)), + inputStream -> export(progress, username, inputStream), this::closeResource, (inputStream, throwable) -> closeResource(inputStream), this::closeResource ); } - private InputStream zipMailboxesContent(Username username) throws IOException { + InputStream zipMailboxesContent(Progress progress, Username username) throws IOException { + progress.setStage(Stage.ZIPPING); PipedOutputStream out = new PipedOutputStream(); PipedInputStream in = new PipedInputStream(out); @@ -83,16 +109,18 @@ public class ExportService { return in; } - private Mono<Task.Result> export(Username username, InputStream inputStream) { - return Mono.usingWhen( + Mono<Task.Result> export(Progress progress, Username username, InputStream inputStream) { + return Mono.fromRunnable(() -> progress.setStage(Stage.EXPORTING)) + .then(Mono.usingWhen( blobStore.save(blobStore.getDefaultBucketName(), inputStream, BlobStore.StoragePolicy.LOW_COST), blobId -> export(username, blobId), this::deleteBlob) + .doOnSuccess(any -> progress.setStage(Stage.COMPLETED)) .thenReturn(Task.Result.COMPLETED) .onErrorResume(e -> { LOGGER.error("Error exporting mailboxes of user: {}", username.asString(), e); return Mono.just(Task.Result.PARTIAL); - }); + })); } private Mono<Void> export(Username username, BlobId blobId) { diff --git a/server/protocols/webadmin/webadmin-mailbox/src/main/java/org/apache/james/webadmin/service/MailboxesExportTask.java b/server/protocols/webadmin/webadmin-mailbox/src/main/java/org/apache/james/webadmin/service/MailboxesExportTask.java index 42b0c53..fde9e53 100644 --- a/server/protocols/webadmin/webadmin-mailbox/src/main/java/org/apache/james/webadmin/service/MailboxesExportTask.java +++ b/server/protocols/webadmin/webadmin-mailbox/src/main/java/org/apache/james/webadmin/service/MailboxesExportTask.java @@ -39,15 +39,17 @@ public class MailboxesExportTask implements Task { static final TaskType TASK_TYPE = TaskType.of("MailboxesExportTask"); public static class AdditionalInformation implements TaskExecutionDetails.AdditionalInformation { - private static AdditionalInformation from(Username username) { - return new AdditionalInformation(username, Clock.systemUTC().instant()); + private static AdditionalInformation from(ExportService.Progress progress, Username username) { + return new AdditionalInformation(username, progress.getStage(), Clock.systemUTC().instant()); } private final Username username; + private final ExportService.Stage stage; private final Instant timestamp; - public AdditionalInformation(Username username, Instant timestamp) { + public AdditionalInformation(Username username, ExportService.Stage stage, Instant timestamp) { this.username = username; + this.stage = stage; this.timestamp = timestamp; } @@ -55,6 +57,10 @@ public class MailboxesExportTask implements Task { return username.asString(); } + public ExportService.Stage getStage() { + return stage; + } + @Override public Instant timestamp() { return timestamp; @@ -93,15 +99,17 @@ public class MailboxesExportTask implements Task { private final Username username; private final ExportService service; + private final ExportService.Progress progress; MailboxesExportTask(ExportService service, Username username) { this.username = username; this.service = service; + this.progress = new ExportService.Progress(); } @Override public Result run() { - return service.export(username) + return service.export(progress, username) .subscribeOn(Schedulers.elastic()) .block(); } @@ -113,6 +121,6 @@ public class MailboxesExportTask implements Task { @Override public Optional<TaskExecutionDetails.AdditionalInformation> details() { - return Optional.of(AdditionalInformation.from(username)); + return Optional.of(AdditionalInformation.from(progress, username)); } } \ No newline at end of file diff --git a/server/protocols/webadmin/webadmin-mailbox/src/main/java/org/apache/james/webadmin/service/MailboxesExportTaskAdditionalInformationDTO.java b/server/protocols/webadmin/webadmin-mailbox/src/main/java/org/apache/james/webadmin/service/MailboxesExportTaskAdditionalInformationDTO.java index 3050117..40cb06f 100644 --- a/server/protocols/webadmin/webadmin-mailbox/src/main/java/org/apache/james/webadmin/service/MailboxesExportTaskAdditionalInformationDTO.java +++ b/server/protocols/webadmin/webadmin-mailbox/src/main/java/org/apache/james/webadmin/service/MailboxesExportTaskAdditionalInformationDTO.java @@ -35,24 +35,29 @@ public class MailboxesExportTaskAdditionalInformationDTO implements AdditionalIn .convertToDTO(MailboxesExportTaskAdditionalInformationDTO.class) .toDomainObjectConverter(dto -> new MailboxesExportTask.AdditionalInformation( Username.of(dto.username), + dto.getStage(), dto.timestamp)) .toDTOConverter((details, type) -> new MailboxesExportTaskAdditionalInformationDTO( type, details.timestamp(), - details.getUsername())) + details.getUsername(), + details.getStage())) .typeName(MailboxesExportTask.TASK_TYPE.asString()) .withFactory(AdditionalInformationDTOModule::new); private final String username; private final Instant timestamp; private final String type; + private final ExportService.Stage stage; private MailboxesExportTaskAdditionalInformationDTO(@JsonProperty("type") String type, @JsonProperty("timestamp") Instant timestamp, - @JsonProperty("username") String username) { + @JsonProperty("username") String username, + @JsonProperty("stage") ExportService.Stage stage) { this.type = type; this.timestamp = timestamp; this.username = username; + this.stage = stage; } @Override @@ -68,4 +73,8 @@ public class MailboxesExportTaskAdditionalInformationDTO implements AdditionalIn public String getUsername() { return username; } + + public ExportService.Stage getStage() { + return stage; + } } diff --git a/server/protocols/webadmin/webadmin-mailbox/src/test/java/org/apache/james/webadmin/service/ExportServiceTest.java b/server/protocols/webadmin/webadmin-mailbox/src/test/java/org/apache/james/webadmin/service/ExportServiceTest.java index acf02a2..ea86287 100644 --- a/server/protocols/webadmin/webadmin-mailbox/src/test/java/org/apache/james/webadmin/service/ExportServiceTest.java +++ b/server/protocols/webadmin/webadmin-mailbox/src/test/java/org/apache/james/webadmin/service/ExportServiceTest.java @@ -26,8 +26,11 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doReturn; +import java.io.ByteArrayInputStream; import java.io.File; import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; import javax.mail.MessagingException; @@ -49,6 +52,7 @@ import org.assertj.core.api.SoftAssertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mockito; import com.google.common.base.Strings; import com.google.common.io.Files; @@ -58,10 +62,8 @@ import reactor.core.publisher.Mono; @ExtendWith(FileSystemExtension.class) class ExportServiceTest { - private static final String JAMES_HOST = "james-host"; private static final Domain DOMAIN = Domain.of("domain.tld"); private static final Username UNKNOWN_USER = Username.fromLocalPartWithDomain("unknown", DOMAIN); - private static final String PASSWORD = "password"; private static final String CORRESPONDING_FILE_HEADER = "corresponding-file"; private static final String MESSAGE_CONTENT = "MIME-Version: 1.0\r\n" + "Subject: test\r\n" + @@ -74,11 +76,13 @@ class ExportServiceTest { private ExportService testee; private ExportServiceTestSystem testSystem; + private ExportService.Progress progress; @BeforeEach void setUp(FileSystem fileSystem) throws Exception { testSystem = new ExportServiceTestSystem(fileSystem); - testee = new ExportService(testSystem.backup, testSystem.blobStore, testSystem.blobExport, testSystem.usersRepository); + testee = Mockito.spy(new ExportService(testSystem.backup, testSystem.blobStore, testSystem.blobExport, testSystem.usersRepository)); + progress = new ExportService.Progress(); } private String getFileUrl() throws MessagingException { @@ -87,13 +91,13 @@ class ExportServiceTest { @Test void exportUserMailboxesDataShouldReturnCompletedWhenUserDoesNotExist() { - assertThat(testee.export(UNKNOWN_USER).block()) + assertThat(testee.export(progress, UNKNOWN_USER).block()) .isEqualTo(Task.Result.COMPLETED); } @Test void exportUserMailboxesDataShouldReturnCompletedWhenExistingUserWithoutMailboxes() { - assertThat(testee.export(BOB).block()) + assertThat(testee.export(progress, BOB).block()) .isEqualTo(Task.Result.COMPLETED); } @@ -101,7 +105,7 @@ class ExportServiceTest { void exportUserMailboxesDataShouldReturnCompletedWhenExistingUser() throws Exception { createAMailboxWithAMail(MESSAGE_CONTENT); - assertThat(testee.export(BOB).block()) + assertThat(testee.export(progress, BOB).block()) .isEqualTo(Task.Result.COMPLETED); } @@ -116,7 +120,7 @@ class ExportServiceTest { @Test void exportUserMailboxesDataShouldProduceAnEmptyZipWhenUserDoesNotExist() throws Exception { - testee.export(UNKNOWN_USER).block(); + testee.export(progress, UNKNOWN_USER).block(); ZipAssert.assertThatZip(new FileInputStream(getFileUrl())) .hasNoEntry(); @@ -124,7 +128,7 @@ class ExportServiceTest { @Test void exportUserMailboxesDataShouldProduceAnEmptyZipWhenExistingUserWithoutAnyMailboxes() throws Exception { - testee.export(BOB).block(); + testee.export(progress, BOB).block(); ZipAssert.assertThatZip(new FileInputStream(getFileUrl())) .hasNoEntry(); @@ -134,7 +138,7 @@ class ExportServiceTest { void exportUserMailboxesDataShouldProduceAZipWithEntry() throws Exception { ComposedMessageId id = createAMailboxWithAMail(MESSAGE_CONTENT); - testee.export(BOB).block(); + testee.export(progress, BOB).block(); ZipAssert.assertThatZip(new FileInputStream(getFileUrl())) .containsOnlyEntriesMatching( @@ -146,7 +150,7 @@ class ExportServiceTest { void exportUserMailboxesDataShouldProduceAFileWithExpectedExtension() throws Exception { createAMailboxWithAMail(MESSAGE_CONTENT); - testee.export(BOB).block(); + testee.export(progress, BOB).block(); assertThat(Files.getFileExtension(getFileUrl())).isEqualTo(FileExtension.ZIP.getExtension()); } @@ -155,7 +159,7 @@ class ExportServiceTest { void exportUserMailboxesDataShouldProduceAFileWithExpectedName() throws Exception { createAMailboxWithAMail(MESSAGE_CONTENT); - testee.export(BOB).block(); + testee.export(progress, BOB).block(); File file = new File(getFileUrl()); @@ -166,7 +170,7 @@ class ExportServiceTest { void exportUserMailboxesWithSizableDataShouldProduceAFile() throws Exception { ComposedMessageId id = createAMailboxWithAMail(TWELVE_MEGABYTES_STRING); - testee.export(BOB).block(); + testee.export(progress, BOB).block(); ZipAssert.assertThatZip(new FileInputStream(getFileUrl())) .containsOnlyEntriesMatching( @@ -178,7 +182,7 @@ class ExportServiceTest { void exportUserMailboxesDataShouldDeleteBlobAfterCompletion() throws Exception { createAMailboxWithAMail(MESSAGE_CONTENT); - testee.export(BOB).block(); + testee.export(progress, BOB).block(); String fileName = Files.getNameWithoutExtension(getFileUrl()); String blobId = fileName.substring(fileName.lastIndexOf("-") + 1); @@ -199,7 +203,7 @@ class ExportServiceTest { .when(testSystem.blobStore) .delete(any(), any()); - Task.Result result = testee.export(BOB).block(); + Task.Result result = testee.export(progress, BOB).block(); String fileName = Files.getNameWithoutExtension(getFileUrl()); String blobId = fileName.substring(fileName.lastIndexOf("-") + 1); @@ -208,4 +212,29 @@ class ExportServiceTest { assertThat(result).isEqualTo(Task.Result.COMPLETED); } + + @Test + void exportUserMailboxesDataShouldUpdateProgressWhenZipping() throws IOException { + testee.zipMailboxesContent(progress, BOB); + + assertThat(progress.getStage()).isEqualTo(ExportService.Stage.ZIPPING); + } + + @Test + void exportUserMailboxesDataShouldUpdateProgressWhenExporting() { + doReturn(Mono.error(new RuntimeException())) + .when(testSystem.blobStore) + .save(any(), any(InputStream.class), any()); + + testee.export(progress, BOB, new ByteArrayInputStream(MESSAGE_CONTENT.getBytes())).block(); + + assertThat(progress.getStage()).isEqualTo(ExportService.Stage.EXPORTING); + } + + @Test + void exportUserMailboxesDataShouldUpdateProgressWhenComplete() { + testee.export(progress, BOB).block(); + + assertThat(progress.getStage()).isEqualTo(ExportService.Stage.COMPLETED); + } } \ No newline at end of file diff --git a/server/protocols/webadmin/webadmin-mailbox/src/test/java/org/apache/james/webadmin/service/MailboxesExportRequestToTaskTest.java b/server/protocols/webadmin/webadmin-mailbox/src/test/java/org/apache/james/webadmin/service/MailboxesExportRequestToTaskTest.java index 6887eba..949bd4a 100644 --- a/server/protocols/webadmin/webadmin-mailbox/src/test/java/org/apache/james/webadmin/service/MailboxesExportRequestToTaskTest.java +++ b/server/protocols/webadmin/webadmin-mailbox/src/test/java/org/apache/james/webadmin/service/MailboxesExportRequestToTaskTest.java @@ -221,6 +221,7 @@ class MailboxesExportRequestToTaskTest { .body("taskId", is(taskId)) .body("type", is("MailboxesExportTask")) .body("additionalInformation.username", is(BOB.asString())) + .body("additionalInformation.stage", is(ExportService.Stage.COMPLETED.toString())) .body("startedDate", is(notNullValue())) .body("submitDate", is(notNullValue())) .body("completedDate", is(notNullValue())); @@ -258,6 +259,7 @@ class MailboxesExportRequestToTaskTest { .body("taskId", is(taskId)) .body("type", is("MailboxesExportTask")) .body("additionalInformation.username", is(BOB.asString())) + .body("additionalInformation.stage", is(ExportService.Stage.COMPLETED.toString())) .body("startedDate", is(notNullValue())) .body("submitDate", is(notNullValue())) .body("completedDate", is(notNullValue())); @@ -333,6 +335,7 @@ class MailboxesExportRequestToTaskTest { .body("taskId", is(taskId)) .body("type", is("MailboxesExportTask")) .body("additionalInformation.username", is(BOB.asString())) + .body("additionalInformation.stage", is(ExportService.Stage.COMPLETED.toString())) .body("startedDate", is(notNullValue())) .body("submitDate", is(notNullValue())) .body("completedDate", is(notNullValue())); diff --git a/server/protocols/webadmin/webadmin-mailbox/src/test/java/org/apache/james/webadmin/service/MailboxesExportTaskAdditionalInformationDTOTest.java b/server/protocols/webadmin/webadmin-mailbox/src/test/java/org/apache/james/webadmin/service/MailboxesExportTaskAdditionalInformationDTOTest.java index 7102633..711219e 100644 --- a/server/protocols/webadmin/webadmin-mailbox/src/test/java/org/apache/james/webadmin/service/MailboxesExportTaskAdditionalInformationDTOTest.java +++ b/server/protocols/webadmin/webadmin-mailbox/src/test/java/org/apache/james/webadmin/service/MailboxesExportTaskAdditionalInformationDTOTest.java @@ -29,7 +29,7 @@ import org.junit.jupiter.api.Test; class MailboxesExportTaskAdditionalInformationDTOTest { private static final Instant INSTANT = Instant.parse("2007-12-03T10:15:30.00Z"); private static final MailboxesExportTask.AdditionalInformation DOMAIN_OBJECT = new MailboxesExportTask.AdditionalInformation( - Username.of("bob"), INSTANT); + Username.of("bob"), ExportService.Stage.STARTING, INSTANT); @Test void shouldMatchJsonSerializationContract() throws Exception { diff --git a/server/protocols/webadmin/webadmin-mailbox/src/test/resources/json/mailboxesExport.additionalInformation.json b/server/protocols/webadmin/webadmin-mailbox/src/test/resources/json/mailboxesExport.additionalInformation.json index 181a96c..ac16f5c 100644 --- a/server/protocols/webadmin/webadmin-mailbox/src/test/resources/json/mailboxesExport.additionalInformation.json +++ b/server/protocols/webadmin/webadmin-mailbox/src/test/resources/json/mailboxesExport.additionalInformation.json @@ -1,5 +1,6 @@ { "type":"MailboxesExportTask", "timestamp":"2007-12-03T10:15:30Z", - "username": "bob" + "username": "bob", + "stage": "STARTING" } \ No newline at end of file --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
