This is an automated email from the ASF dual-hosted git repository. btellier pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/james-project.git
commit 38b29e3f4c9bb920a3a11a04aa3016c0f18d0b8a Author: Tran Tien Duc <[email protected]> AuthorDate: Mon Dec 2 12:02:18 2019 +0700 JAMES-2992 Storing previews when reading JMAP message --- .../org/apache/james/jmap/api/model/Preview.java | 2 + .../draft/model/message/view/MessageFullView.java | 11 ++- .../model/message/view/MessageFullViewFactory.java | 59 +++++++++++++++- .../jmap/draft/methods/GetMessagesMethodTest.java | 3 +- .../jmap/draft/methods/MessageSenderTest.java | 4 +- .../methods/SetMessagesCreationProcessorTest.java | 5 +- .../message/view/MessageFastViewFactoryTest.java | 3 +- .../message/view/MessageFullViewFactoryTest.java | 78 +++++++++++++++++++++- 8 files changed, 151 insertions(+), 14 deletions(-) diff --git a/server/data/data-jmap/src/main/java/org/apache/james/jmap/api/model/Preview.java b/server/data/data-jmap/src/main/java/org/apache/james/jmap/api/model/Preview.java index ed69078..d4f7fb4 100644 --- a/server/data/data-jmap/src/main/java/org/apache/james/jmap/api/model/Preview.java +++ b/server/data/data-jmap/src/main/java/org/apache/james/jmap/api/model/Preview.java @@ -29,6 +29,8 @@ import com.google.common.base.Preconditions; public class Preview { + public static final Preview EMPTY = Preview.from(""); + private static final int MAX_LENGTH = 256; public static Preview from(String value) { diff --git a/server/protocols/jmap-draft/src/main/java/org/apache/james/jmap/draft/model/message/view/MessageFullView.java b/server/protocols/jmap-draft/src/main/java/org/apache/james/jmap/draft/model/message/view/MessageFullView.java index 528eda2..2fbdfcc 100644 --- a/server/protocols/jmap-draft/src/main/java/org/apache/james/jmap/draft/model/message/view/MessageFullView.java +++ b/server/protocols/jmap-draft/src/main/java/org/apache/james/jmap/draft/model/message/view/MessageFullView.java @@ -76,7 +76,7 @@ public class MessageFullView extends MessageFastView { public Builder attachments(List<Attachment> attachments) { this.attachments.addAll(attachments); - boolean hasAttachments = this.hasAttachment(this.attachments.build()); + boolean hasAttachments = MessageFullView.hasAttachment(this.attachments.build()); return super.hasAttachment(hasAttachments); } @@ -100,12 +100,11 @@ public class MessageFullView extends MessageFastView { super.checkState(); Preconditions.checkState(areAttachedMessagesKeysInAttachments(attachments, attachedMessages), "'attachedMessages' keys must be in 'attachements'"); } + } - private boolean hasAttachment(List<Attachment> attachments) { - return attachments.stream() - .anyMatch(attachment -> !attachment.isInlinedWithCid()); - } - + static boolean hasAttachment(List<Attachment> attachments) { + return attachments.stream() + .anyMatch(attachment -> !attachment.isInlinedWithCid()); } protected static boolean areAttachedMessagesKeysInAttachments(ImmutableList<Attachment> attachments, ImmutableMap<BlobId, SubMessage> attachedMessages) { diff --git a/server/protocols/jmap-draft/src/main/java/org/apache/james/jmap/draft/model/message/view/MessageFullViewFactory.java b/server/protocols/jmap-draft/src/main/java/org/apache/james/jmap/draft/model/message/view/MessageFullViewFactory.java index f6361e8..e3223db 100644 --- a/server/protocols/jmap-draft/src/main/java/org/apache/james/jmap/draft/model/message/view/MessageFullViewFactory.java +++ b/server/protocols/jmap-draft/src/main/java/org/apache/james/jmap/draft/model/message/view/MessageFullViewFactory.java @@ -26,11 +26,14 @@ import java.util.Date; import java.util.List; import java.util.Optional; import java.util.Set; +import java.util.function.Supplier; import javax.inject.Inject; import javax.mail.internet.SharedInputStream; import org.apache.james.jmap.api.model.Preview; +import org.apache.james.jmap.api.projections.MessageFastViewPrecomputedProperties; +import org.apache.james.jmap.api.projections.MessageFastViewProjection; import org.apache.james.jmap.draft.model.Attachment; import org.apache.james.jmap.draft.model.BlobId; import org.apache.james.jmap.draft.model.Emailer; @@ -50,25 +53,36 @@ import org.apache.james.mailbox.model.MessageResult; import org.apache.james.mime4j.dom.Message; import org.apache.james.util.mime.MessageContentExtractor; import org.apache.james.util.mime.MessageContentExtractor.MessageContent; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import com.github.steveash.guavate.Guavate; import com.google.common.base.Preconditions; import com.google.common.base.Strings; import com.google.common.collect.Sets; +import reactor.core.publisher.Mono; +import reactor.core.scheduler.Schedulers; + public class MessageFullViewFactory implements MessageViewFactory<MessageFullView> { + + private static final Logger LOGGER = LoggerFactory.getLogger(MessageFullViewFactory.class); + private final BlobManager blobManager; private final MessageContentExtractor messageContentExtractor; private final HtmlTextExtractor htmlTextExtractor; private final MessageIdManager messageIdManager; + private final MessageFastViewProjection fastViewProjection; @Inject public MessageFullViewFactory(BlobManager blobManager, MessageContentExtractor messageContentExtractor, - HtmlTextExtractor htmlTextExtractor, MessageIdManager messageIdManager) { + HtmlTextExtractor htmlTextExtractor, MessageIdManager messageIdManager, + MessageFastViewProjection fastViewProjection) { this.blobManager = blobManager; this.messageContentExtractor = messageContentExtractor; this.htmlTextExtractor = htmlTextExtractor; this.messageIdManager = messageIdManager; + this.fastViewProjection = fastViewProjection; } @Override @@ -84,7 +98,10 @@ public class MessageFullViewFactory implements MessageViewFactory<MessageFullVie Optional<String> mainTextContent = mainTextContent(messageContent); Optional<String> textBody = computeTextBodyIfNeeded(messageContent, mainTextContent); - Optional<Preview> preview = mainTextContent.map(Preview::compute); + MessageFastViewPrecomputedProperties messageProjection = retrieveProjection( + messageContent, + message.getMessageId(), + () -> MessageFullView.hasAttachment(getAttachments(message.getAttachments()))); return MessageFullView.builder() .id(message.getMessageId()) @@ -104,11 +121,47 @@ public class MessageFullViewFactory implements MessageViewFactory<MessageFullVie .date(getDateFromHeaderOrInternalDateOtherwise(mimeMessage, message)) .textBody(textBody) .htmlBody(htmlBody) - .preview(preview) + .preview(messageProjection.getPreview()) .attachments(getAttachments(message.getAttachments())) .build(); } + private MessageFastViewPrecomputedProperties retrieveProjection(MessageContent messageContent, + MessageId messageId, Supplier<Boolean> hasAttachments) { + return Mono.from(fastViewProjection.retrieve(messageId)) + .onErrorResume(throwable -> fallBackToCompute(messageContent, hasAttachments, throwable)) + .switchIfEmpty(computeThenStoreAsync(messageContent, messageId, hasAttachments)) + .subscribeOn(Schedulers.boundedElastic()) + .block(); + } + + private Mono<MessageFastViewPrecomputedProperties> fallBackToCompute(MessageContent messageContent, + Supplier<Boolean> hasAttachments, + Throwable throwable) { + LOGGER.error("Cannot retrieve the computed preview from MessageFastViewProjection", throwable); + return computeProjection(messageContent, hasAttachments); + } + + private Mono<MessageFastViewPrecomputedProperties> computeThenStoreAsync(MessageContent messageContent, + MessageId messageId, + Supplier<Boolean> hasAttachments) { + return computeProjection(messageContent, hasAttachments) + .doOnNext(projection -> Mono.from(fastViewProjection.store(messageId, projection)) + .doOnError(throwable -> LOGGER.error("Cannot store the projection to MessageFastViewProjection", throwable)) + .subscribeOn(Schedulers.boundedElastic()) + .subscribe()); + } + + private Mono<MessageFastViewPrecomputedProperties> computeProjection(MessageContent messageContent, Supplier<Boolean> hasAttachments) { + return Mono.justOrEmpty(mainTextContent(messageContent)) + .map(Preview::compute) + .switchIfEmpty(Mono.just(Preview.EMPTY)) + .map(extractedPreview -> MessageFastViewPrecomputedProperties.builder() + .preview(extractedPreview) + .hasAttachment(hasAttachments.get()) + .build()); + } + private Instant getDateFromHeaderOrInternalDateOtherwise(Message mimeMessage, MessageFullViewFactory.MetaDataWithContent message) { return Optional.ofNullable(mimeMessage.getDate()) .map(Date::toInstant) diff --git a/server/protocols/jmap-draft/src/test/java/org/apache/james/jmap/draft/methods/GetMessagesMethodTest.java b/server/protocols/jmap-draft/src/test/java/org/apache/james/jmap/draft/methods/GetMessagesMethodTest.java index 5a3686b..fdd7059 100644 --- a/server/protocols/jmap-draft/src/test/java/org/apache/james/jmap/draft/methods/GetMessagesMethodTest.java +++ b/server/protocols/jmap-draft/src/test/java/org/apache/james/jmap/draft/methods/GetMessagesMethodTest.java @@ -123,7 +123,8 @@ public class GetMessagesMethodTest { messageMetadataViewFactory = spy(new MessageMetadataViewFactory(blobManager, messageIdManager)); MessageFullViewFactory messageFullViewFactory = new MessageFullViewFactory(blobManager, messageContentExtractor, - htmlTextExtractor, messageIdManager); + htmlTextExtractor, messageIdManager, + new MemoryMessageFastViewProjection()); MessageFastViewFactory messageFastViewFactory = new MessageFastViewFactory(blobManager, messageIdManager, new MemoryMessageFastViewProjection(), messageFullViewFactory); diff --git a/server/protocols/jmap-draft/src/test/java/org/apache/james/jmap/draft/methods/MessageSenderTest.java b/server/protocols/jmap-draft/src/test/java/org/apache/james/jmap/draft/methods/MessageSenderTest.java index f8f768c..f4a7573 100644 --- a/server/protocols/jmap-draft/src/test/java/org/apache/james/jmap/draft/methods/MessageSenderTest.java +++ b/server/protocols/jmap-draft/src/test/java/org/apache/james/jmap/draft/methods/MessageSenderTest.java @@ -40,6 +40,7 @@ import org.apache.james.jmap.draft.model.message.view.MessageFullView; import org.apache.james.jmap.draft.model.message.view.MessageFullViewFactory; import org.apache.james.jmap.draft.model.message.view.MessageFullViewFactory.MetaDataWithContent; import org.apache.james.jmap.draft.utils.HtmlTextExtractor; +import org.apache.james.jmap.memory.projections.MemoryMessageFastViewProjection; import org.apache.james.mailbox.BlobManager; import org.apache.james.mailbox.MessageIdManager; import org.apache.james.mailbox.MessageUid; @@ -89,7 +90,8 @@ class MessageSenderTest { BlobManager blobManager = mock(BlobManager.class); when(blobManager.toBlobId(any(MessageId.class))).thenReturn(BlobId.fromString("fake")); MessageIdManager messageIdManager = mock(MessageIdManager.class); - MessageFullViewFactory messageFullViewFactory = new MessageFullViewFactory(blobManager, messageContentExtractor, htmlTextExtractor, messageIdManager); + MessageFullViewFactory messageFullViewFactory = new MessageFullViewFactory(blobManager, messageContentExtractor, htmlTextExtractor, messageIdManager, + new MemoryMessageFastViewProjection()); jmapMessage = messageFullViewFactory.fromMetaDataWithContent(message); envelope = EnvelopeUtils.fromMessage(jmapMessage); } diff --git a/server/protocols/jmap-draft/src/test/java/org/apache/james/jmap/draft/methods/SetMessagesCreationProcessorTest.java b/server/protocols/jmap-draft/src/test/java/org/apache/james/jmap/draft/methods/SetMessagesCreationProcessorTest.java index 92f0adb..6965932 100644 --- a/server/protocols/jmap-draft/src/test/java/org/apache/james/jmap/draft/methods/SetMessagesCreationProcessorTest.java +++ b/server/protocols/jmap-draft/src/test/java/org/apache/james/jmap/draft/methods/SetMessagesCreationProcessorTest.java @@ -43,6 +43,7 @@ import org.apache.james.jmap.draft.model.message.view.MessageFullViewFactory; import org.apache.james.jmap.draft.send.MailMetadata; import org.apache.james.jmap.draft.send.MailSpool; import org.apache.james.jmap.draft.utils.HtmlTextExtractor; +import org.apache.james.jmap.memory.projections.MemoryMessageFastViewProjection; import org.apache.james.mailbox.AttachmentManager; import org.apache.james.mailbox.BlobManager; import org.apache.james.mailbox.MailboxManager; @@ -125,7 +126,9 @@ public class SetMessagesCreationProcessorTest { BlobManager blobManager = mock(BlobManager.class); when(blobManager.toBlobId(any(MessageId.class))).thenReturn(org.apache.james.mailbox.model.BlobId.fromString("fake")); MessageIdManager messageIdManager = mock(MessageIdManager.class); - messageFullViewFactory = new MessageFullViewFactory(blobManager, messageContentExtractor, htmlTextExtractor, messageIdManager); + messageFullViewFactory = new MessageFullViewFactory(blobManager, messageContentExtractor, htmlTextExtractor, + messageIdManager, + new MemoryMessageFastViewProjection()); mockedMailSpool = mock(MailSpool.class); mockedAttachmentManager = mock(AttachmentManager.class); mockedMailboxManager = mock(MailboxManager.class); diff --git a/server/protocols/jmap-draft/src/test/java/org/apache/james/jmap/draft/model/message/view/MessageFastViewFactoryTest.java b/server/protocols/jmap-draft/src/test/java/org/apache/james/jmap/draft/model/message/view/MessageFastViewFactoryTest.java index a254fb9..c7da512 100644 --- a/server/protocols/jmap-draft/src/test/java/org/apache/james/jmap/draft/model/message/view/MessageFastViewFactoryTest.java +++ b/server/protocols/jmap-draft/src/test/java/org/apache/james/jmap/draft/model/message/view/MessageFastViewFactoryTest.java @@ -144,7 +144,8 @@ class MessageFastViewFactoryTest { .block(); blobManager = resources.getBlobManager(); - messageFullViewFactory = new MessageFullViewFactory(blobManager, messageContentExtractor, htmlTextExtractor, messageIdManager); + messageFullViewFactory = new MessageFullViewFactory(blobManager, messageContentExtractor, htmlTextExtractor, messageIdManager, + new MemoryMessageFastViewProjection()); messageFastViewFactory = new MessageFastViewFactory(blobManager, messageIdManager, fastViewProjection, messageFullViewFactory); } diff --git a/server/protocols/jmap-draft/src/test/java/org/apache/james/jmap/draft/model/message/view/MessageFullViewFactoryTest.java b/server/protocols/jmap-draft/src/test/java/org/apache/james/jmap/draft/model/message/view/MessageFullViewFactoryTest.java index 38363c6..c3fb39f 100644 --- a/server/protocols/jmap-draft/src/test/java/org/apache/james/jmap/draft/model/message/view/MessageFullViewFactoryTest.java +++ b/server/protocols/jmap-draft/src/test/java/org/apache/james/jmap/draft/model/message/view/MessageFullViewFactoryTest.java @@ -25,16 +25,21 @@ import static org.apache.james.jmap.draft.model.message.view.MessageViewFixture. import static org.apache.james.jmap.draft.model.message.view.MessageViewFixture.JACK_EMAIL; import static org.apache.james.jmap.draft.model.message.view.MessageViewFixture.JACOB_EMAIL; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.spy; import java.io.ByteArrayInputStream; import java.nio.charset.StandardCharsets; import java.time.Instant; +import java.util.List; import java.util.Optional; import javax.mail.Flags; import org.apache.commons.lang3.StringUtils; import org.apache.james.jmap.api.model.Preview; +import org.apache.james.jmap.api.projections.MessageFastViewPrecomputedProperties; import org.apache.james.jmap.draft.model.Attachment; import org.apache.james.jmap.draft.model.BlobId; import org.apache.james.jmap.draft.model.Emailer; @@ -45,6 +50,7 @@ import org.apache.james.jmap.draft.model.PreviewDTO; import org.apache.james.jmap.draft.model.message.view.MessageFullViewFactory.MetaDataWithContent; import org.apache.james.jmap.draft.utils.HtmlTextExtractor; import org.apache.james.jmap.draft.utils.JsoupHtmlTextExtractor; +import org.apache.james.jmap.memory.projections.MemoryMessageFastViewProjection; import org.apache.james.mailbox.MailboxSession; import org.apache.james.mailbox.MessageIdManager; import org.apache.james.mailbox.MessageManager; @@ -55,24 +61,32 @@ import org.apache.james.mailbox.inmemory.manager.InMemoryIntegrationResources; import org.apache.james.mailbox.model.AttachmentId; import org.apache.james.mailbox.model.Cid; import org.apache.james.mailbox.model.ComposedMessageId; +import org.apache.james.mailbox.model.FetchGroup; import org.apache.james.mailbox.model.MailboxId; import org.apache.james.mailbox.model.MailboxPath; import org.apache.james.mailbox.model.MessageAttachment; +import org.apache.james.mailbox.model.MessageId; import org.apache.james.mailbox.model.MessageRange; +import org.apache.james.mailbox.model.MessageResult; import org.apache.james.mailbox.model.TestMessageId; import org.apache.james.util.ClassLoaderUtils; import org.apache.james.util.mime.MessageContentExtractor; import org.assertj.core.api.SoftAssertions; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import reactor.core.publisher.Mono; + class MessageFullViewFactoryTest { private static final String FORWARDED = "forwarded"; private static final InMemoryId MAILBOX_ID = InMemoryId.of(18L); private static final Instant INTERNAL_DATE = Instant.parse("2012-02-03T14:30:42Z"); + private static final String DEFAULT_PREVIEW_AS_STRING = "blabla bloblo"; + private static final Preview DEFAULT_PREVIEW = Preview.from(DEFAULT_PREVIEW_AS_STRING); private MessageIdManager messageIdManager; private MailboxSession session; @@ -80,6 +94,7 @@ class MessageFullViewFactoryTest { private MessageManager bobMailbox; private ComposedMessageId message1; private MessageFullViewFactory messageFullViewFactory; + private MemoryMessageFastViewProjection fastViewProjection; @BeforeEach void setUp() throws Exception { @@ -103,7 +118,10 @@ class MessageFullViewFactoryTest { .build(ClassLoaderUtils.getSystemResourceAsSharedStream("fullMessage.eml")), session); - messageFullViewFactory = new MessageFullViewFactory(resources.getBlobManager(), messageContentExtractor, htmlTextExtractor, messageIdManager); + fastViewProjection = spy(new MemoryMessageFastViewProjection()); + messageFullViewFactory = new MessageFullViewFactory(resources.getBlobManager(), messageContentExtractor, htmlTextExtractor, + messageIdManager, + fastViewProjection); } @Test @@ -851,4 +869,62 @@ class MessageFullViewFactoryTest { MessageFullView testee = messageFullViewFactory.fromMetaDataWithContent(testMail); assertThat(testee.getKeywords()).containsAllEntriesOf(keywords.asMap()); } + + @Nested + class WithProjectionInvolvedTest { + + @Test + void fromMessageResultsShouldComputeWhenProjectionReturnEmpty() throws Exception { + List<MessageResult> messages = messageIdManager + .getMessages(ImmutableList.of(message1.getMessageId()), FetchGroup.FULL_CONTENT, session); + messageFullViewFactory.fromMessageResults(messages); + + assertThat(Mono.from(fastViewProjection.retrieve(message1.getMessageId())).block()) + .extracting(MessageFastViewPrecomputedProperties::getPreview) + .isEqualTo(DEFAULT_PREVIEW); + } + + @Test + void fromMessageResultsShouldUseReturnedPreviewFromProjections() throws Exception { + String preview = "my pre computed preview"; + Mono.from(fastViewProjection.store(message1.getMessageId(), MessageFastViewPrecomputedProperties.builder() + .preview(Preview.from(preview)) + .hasAttachment(false) + .build())) + .block(); + + List<MessageResult> messages = messageIdManager + .getMessages(ImmutableList.of(message1.getMessageId()), FetchGroup.FULL_CONTENT, session); + + assertThat(messageFullViewFactory.fromMessageResults(messages)) + .extracting(MessageFullView::getPreview) + .isEqualTo(PreviewDTO.of(preview)); + } + + @Test + void fromMessageResultsShouldFallbackToComputeWhenProjectionRetrievingError() throws Exception { + doReturn(Mono.error(new RuntimeException("mock exception"))) + .when(fastViewProjection).retrieve(any(MessageId.class)); + + List<MessageResult> messages = messageIdManager + .getMessages(ImmutableList.of(message1.getMessageId()), FetchGroup.FULL_CONTENT, session); + + assertThat(messageFullViewFactory.fromMessageResults(messages)) + .extracting(MessageFullView::getPreview) + .isEqualTo(PreviewDTO.of(DEFAULT_PREVIEW_AS_STRING)); + } + + @Test + void fromMessageResultsShouldNotBeAffectedByProjectionStoringError() throws Exception { + doReturn(Mono.error(new RuntimeException("mock exception"))) + .when(fastViewProjection).store(any(), any()); + + List<MessageResult> messages = messageIdManager + .getMessages(ImmutableList.of(message1.getMessageId()), FetchGroup.FULL_CONTENT, session); + + assertThat(messageFullViewFactory.fromMessageResults(messages)) + .extracting(MessageFullView::getPreview) + .isEqualTo(PreviewDTO.of(DEFAULT_PREVIEW_AS_STRING)); + } + } } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
