JAMES-1934 indexation handle invalid messageId instead of looking for backend indexation support
Project: http://git-wip-us.apache.org/repos/asf/james-project/repo Commit: http://git-wip-us.apache.org/repos/asf/james-project/commit/cdfe3f71 Tree: http://git-wip-us.apache.org/repos/asf/james-project/tree/cdfe3f71 Diff: http://git-wip-us.apache.org/repos/asf/james-project/diff/cdfe3f71 Branch: refs/heads/master Commit: cdfe3f71371c1cddc2be47d04d56380591823dd5 Parents: 983de8d Author: Luc DUZAN <[email protected]> Authored: Fri Feb 10 14:09:28 2017 +0100 Committer: Benoit Tellier <[email protected]> Committed: Wed Feb 15 06:59:46 2017 +0700 ---------------------------------------------------------------------- .../elasticsearch/json/IndexableMessage.java | 4 +- .../json/MessageToElasticSearchJson.java | 72 ++++----------- .../ElasticSearchIntegrationTest.java | 5 +- .../json/IndexableMessageTest.java | 93 +++++++++++++++++++ .../json/MessageToElasticSearchJsonTest.java | 52 +++++------ .../lucene/search/LuceneMessageSearchIndex.java | 28 +++--- .../LuceneMailboxMessageSearchIndexTest.java | 3 +- .../store/search/MessageSearchIndex.java | 5 -- .../james/mailbox/store/search/SearchUtil.java | 14 +++ .../mailbox/store/search/SearchUtilTest.java | 94 +++++++++++++++++++- .../host/ElasticSearchHostSystem.java | 3 +- 11 files changed, 255 insertions(+), 118 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/james-project/blob/cdfe3f71/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/IndexableMessage.java ---------------------------------------------------------------------- diff --git a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/IndexableMessage.java b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/IndexableMessage.java index c6e3ce7..e5ad24e 100644 --- a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/IndexableMessage.java +++ b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/IndexableMessage.java @@ -36,6 +36,7 @@ import org.apache.james.mailbox.store.mail.model.MailboxMessage; import org.apache.james.mailbox.store.mail.model.Property; import org.apache.james.mailbox.store.mail.model.impl.PropertyBuilder; import org.apache.james.mailbox.store.mail.model.impl.SimpleProperty; +import org.apache.james.mailbox.store.search.SearchUtil; import org.apache.james.mime4j.MimeException; import com.fasterxml.jackson.annotation.JsonProperty; @@ -69,7 +70,6 @@ public class IndexableMessage { public IndexableMessage build() { Preconditions.checkNotNull(message.getMailboxId()); - Preconditions.checkNotNull(message.getMessageId()); Preconditions.checkNotNull(users); Preconditions.checkNotNull(textExtractor); Preconditions.checkNotNull(indexAttachments); @@ -115,7 +115,7 @@ public class IndexableMessage { } private IndexableMessage instanciateIndexedMessage() throws IOException, MimeException { - String messageId = message.getMessageId().serialize(); + String messageId = SearchUtil.getSerializedMessageIdIfSupportedByUnderlyingStorageOrNull(message); MimePart parsingResult = new MimePartParser(message, textExtractor).parse(); List<String> stringifiedUsers = users.stream() http://git-wip-us.apache.org/repos/asf/james-project/blob/cdfe3f71/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/MessageToElasticSearchJson.java ---------------------------------------------------------------------- diff --git a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/MessageToElasticSearchJson.java b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/MessageToElasticSearchJson.java index d4c896e..aacd790 100644 --- a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/MessageToElasticSearchJson.java +++ b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/MessageToElasticSearchJson.java @@ -25,13 +25,11 @@ import java.util.List; import javax.inject.Inject; import javax.mail.Flags; -import org.apache.commons.lang.NotImplementedException; import org.apache.james.mailbox.MailboxManager; import org.apache.james.mailbox.MailboxSession.User; import org.apache.james.mailbox.elasticsearch.IndexAttachments; import org.apache.james.mailbox.extractor.TextExtractor; import org.apache.james.mailbox.store.mail.model.MailboxMessage; -import org.apache.james.mailbox.store.search.MessageSearchIndex; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; @@ -45,81 +43,45 @@ public class MessageToElasticSearchJson { private final TextExtractor textExtractor; private final ZoneId zoneId; private final IndexAttachments indexAttachments; - private final MessageSearchIndex.IndexMessageId indexMessageId; - public MessageToElasticSearchJson(TextExtractor textExtractor, ZoneId zoneId, IndexAttachments indexAttachments, MessageSearchIndex.IndexMessageId indexMessageId) { + public MessageToElasticSearchJson(TextExtractor textExtractor, ZoneId zoneId, IndexAttachments indexAttachments) { this.textExtractor = textExtractor; this.zoneId = zoneId; this.indexAttachments = indexAttachments; this.mapper = new ObjectMapper(); this.mapper.registerModule(new GuavaModule()); this.mapper.registerModule(new Jdk8Module()); - this.indexMessageId = indexMessageId; } @Inject public MessageToElasticSearchJson(TextExtractor textExtractor, IndexAttachments indexAttachments, MailboxManager mailboxManager) { - this(textExtractor, ZoneId.systemDefault(), indexAttachments, indexMessageId(mailboxManager)); - } - - private static MessageSearchIndex.IndexMessageId indexMessageId(MailboxManager mailboxManager) { - if (mailboxManager.getSupportedMessageCapabilities().contains(MailboxManager.MessageCapabilities.UniqueID)) { - return MessageSearchIndex.IndexMessageId.Required; - } - return MessageSearchIndex.IndexMessageId.Optional; + this(textExtractor, ZoneId.systemDefault(), indexAttachments); } public String convertToJson(MailboxMessage message, List<User> users) throws JsonProcessingException { Preconditions.checkNotNull(message); - switch (indexMessageId) { - case Required: - return mapper.writeValueAsString(IndexableMessage.builder() - .message(message) - .users(users) - .extractor(textExtractor) - .zoneId(zoneId) - .indexAttachments(indexAttachments) - .build()); - case Optional: - return mapper.writeValueAsString(IndexableMessage.builder() - .message(message) - .users(users) - .extractor(textExtractor) - .zoneId(zoneId) - .indexAttachments(indexAttachments) - .build()); - default: - throw new NotImplementedException(); - } + + return mapper.writeValueAsString(IndexableMessage.builder() + .message(message) + .users(users) + .extractor(textExtractor) + .zoneId(zoneId) + .indexAttachments(indexAttachments) + .build()); } public String convertToJsonWithoutAttachment(MailboxMessage message, List<User> users) throws JsonProcessingException { - Preconditions.checkNotNull(message); - switch (indexMessageId) { - case Required: - return mapper.writeValueAsString(IndexableMessage.builder() - .message(message) - .users(users) - .extractor(textExtractor) - .zoneId(zoneId) - .indexAttachments(IndexAttachments.NO) - .build()); - case Optional: - return mapper.writeValueAsString(IndexableMessage.builder() - .message(message) - .users(users) - .extractor(textExtractor) - .zoneId(zoneId) - .indexAttachments(IndexAttachments.NO) - .build()); - default: - throw new NotImplementedException(); - } + return mapper.writeValueAsString(IndexableMessage.builder() + .message(message) + .users(users) + .extractor(textExtractor) + .zoneId(zoneId) + .indexAttachments(IndexAttachments.NO) + .build()); } public String getUpdatedJsonMessagePart(Flags flags, long modSeq) throws JsonProcessingException { Preconditions.checkNotNull(flags); return mapper.writeValueAsString(new MessageUpdateJson(flags, modSeq)); } - } http://git-wip-us.apache.org/repos/asf/james-project/blob/cdfe3f71/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/ElasticSearchIntegrationTest.java ---------------------------------------------------------------------- diff --git a/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/ElasticSearchIntegrationTest.java b/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/ElasticSearchIntegrationTest.java index c406419..7ae3d09 100644 --- a/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/ElasticSearchIntegrationTest.java +++ b/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/ElasticSearchIntegrationTest.java @@ -46,7 +46,6 @@ import org.apache.james.mailbox.store.MailboxSessionMapperFactory; import org.apache.james.mailbox.store.extractor.DefaultTextExtractor; import org.apache.james.mailbox.store.mail.model.impl.MessageParser; import org.apache.james.mailbox.store.search.AbstractMessageSearchIndexTest; -import org.apache.james.mailbox.store.search.MessageSearchIndex; import org.elasticsearch.client.Client; import org.junit.Rule; import org.junit.rules.RuleChain; @@ -90,7 +89,7 @@ public class ElasticSearchIntegrationTest extends AbstractMessageSearchIndexTest MailboxElasticsearchConstants.MAILBOX_INDEX, MailboxElasticsearchConstants.MESSAGE_TYPE), new ElasticSearchSearcher(client, new QueryConverter(new CriterionConverter()), SEARCH_SIZE, new InMemoryId.Factory(), messageIdFactory), - new MessageToElasticSearchJson(new DefaultTextExtractor(), ZoneId.of("Europe/Paris"), IndexAttachments.YES, MessageSearchIndex.IndexMessageId.Required)); + new MessageToElasticSearchJson(new DefaultTextExtractor(), ZoneId.of("Europe/Paris"), IndexAttachments.YES)); storeMailboxManager = new InMemoryMailboxManager( mapperFactory, new FakeAuthenticator(), @@ -103,4 +102,4 @@ public class ElasticSearchIntegrationTest extends AbstractMessageSearchIndexTest storeMailboxManager.setMessageSearchIndex(messageSearchIndex); storeMailboxManager.init(); } -} +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/james-project/blob/cdfe3f71/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/json/IndexableMessageTest.java ---------------------------------------------------------------------- diff --git a/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/json/IndexableMessageTest.java b/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/json/IndexableMessageTest.java index 56aedb3..726eb69 100644 --- a/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/json/IndexableMessageTest.java +++ b/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/json/IndexableMessageTest.java @@ -37,6 +37,7 @@ import org.apache.james.mailbox.extractor.ParsedContent; import org.apache.james.mailbox.extractor.TextExtractor; import org.apache.james.mailbox.inmemory.InMemoryMessageId; import org.apache.james.mailbox.mock.MockMailboxSession; +import org.apache.james.mailbox.model.MessageId; import org.apache.james.mailbox.model.TestId; import org.apache.james.mailbox.store.extractor.DefaultTextExtractor; import org.apache.james.mailbox.store.mail.model.MailboxMessage; @@ -482,5 +483,97 @@ public class IndexableMessageTest { // Then assertThat(indexableMessage.getText()).contains("subject should be parsed"); } + + @Test + public void shouldHandleCorrectlyMessageIdHavingSerializeMethodThatThrowAnException() throws Exception { + MessageId invalidMessageIdThatThrowException = mock(MessageId.class); + when(invalidMessageIdThatThrowException.serialize()) + .thenThrow(new UnsupportedOperationException()); + + // When + MailboxMessage mailboxMessage = mock(MailboxMessage.class); + TestId mailboxId = TestId.of(1); + when(mailboxMessage.getMailboxId()) + .thenReturn(mailboxId); + when(mailboxMessage.getMessageId()) + .thenReturn(invalidMessageIdThatThrowException); + when(mailboxMessage.getFullContent()) + .thenReturn(new ByteArrayInputStream(IOUtils.toByteArray(ClassLoader.getSystemResourceAsStream("eml/bodyMakeTikaToFail.eml")))); + when(mailboxMessage.createFlags()) + .thenReturn(new Flags()); + when(mailboxMessage.getUid()) + .thenReturn(MESSAGE_UID); + IndexableMessage indexableMessage = IndexableMessage.builder() + .message(mailboxMessage) + .users(ImmutableList.of(new MockMailboxSession("username").getUser())) + .extractor(new TikaTextExtractor()) + .zoneId(ZoneId.of("Europe/Paris")) + .indexAttachments(IndexAttachments.YES) + .build(); + + // Then + assertThat(indexableMessage.getMessageId()).isNull(); + } + + @Test + public void shouldHandleCorrectlyMessageIdHavingSerializeMethodThatReturnNull() throws Exception { + MessageId invalidMessageIdThatReturnNull = mock(MessageId.class); + when(invalidMessageIdThatReturnNull.serialize()) + .thenReturn(null); + + // When + MailboxMessage mailboxMessage = mock(MailboxMessage.class); + TestId mailboxId = TestId.of(1); + when(mailboxMessage.getMailboxId()) + .thenReturn(mailboxId); + when(mailboxMessage.getMessageId()) + .thenReturn(invalidMessageIdThatReturnNull); + when(mailboxMessage.getFullContent()) + .thenReturn(new ByteArrayInputStream(IOUtils.toByteArray(ClassLoader.getSystemResourceAsStream("eml/bodyMakeTikaToFail.eml")))); + when(mailboxMessage.createFlags()) + .thenReturn(new Flags()); + when(mailboxMessage.getUid()) + .thenReturn(MESSAGE_UID); + + IndexableMessage indexableMessage = IndexableMessage.builder() + .message(mailboxMessage) + .users(ImmutableList.of(new MockMailboxSession("username").getUser())) + .extractor(new TikaTextExtractor()) + .zoneId(ZoneId.of("Europe/Paris")) + .indexAttachments(IndexAttachments.YES) + .build(); + + // Then + assertThat(indexableMessage.getMessageId()).isNull(); + } + + @Test + public void shouldHandleCorrectlyNullMessageId() throws Exception { + + // When + MailboxMessage mailboxMessage = mock(MailboxMessage.class); + TestId mailboxId = TestId.of(1); + when(mailboxMessage.getMailboxId()) + .thenReturn(mailboxId); + when(mailboxMessage.getMessageId()) + .thenReturn(null); + when(mailboxMessage.getFullContent()) + .thenReturn(new ByteArrayInputStream(IOUtils.toByteArray(ClassLoader.getSystemResourceAsStream("eml/bodyMakeTikaToFail.eml")))); + when(mailboxMessage.createFlags()) + .thenReturn(new Flags()); + when(mailboxMessage.getUid()) + .thenReturn(MESSAGE_UID); + + IndexableMessage indexableMessage = IndexableMessage.builder() + .message(mailboxMessage) + .users(ImmutableList.of(new MockMailboxSession("username").getUser())) + .extractor(new TikaTextExtractor()) + .zoneId(ZoneId.of("Europe/Paris")) + .indexAttachments(IndexAttachments.YES) + .build(); + + // Then + assertThat(indexableMessage.getMessageId()).isNull(); + } } http://git-wip-us.apache.org/repos/asf/james-project/blob/cdfe3f71/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/json/MessageToElasticSearchJsonTest.java ---------------------------------------------------------------------- diff --git a/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/json/MessageToElasticSearchJsonTest.java b/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/json/MessageToElasticSearchJsonTest.java index f508766..ba27b58 100644 --- a/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/json/MessageToElasticSearchJsonTest.java +++ b/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/json/MessageToElasticSearchJsonTest.java @@ -45,7 +45,6 @@ import org.apache.james.mailbox.store.extractor.DefaultTextExtractor; import org.apache.james.mailbox.store.mail.model.MailboxMessage; import org.apache.james.mailbox.store.mail.model.impl.PropertyBuilder; import org.apache.james.mailbox.store.mail.model.impl.SimpleMailboxMessage; -import org.apache.james.mailbox.store.search.MessageSearchIndex; import org.apache.james.mailbox.tika.extractor.TikaTextExtractor; import org.junit.Before; import org.junit.Test; @@ -82,8 +81,7 @@ public class MessageToElasticSearchJsonTest { public void convertToJsonShouldThrowWhenNoUser() throws Exception { MessageToElasticSearchJson messageToElasticSearchJson = new MessageToElasticSearchJson( new DefaultTextExtractor(), - ZoneId.of("Europe/Paris"), IndexAttachments.YES, - MessageSearchIndex.IndexMessageId.Required); + ZoneId.of("Europe/Paris"), IndexAttachments.YES); MailboxMessage spamMail = new SimpleMailboxMessage(MESSAGE_ID, date, SIZE, @@ -102,8 +100,7 @@ public class MessageToElasticSearchJsonTest { public void spamEmailShouldBeWellConvertedToJson() throws IOException { MessageToElasticSearchJson messageToElasticSearchJson = new MessageToElasticSearchJson( new DefaultTextExtractor(), - ZoneId.of("Europe/Paris"), IndexAttachments.YES, - MessageSearchIndex.IndexMessageId.Required); + ZoneId.of("Europe/Paris"), IndexAttachments.YES); MailboxMessage spamMail = new SimpleMailboxMessage(MESSAGE_ID, date, SIZE, @@ -123,8 +120,7 @@ public class MessageToElasticSearchJsonTest { public void htmlEmailShouldBeWellConvertedToJson() throws IOException { MessageToElasticSearchJson messageToElasticSearchJson = new MessageToElasticSearchJson( new DefaultTextExtractor(), - ZoneId.of("Europe/Paris"), IndexAttachments.YES, - MessageSearchIndex.IndexMessageId.Required); + ZoneId.of("Europe/Paris"), IndexAttachments.YES); MailboxMessage htmlMail = new SimpleMailboxMessage(MESSAGE_ID, date, SIZE, @@ -144,8 +140,7 @@ public class MessageToElasticSearchJsonTest { public void pgpSignedEmailShouldBeWellConvertedToJson() throws IOException { MessageToElasticSearchJson messageToElasticSearchJson = new MessageToElasticSearchJson( new DefaultTextExtractor(), - ZoneId.of("Europe/Paris"), IndexAttachments.YES, - MessageSearchIndex.IndexMessageId.Required); + ZoneId.of("Europe/Paris"), IndexAttachments.YES); MailboxMessage pgpSignedMail = new SimpleMailboxMessage(MESSAGE_ID, date, SIZE, @@ -165,8 +160,7 @@ public class MessageToElasticSearchJsonTest { public void simpleEmailShouldBeWellConvertedToJson() throws IOException { MessageToElasticSearchJson messageToElasticSearchJson = new MessageToElasticSearchJson( new DefaultTextExtractor(), - ZoneId.of("Europe/Paris"), IndexAttachments.YES, - MessageSearchIndex.IndexMessageId.Required); + ZoneId.of("Europe/Paris"), IndexAttachments.YES); MailboxMessage mail = new SimpleMailboxMessage(MESSAGE_ID, date, SIZE, @@ -187,8 +181,7 @@ public class MessageToElasticSearchJsonTest { public void recursiveEmailShouldBeWellConvertedToJson() throws IOException { MessageToElasticSearchJson messageToElasticSearchJson = new MessageToElasticSearchJson( new DefaultTextExtractor(), - ZoneId.of("Europe/Paris"), IndexAttachments.YES, - MessageSearchIndex.IndexMessageId.Required); + ZoneId.of("Europe/Paris"), IndexAttachments.YES); MailboxMessage recursiveMail = new SimpleMailboxMessage(MESSAGE_ID, date, SIZE, @@ -208,8 +201,7 @@ public class MessageToElasticSearchJsonTest { public void emailWithNoInternalDateShouldUseNowDate() throws IOException { MessageToElasticSearchJson messageToElasticSearchJson = new MessageToElasticSearchJson( new DefaultTextExtractor(), - ZoneId.of("Europe/Paris"), IndexAttachments.YES, - MessageSearchIndex.IndexMessageId.Required); + ZoneId.of("Europe/Paris"), IndexAttachments.YES); MailboxMessage mailWithNoInternalDate = new SimpleMailboxMessage(MESSAGE_ID, null, SIZE, @@ -243,8 +235,8 @@ public class MessageToElasticSearchJsonTest { // When MessageToElasticSearchJson messageToElasticSearchJson = new MessageToElasticSearchJson( new DefaultTextExtractor(), - ZoneId.of("Europe/Paris"), IndexAttachments.YES, - MessageSearchIndex.IndexMessageId.Required); + ZoneId.of("Europe/Paris"), + IndexAttachments.YES); String convertToJson = messageToElasticSearchJson.convertToJson(mailWithNoInternalDate, ImmutableList.of(new MockMailboxSession("username").getUser())); // Then @@ -271,8 +263,8 @@ public class MessageToElasticSearchJsonTest { // When MessageToElasticSearchJson messageToElasticSearchJson = new MessageToElasticSearchJson( new DefaultTextExtractor(), - ZoneId.of("Europe/Paris"), IndexAttachments.NO, - MessageSearchIndex.IndexMessageId.Required); + ZoneId.of("Europe/Paris"), + IndexAttachments.NO); String convertToJson = messageToElasticSearchJson.convertToJson(mailWithNoInternalDate, ImmutableList.of(new MockMailboxSession("username").getUser())); // Then @@ -286,8 +278,7 @@ public class MessageToElasticSearchJsonTest { public void emailWithNoMailboxIdShouldThrow() throws IOException { MessageToElasticSearchJson messageToElasticSearchJson = new MessageToElasticSearchJson( new DefaultTextExtractor(), - ZoneId.of("Europe/Paris"), IndexAttachments.YES, - MessageSearchIndex.IndexMessageId.Required); + ZoneId.of("Europe/Paris"), IndexAttachments.YES); MailboxMessage mailWithNoMailboxId; try { mailWithNoMailboxId = new SimpleMailboxMessage(MESSAGE_ID, date, @@ -309,8 +300,8 @@ public class MessageToElasticSearchJsonTest { public void getUpdatedJsonMessagePartShouldBehaveWellOnEmptyFlags() throws Exception { MessageToElasticSearchJson messageToElasticSearchJson = new MessageToElasticSearchJson( new DefaultTextExtractor(), - ZoneId.of("Europe/Paris"), IndexAttachments.YES, - MessageSearchIndex.IndexMessageId.Required); + ZoneId.of("Europe/Paris"), + IndexAttachments.YES); assertThatJson(messageToElasticSearchJson.getUpdatedJsonMessagePart(new Flags(), MOD_SEQ)) .isEqualTo("{\"modSeq\":42,\"isAnswered\":false,\"isDeleted\":false,\"isDraft\":false,\"isFlagged\":false,\"isRecent\":false,\"userFlags\":[],\"isUnread\":true}"); } @@ -319,8 +310,8 @@ public class MessageToElasticSearchJsonTest { public void getUpdatedJsonMessagePartShouldBehaveWellOnNonEmptyFlags() throws Exception { MessageToElasticSearchJson messageToElasticSearchJson = new MessageToElasticSearchJson( new DefaultTextExtractor(), - ZoneId.of("Europe/Paris"), IndexAttachments.YES, - MessageSearchIndex.IndexMessageId.Required); + ZoneId.of("Europe/Paris"), + IndexAttachments.YES); assertThatJson(messageToElasticSearchJson.getUpdatedJsonMessagePart(new FlagsBuilder().add(Flags.Flag.DELETED, Flags.Flag.FLAGGED).add("user").build(), MOD_SEQ)) .isEqualTo("{\"modSeq\":42,\"isAnswered\":false,\"isDeleted\":true,\"isDraft\":false,\"isFlagged\":true,\"isRecent\":false,\"userFlags\":[\"user\"],\"isUnread\":true}"); } @@ -329,8 +320,8 @@ public class MessageToElasticSearchJsonTest { public void getUpdatedJsonMessagePartShouldThrowIfFlagsIsNull() throws Exception { MessageToElasticSearchJson messageToElasticSearchJson = new MessageToElasticSearchJson( new DefaultTextExtractor(), - ZoneId.of("Europe/Paris"), IndexAttachments.YES, - MessageSearchIndex.IndexMessageId.Required); + ZoneId.of("Europe/Paris"), + IndexAttachments.YES); messageToElasticSearchJson.getUpdatedJsonMessagePart(null, MOD_SEQ); } @@ -338,8 +329,8 @@ public class MessageToElasticSearchJsonTest { public void spamEmailShouldBeWellConvertedToJsonWithApacheTika() throws IOException { MessageToElasticSearchJson messageToElasticSearchJson = new MessageToElasticSearchJson( new TikaTextExtractor(), - ZoneId.of("Europe/Paris"), IndexAttachments.YES, - MessageSearchIndex.IndexMessageId.Required); + ZoneId.of("Europe/Paris"), + IndexAttachments.YES); MailboxMessage spamMail = new SimpleMailboxMessage(MESSAGE_ID, date, SIZE, BODY_START_OCTET, @@ -372,8 +363,7 @@ public class MessageToElasticSearchJsonTest { MessageToElasticSearchJson messageToElasticSearchJson = new MessageToElasticSearchJson( new DefaultTextExtractor(), ZoneId.of("Europe/Paris"), - IndexAttachments.NO, - MessageSearchIndex.IndexMessageId.Required); + IndexAttachments.NO); String convertToJsonWithoutAttachment = messageToElasticSearchJson.convertToJsonWithoutAttachment(message, ImmutableList.of(new MockMailboxSession("username").getUser())); // Then http://git-wip-us.apache.org/repos/asf/james-project/blob/cdfe3f71/mailbox/lucene/src/main/java/org/apache/james/mailbox/lucene/search/LuceneMessageSearchIndex.java ---------------------------------------------------------------------- diff --git a/mailbox/lucene/src/main/java/org/apache/james/mailbox/lucene/search/LuceneMessageSearchIndex.java b/mailbox/lucene/src/main/java/org/apache/james/mailbox/lucene/search/LuceneMessageSearchIndex.java index eb7ec7c..b46f9ad 100644 --- a/mailbox/lucene/src/main/java/org/apache/james/mailbox/lucene/search/LuceneMessageSearchIndex.java +++ b/mailbox/lucene/src/main/java/org/apache/james/mailbox/lucene/search/LuceneMessageSearchIndex.java @@ -70,7 +70,6 @@ import org.apache.james.mailbox.store.mail.model.Mailbox; import org.apache.james.mailbox.store.mail.model.MailboxMessage; import org.apache.james.mailbox.store.mail.model.impl.PropertyBuilder; import org.apache.james.mailbox.store.search.ListeningMessageSearchIndex; -import org.apache.james.mailbox.store.search.MessageSearchIndex; import org.apache.james.mailbox.store.search.SearchUtil; import org.apache.james.mime4j.MimeException; import org.apache.james.mime4j.dom.Header; @@ -358,7 +357,6 @@ public class LuceneMessageSearchIndex extends ListeningMessageSearchIndex { private final MailboxId.Factory mailboxIdFactory; private final MessageId.Factory messageIdFactory; private final IndexWriter writer; - private final IndexMessageId shouldIndexMessageId; private int maxQueryResults = DEFAULT_MAX_QUERY_RESULTS; @@ -366,31 +364,29 @@ public class LuceneMessageSearchIndex extends ListeningMessageSearchIndex { @Inject public LuceneMessageSearchIndex(MessageMapperFactory factory, MailboxId.Factory mailboxIdFactory, Directory directory, MessageId.Factory messageIdFactory, MailboxManager mailboxManager) throws CorruptIndexException, LockObtainFailedException, IOException { - this(factory, mailboxIdFactory, directory, false, true, messageIdFactory, indexMessageId(mailboxManager)); + this(factory, mailboxIdFactory, directory, false, true, messageIdFactory); } - private static MessageSearchIndex.IndexMessageId indexMessageId(MailboxManager mailboxManager) { - if (mailboxManager.getSupportedMessageCapabilities().contains(MailboxManager.MessageCapabilities.UniqueID)) { - return MessageSearchIndex.IndexMessageId.Required; - } - return MessageSearchIndex.IndexMessageId.Optional; - } - - public LuceneMessageSearchIndex(MessageMapperFactory factory, MailboxId.Factory mailboxIdFactory, Directory directory, boolean dropIndexOnStart, boolean lenient, MessageId.Factory messageIdFactory, IndexMessageId shouldIndexMessageId) throws CorruptIndexException, LockObtainFailedException, IOException { + public LuceneMessageSearchIndex( + MessageMapperFactory factory, + MailboxId.Factory mailboxIdFactory, + Directory directory, + boolean dropIndexOnStart, + boolean lenient, + MessageId.Factory messageIdFactory) + throws CorruptIndexException, LockObtainFailedException, IOException { super(factory); this.mailboxIdFactory = mailboxIdFactory; this.messageIdFactory = messageIdFactory; this.writer = new IndexWriter(directory, createConfig(createAnalyzer(lenient), dropIndexOnStart)); - this.shouldIndexMessageId = shouldIndexMessageId; } - public LuceneMessageSearchIndex(MessageMapperFactory factory, MailboxId.Factory mailboxIdFactory, MessageId.Factory messageIdFactory, IndexWriter writer, IndexMessageId shouldIndexMessageId) { + public LuceneMessageSearchIndex(MessageMapperFactory factory, MailboxId.Factory mailboxIdFactory, MessageId.Factory messageIdFactory, IndexWriter writer) { super(factory); this.mailboxIdFactory = mailboxIdFactory; this.messageIdFactory = messageIdFactory; this.writer = writer; - this.shouldIndexMessageId = shouldIndexMessageId; } @Override @@ -561,9 +557,7 @@ public class LuceneMessageSearchIndex extends ListeningMessageSearchIndex { doc.add(new Field(MAILBOX_ID_FIELD, membership.getMailboxId().serialize().toUpperCase(Locale.ENGLISH), Store.YES, Index.NOT_ANALYZED)); doc.add(new NumericField(UID_FIELD,Store.YES, true).setLongValue(membership.getUid().asLong())); doc.add(new Field(HAS_ATTACHMENT_FIELD, Boolean.toString(hasAttachment(membership)), Store.YES, Index.NOT_ANALYZED)); - if (shouldIndexMessageId == IndexMessageId.Required) { - doc.add(new Field(MESSAGE_ID_FIELD, membership.getMessageId().serialize(), Store.YES, Index.NOT_ANALYZED)); - } + doc.add(new Field(MESSAGE_ID_FIELD, SearchUtil.getSerializedMessageIdIfSupportedByUnderlyingStorageOrNull(membership), Store.YES, Index.NOT_ANALYZED)); // create an unqiue key for the document which can be used later on updates to find the document doc.add(new Field(ID_FIELD, membership.getMailboxId().serialize().toUpperCase(Locale.ENGLISH) +"-" + Long.toString(membership.getUid().asLong()), Store.YES, Index.NOT_ANALYZED)); http://git-wip-us.apache.org/repos/asf/james-project/blob/cdfe3f71/mailbox/lucene/src/test/java/org/apache/james/mailbox/lucene/search/LuceneMailboxMessageSearchIndexTest.java ---------------------------------------------------------------------- diff --git a/mailbox/lucene/src/test/java/org/apache/james/mailbox/lucene/search/LuceneMailboxMessageSearchIndexTest.java b/mailbox/lucene/src/test/java/org/apache/james/mailbox/lucene/search/LuceneMailboxMessageSearchIndexTest.java index e93cdb6..d8c39e2 100644 --- a/mailbox/lucene/src/test/java/org/apache/james/mailbox/lucene/search/LuceneMailboxMessageSearchIndexTest.java +++ b/mailbox/lucene/src/test/java/org/apache/james/mailbox/lucene/search/LuceneMailboxMessageSearchIndexTest.java @@ -50,7 +50,6 @@ import org.apache.james.mailbox.model.TestMessageId; import org.apache.james.mailbox.store.MessageBuilder; import org.apache.james.mailbox.store.SimpleMailboxMembership; import org.apache.james.mailbox.store.mail.model.Mailbox; -import org.apache.james.mailbox.store.search.MessageSearchIndex; import org.apache.lucene.store.RAMDirectory; import org.junit.Before; import org.junit.Test; @@ -100,7 +99,7 @@ public class LuceneMailboxMessageSearchIndexTest { id3 = factory.generate(); id4 = factory.generate(); id5 = factory.generate(); - index = new LuceneMessageSearchIndex(null, new TestId.Factory(), new RAMDirectory(), true, useLenient(), factory, MessageSearchIndex.IndexMessageId.Required); + index = new LuceneMessageSearchIndex(null, new TestId.Factory(), new RAMDirectory(), true, useLenient(), factory); index.setEnableSuffixMatch(true); Map<String, String> headersSubject = new HashMap<String, String>(); headersSubject.put("Subject", "test (fwd)"); http://git-wip-us.apache.org/repos/asf/james-project/blob/cdfe3f71/mailbox/store/src/main/java/org/apache/james/mailbox/store/search/MessageSearchIndex.java ---------------------------------------------------------------------- diff --git a/mailbox/store/src/main/java/org/apache/james/mailbox/store/search/MessageSearchIndex.java b/mailbox/store/src/main/java/org/apache/james/mailbox/store/search/MessageSearchIndex.java index 162598a..08adfd3 100644 --- a/mailbox/store/src/main/java/org/apache/james/mailbox/store/search/MessageSearchIndex.java +++ b/mailbox/store/src/main/java/org/apache/james/mailbox/store/search/MessageSearchIndex.java @@ -43,11 +43,6 @@ import com.google.common.base.Optional; */ public interface MessageSearchIndex { - enum IndexMessageId { - Required, - Optional - } - /** * Return all uids of the previous indexed {@link Mailbox}'s which match the {@link SearchQuery} */ http://git-wip-us.apache.org/repos/asf/james-project/blob/cdfe3f71/mailbox/store/src/main/java/org/apache/james/mailbox/store/search/SearchUtil.java ---------------------------------------------------------------------- diff --git a/mailbox/store/src/main/java/org/apache/james/mailbox/store/search/SearchUtil.java b/mailbox/store/src/main/java/org/apache/james/mailbox/store/search/SearchUtil.java index 4a9a721..2c899fa 100644 --- a/mailbox/store/src/main/java/org/apache/james/mailbox/store/search/SearchUtil.java +++ b/mailbox/store/src/main/java/org/apache/james/mailbox/store/search/SearchUtil.java @@ -21,6 +21,8 @@ package org.apache.james.mailbox.store.search; import java.nio.charset.Charset; import java.util.Locale; +import org.apache.james.mailbox.model.MessageId; +import org.apache.james.mailbox.store.mail.model.MailboxMessage; import org.apache.james.mime4j.codec.DecodeMonitor; import org.apache.james.mime4j.codec.DecoderUtil; import org.apache.james.mime4j.dom.address.Address; @@ -171,6 +173,18 @@ public class SearchUtil { } return ""; } + + + public static String getSerializedMessageIdIfSupportedByUnderlyingStorageOrNull(MailboxMessage mailboxMessage) { + try { + MessageId messageId = mailboxMessage.getMessageId(); + if (messageId != null) { + return messageId.serialize(); + } + } catch(UnsupportedOperationException e) {} + + return null; + } /** http://git-wip-us.apache.org/repos/asf/james-project/blob/cdfe3f71/mailbox/store/src/test/java/org/apache/james/mailbox/store/search/SearchUtilTest.java ---------------------------------------------------------------------- diff --git a/mailbox/store/src/test/java/org/apache/james/mailbox/store/search/SearchUtilTest.java b/mailbox/store/src/test/java/org/apache/james/mailbox/store/search/SearchUtilTest.java index 67b9066..542ef9c 100644 --- a/mailbox/store/src/test/java/org/apache/james/mailbox/store/search/SearchUtilTest.java +++ b/mailbox/store/src/test/java/org/apache/james/mailbox/store/search/SearchUtilTest.java @@ -18,8 +18,13 @@ ****************************************************************/ package org.apache.james.mailbox.store.search; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import org.apache.james.mailbox.model.MessageId; +import org.apache.james.mailbox.store.mail.model.MailboxMessage; import org.junit.Test; public class SearchUtilTest { @@ -65,4 +70,91 @@ public class SearchUtilTest { public void testComplexExtraction() { assertEquals("Test", SearchUtil.getBaseSubject("Re: re:re: fwd:[fwd: \t Test] (fwd) (fwd)(fwd) ")); } + + @Test + public void getMessageIdIfSupportedByUnderlyingStorageOrNullForNullMessageIdShouldReturnNull() { + //given + MailboxMessage message = mock(MailboxMessage.class); + when(message.getMessageId()) + .thenReturn(null); + + //when + String serialiazedMessageId = SearchUtil.getSerializedMessageIdIfSupportedByUnderlyingStorageOrNull(message); + + //expect + assertNull(serialiazedMessageId); + } + + @Test + public void getSerializedMessageIdIfSupportedByUnderlyingStorageOrNullForMessageIdThatSerializeReturnNullShouldReturnNull() { + //given + MessageId invalidMessageIdThatReturnNull = mock(MessageId.class); + when(invalidMessageIdThatReturnNull.serialize()) + .thenReturn(null); + + MailboxMessage message = mock(MailboxMessage.class); + when(message.getMessageId()) + .thenReturn(invalidMessageIdThatReturnNull); + + //when + String serialiazedMessageId = SearchUtil.getSerializedMessageIdIfSupportedByUnderlyingStorageOrNull(message); + + //expect + assertNull(serialiazedMessageId); + } + + @Test + public void getSerializedMessageIdIfSupportedByUnderlyingStorageOrNullForMessageIdThatSerializeReturnUnsupportedOperationExceptionShouldReturnNull() { + //given + MessageId invalidMessageIdThatThrowException = mock(MessageId.class); + when(invalidMessageIdThatThrowException.serialize()) + .thenThrow(new UnsupportedOperationException()); + + MailboxMessage message = mock(MailboxMessage.class); + when(message.getMessageId()) + .thenReturn(invalidMessageIdThatThrowException); + + //when + String serialiazedMessageId = SearchUtil.getSerializedMessageIdIfSupportedByUnderlyingStorageOrNull(message); + + //expect + assertNull(serialiazedMessageId); + } + + @Test + public void getSerializedMessageIdIfSupportedByUnderlyingStorageOrNullForValidMessageIdShouldReturnSerializedId() { + //given + String messageIdString = "http://www.labraxeenne.com/#/";; + MessageId messageId = mock(MessageId.class); + when(messageId.serialize()) + .thenReturn(messageIdString); + + MailboxMessage message = mock(MailboxMessage.class); + when(message.getMessageId()) + .thenReturn(messageId); + + //when + String serialiazedMessageId = SearchUtil.getSerializedMessageIdIfSupportedByUnderlyingStorageOrNull(message); + + //expect + assertEquals(serialiazedMessageId, messageIdString); + } + + @Test + public void getSerializedMessageIdIfSupportedByUnderlyingStorageOrNullForMessageIdThatReturnUssuportedOperationException() { + //given + MessageId messageId = mock(MessageId.class); + when(messageId.serialize()) + .thenThrow(new UnsupportedOperationException()); + + MailboxMessage message = mock(MailboxMessage.class); + when(message.getMessageId()) + .thenReturn(messageId); + + //when + String serialiazedMessageId = SearchUtil.getSerializedMessageIdIfSupportedByUnderlyingStorageOrNull(message); + + //expect + assertEquals(serialiazedMessageId, null); + } } http://git-wip-us.apache.org/repos/asf/james-project/blob/cdfe3f71/mpt/impl/imap-mailbox/elasticsearch/src/test/java/org/apache/james/mpt/imapmailbox/elasticsearch/host/ElasticSearchHostSystem.java ---------------------------------------------------------------------- diff --git a/mpt/impl/imap-mailbox/elasticsearch/src/test/java/org/apache/james/mpt/imapmailbox/elasticsearch/host/ElasticSearchHostSystem.java b/mpt/impl/imap-mailbox/elasticsearch/src/test/java/org/apache/james/mpt/imapmailbox/elasticsearch/host/ElasticSearchHostSystem.java index a745032..72e93b0 100644 --- a/mpt/impl/imap-mailbox/elasticsearch/src/test/java/org/apache/james/mpt/imapmailbox/elasticsearch/host/ElasticSearchHostSystem.java +++ b/mpt/impl/imap-mailbox/elasticsearch/src/test/java/org/apache/james/mpt/imapmailbox/elasticsearch/host/ElasticSearchHostSystem.java @@ -59,7 +59,6 @@ import org.apache.james.mailbox.store.extractor.DefaultTextExtractor; import org.apache.james.mailbox.store.mail.model.impl.MessageParser; import org.apache.james.mailbox.store.quota.DefaultQuotaRootResolver; import org.apache.james.mailbox.store.quota.NoQuotaManager; -import org.apache.james.mailbox.store.search.MessageSearchIndex; import org.apache.james.mpt.api.ImapFeatures; import org.apache.james.mpt.api.ImapFeatures.Feature; import org.apache.james.mpt.host.JamesImapHostSystem; @@ -111,7 +110,7 @@ public class ElasticSearchHostSystem extends JamesImapHostSystem { factory, new ElasticSearchIndexer(client, new DeleteByQueryPerformer(client, Executors.newSingleThreadExecutor(), MailboxElasticsearchConstants.MAILBOX_INDEX, MailboxElasticsearchConstants.MESSAGE_TYPE), MailboxElasticsearchConstants.MAILBOX_INDEX, MailboxElasticsearchConstants.MESSAGE_TYPE), new ElasticSearchSearcher(client, new QueryConverter(new CriterionConverter()), new InMemoryId.Factory(), messageIdFactory), - new MessageToElasticSearchJson(new DefaultTextExtractor(), ZoneId.systemDefault(), IndexAttachments.YES, MessageSearchIndex.IndexMessageId.Required)); + new MessageToElasticSearchJson(new DefaultTextExtractor(), ZoneId.systemDefault(), IndexAttachments.YES)); MailboxACLResolver aclResolver = new UnionMailboxACLResolver(); GroupMembershipResolver groupMembershipResolver = new SimpleGroupMembershipResolver(); --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
