JAMES-1894 Rely on MessageID Capability to index MessageId
Project: http://git-wip-us.apache.org/repos/asf/james-project/repo Commit: http://git-wip-us.apache.org/repos/asf/james-project/commit/941a7039 Tree: http://git-wip-us.apache.org/repos/asf/james-project/tree/941a7039 Diff: http://git-wip-us.apache.org/repos/asf/james-project/diff/941a7039 Branch: refs/heads/master Commit: 941a70391b1ad1055f2ca89ae5f306e6cf79330d Parents: 0606d2e Author: Benoit Tellier <btell...@linagora.com> Authored: Fri Dec 23 19:24:18 2016 +0700 Committer: Benoit Tellier <btell...@linagora.com> Committed: Mon Jan 9 22:01:07 2017 +0700 ---------------------------------------------------------------------- ...lasticSearchListeningMessageSearchIndex.java | 1 + .../elasticsearch/json/IndexableMessage.java | 53 ++++++++++---- .../json/IndexableMessageWithMessageId.java | 72 ++++++++++++++++++++ .../json/MessageToElasticSearchJson.java | 27 ++++++-- .../search/ElasticSearchSearcher.java | 24 +++++-- .../ElasticSearchIntegrationTest.java | 3 +- .../json/IndexableMessageTest.java | 23 +++++++ .../MailboxMessageToElasticSearchJsonTest.java | 43 ++++++++---- .../lucene/search/LuceneMessageSearchIndex.java | 45 ++++++++---- .../META-INF/spring/mailbox-index-lucene.xml | 1 + .../LuceneMailboxMessageSearchIndexTest.java | 3 +- .../search/LuceneMessageSearchIndexTest.java | 2 +- .../META-INF/spring/mailbox-index-lucene.xml | 2 + .../store/search/MessageSearchIndex.java | 13 +++- .../mailbox/store/search/MessageSearches.java | 3 +- .../store/search/SimpleMessageSearchIndex.java | 2 +- .../store/src/test/resources/eml/htmlMail.json | 2 +- mailbox/store/src/test/resources/eml/mail.json | 2 +- .../src/test/resources/eml/nonTextual.json | 2 +- .../src/test/resources/eml/pgpSignedMail.json | 2 +- .../src/test/resources/eml/recursiveMail.json | 2 +- .../eml/recursiveMailWithoutAttachments.json | 2 +- .../store/src/test/resources/eml/spamMail.json | 2 +- .../host/ElasticSearchHostSystem.java | 4 +- .../host/LuceneSearchHostSystem.java | 7 +- 25 files changed, 274 insertions(+), 68 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/james-project/blob/941a7039/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/events/ElasticSearchListeningMessageSearchIndex.java ---------------------------------------------------------------------- diff --git a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/events/ElasticSearchListeningMessageSearchIndex.java b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/events/ElasticSearchListeningMessageSearchIndex.java index 281f429..850dd4c 100644 --- a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/events/ElasticSearchListeningMessageSearchIndex.java +++ b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/events/ElasticSearchListeningMessageSearchIndex.java @@ -99,6 +99,7 @@ public class ElasticSearchListeningMessageSearchIndex extends ListeningMessageSe Preconditions.checkArgument(session != null, "'session' is mandatory"); return searcher.search(ImmutableList.of(session.getUser()), searchQuery, Optional.of(limit)) .map(SearchResult::getMessageId) + .map(com.google.common.base.Optional::get) .collect(Guavate.toImmutableList()); } http://git-wip-us.apache.org/repos/asf/james-project/blob/941a7039/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/IndexableMessage.java ---------------------------------------------------------------------- diff --git a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/IndexableMessage.java b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/IndexableMessage.java index dec8b8e..770f558 100644 --- a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/IndexableMessage.java +++ b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/IndexableMessage.java @@ -29,11 +29,9 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.james.mailbox.MailboxSession.User; -import org.apache.james.mailbox.MessageUid; import org.apache.james.mailbox.elasticsearch.IndexAttachments; import org.apache.james.mailbox.elasticsearch.query.DateResolutionFormater; import org.apache.james.mailbox.extractor.TextExtractor; -import org.apache.james.mailbox.model.MessageId; import org.apache.james.mailbox.store.mail.model.MailboxMessage; import org.apache.james.mailbox.store.mail.model.Property; import org.apache.james.mime4j.MimeException; @@ -48,8 +46,8 @@ import com.google.common.collect.Multimap; public class IndexableMessage { - public static IndexableMessage from(MailboxMessage message, List<User> users, TextExtractor textExtractor, - ZoneId zoneId, IndexAttachments indexAttachments) { + public static IndexableMessage from(MailboxMessage message, List<User> users, TextExtractor textExtractor, + ZoneId zoneId, IndexAttachments indexAttachments) { Preconditions.checkNotNull(message.getMailboxId()); Preconditions.checkArgument(!users.isEmpty()); @@ -90,8 +88,7 @@ public class IndexableMessage { } private void copyMessageFields(MailboxMessage message, ZoneId zoneId) { - this.messageId = message.getMessageId(); - this.uid = message.getUid(); + this.uid = message.getUid().asLong(); this.mailboxId = message.getMailboxId().serialize(); this.modSeq = message.getModSeq(); this.size = message.getFullContentOctets(); @@ -129,8 +126,7 @@ public class IndexableMessage { .collect(Collectors.joining(" ")); } - private MessageId messageId; - private MessageUid uid; + private long uid; private String mailboxId; private List<String> users; private long modSeq; @@ -159,14 +155,47 @@ public class IndexableMessage { private Optional<String> bodyHtml; private String text; - @JsonProperty(JsonMessageConstants.MESSAGE_ID) - public String getId() { - return messageId.serialize(); + public IndexableMessage(long uid, String mailboxId, List<String> users, long modSeq, long size, String date, String mediaType, + String subType, boolean isUnRead, boolean isRecent, boolean isFlagged, boolean isDeleted, boolean isDraft, + boolean isAnswered, String[] userFlags, Multimap<String, String> headers, EMailers from, EMailers to, + EMailers cc, EMailers bcc, EMailers replyTo, Subjects subjects, String sentDate, List<Property> properties, + List<MimePart> attachments, Optional<String> bodyText, Optional<String> bodyHtml, String text) { + this.uid = uid; + this.mailboxId = mailboxId; + this.users = users; + this.modSeq = modSeq; + this.size = size; + this.date = date; + this.mediaType = mediaType; + this.subType = subType; + this.isUnRead = isUnRead; + this.isRecent = isRecent; + this.isFlagged = isFlagged; + this.isDeleted = isDeleted; + this.isDraft = isDraft; + this.isAnswered = isAnswered; + this.userFlags = userFlags; + this.headers = headers; + this.from = from; + this.to = to; + this.cc = cc; + this.bcc = bcc; + this.replyTo = replyTo; + this.subjects = subjects; + this.sentDate = sentDate; + this.properties = properties; + this.attachments = attachments; + this.bodyText = bodyText; + this.bodyHtml = bodyHtml; + this.text = text; + } + + public IndexableMessage() { } @JsonProperty(JsonMessageConstants.UID) public Long getUid() { - return uid.asLong(); + return uid; } @JsonProperty(JsonMessageConstants.MAILBOX_ID) http://git-wip-us.apache.org/repos/asf/james-project/blob/941a7039/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/IndexableMessageWithMessageId.java ---------------------------------------------------------------------- diff --git a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/IndexableMessageWithMessageId.java b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/IndexableMessageWithMessageId.java new file mode 100644 index 0000000..8f376dc --- /dev/null +++ b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/IndexableMessageWithMessageId.java @@ -0,0 +1,72 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ + +package org.apache.james.mailbox.elasticsearch.json; + +import java.time.ZoneId; +import java.util.List; +import java.util.Optional; + +import org.apache.james.mailbox.MailboxSession; +import org.apache.james.mailbox.elasticsearch.IndexAttachments; +import org.apache.james.mailbox.extractor.TextExtractor; +import org.apache.james.mailbox.store.mail.model.MailboxMessage; +import org.apache.james.mailbox.store.mail.model.Property; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.collect.Multimap; + +public class IndexableMessageWithMessageId extends IndexableMessage { + + public static IndexableMessage from(MailboxMessage message, List<MailboxSession.User> users, TextExtractor textExtractor, + ZoneId zoneId, IndexAttachments indexAttachments) { + IndexableMessage indexableMessage = IndexableMessage.from(message, users, textExtractor, zoneId, indexAttachments); + return new IndexableMessageWithMessageId(indexableMessage.getUid(), indexableMessage.getMailboxId(), indexableMessage.getUsers(), + indexableMessage.getModSeq(), indexableMessage.getSize(), indexableMessage.getDate(), indexableMessage.getMediaType(), + indexableMessage.getSubType(), indexableMessage.isUnRead(), indexableMessage.isRecent(), indexableMessage.isFlagged(), + indexableMessage.isDeleted(), indexableMessage.isDraft(), indexableMessage.isAnswered(), indexableMessage.getUserFlags(), + indexableMessage.getHeaders(), indexableMessage.getFrom(), indexableMessage.getTo(), indexableMessage.getCc(), indexableMessage.getBcc(), + indexableMessage.getReplyTo(), indexableMessage.getSubjects(), indexableMessage.getSentDate(), indexableMessage.getProperties(), + indexableMessage.getAttachments(), indexableMessage.getBodyText(), indexableMessage.getBodyHtml(), indexableMessage.getText(), + message.getMessageId().serialize()); + } + + private String messageId; + + public IndexableMessageWithMessageId(long uid, String mailboxId, List<String> users, long modSeq, long size, String date, + String mediaType, String subType, boolean isUnRead, boolean isRecent, boolean isFlagged, + boolean isDeleted, boolean isDraft, boolean isAnswered, String[] userFlags, Multimap<String, String> headers, + EMailers from, EMailers to, EMailers cc, EMailers bcc, EMailers replyTo, Subjects subjects, + String sentDate, List<Property> properties, List<MimePart> attachments, Optional<String> bodyText, + Optional<String> bodyHtml, String text, String messageId) { + super(uid, mailboxId, users, modSeq, size, date, mediaType, subType, isUnRead, isRecent, isFlagged, isDeleted, + isDraft, isAnswered, userFlags, headers, from, to, cc, bcc, replyTo, subjects, sentDate, properties, attachments, + bodyText, bodyHtml, text); + this.messageId = messageId; + } + + public IndexableMessageWithMessageId(String messageId) { + this.messageId = messageId; + } + + @JsonProperty(JsonMessageConstants.MESSAGE_ID) + public String getMessageId() { + return messageId; + } +} http://git-wip-us.apache.org/repos/asf/james-project/blob/941a7039/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/MessageToElasticSearchJson.java ---------------------------------------------------------------------- diff --git a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/MessageToElasticSearchJson.java b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/MessageToElasticSearchJson.java index b6d97e9..be23f98 100644 --- a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/MessageToElasticSearchJson.java +++ b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/MessageToElasticSearchJson.java @@ -25,10 +25,13 @@ import java.util.List; import javax.inject.Inject; import javax.mail.Flags; +import org.apache.commons.lang.NotImplementedException; +import org.apache.james.mailbox.MailboxManager; import org.apache.james.mailbox.MailboxSession.User; import org.apache.james.mailbox.elasticsearch.IndexAttachments; import org.apache.james.mailbox.extractor.TextExtractor; import org.apache.james.mailbox.store.mail.model.MailboxMessage; +import org.apache.james.mailbox.store.search.MessageSearchIndex; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; @@ -42,24 +45,40 @@ public class MessageToElasticSearchJson { private final TextExtractor textExtractor; private final ZoneId zoneId; private final IndexAttachments indexAttachments; + private final MessageSearchIndex.IndexMessageId indexMessageId; - public MessageToElasticSearchJson(TextExtractor textExtractor, ZoneId zoneId, IndexAttachments indexAttachments) { + public MessageToElasticSearchJson(TextExtractor textExtractor, ZoneId zoneId, IndexAttachments indexAttachments, MessageSearchIndex.IndexMessageId indexMessageId) { this.textExtractor = textExtractor; this.zoneId = zoneId; this.indexAttachments = indexAttachments; this.mapper = new ObjectMapper(); this.mapper.registerModule(new GuavaModule()); this.mapper.registerModule(new Jdk8Module()); + this.indexMessageId = indexMessageId; } @Inject - public MessageToElasticSearchJson(TextExtractor textExtractor, IndexAttachments indexAttachments) { - this(textExtractor, ZoneId.systemDefault(), indexAttachments); + public MessageToElasticSearchJson(TextExtractor textExtractor, IndexAttachments indexAttachments, MailboxManager mailboxManager) { + this(textExtractor, ZoneId.systemDefault(), indexAttachments, indexMessageId(mailboxManager)); + } + + private static MessageSearchIndex.IndexMessageId indexMessageId(MailboxManager mailboxManager) { + if (mailboxManager.getSupportedMessageCapabilities().contains(MailboxManager.MessageCapabilities.UniqueID)) { + return MessageSearchIndex.IndexMessageId.Required; + } + return MessageSearchIndex.IndexMessageId.Optional; } public String convertToJson(MailboxMessage message, List<User> users) throws JsonProcessingException { Preconditions.checkNotNull(message); - return mapper.writeValueAsString(IndexableMessage.from(message, users, textExtractor, zoneId, indexAttachments)); + switch (indexMessageId) { + case Required: + return mapper.writeValueAsString(IndexableMessageWithMessageId.from(message, users, textExtractor, zoneId, indexAttachments)); + case Optional: + return mapper.writeValueAsString(IndexableMessage.from(message, users, textExtractor, zoneId, indexAttachments)); + default: + throw new NotImplementedException(); + } } public String getUpdatedJsonMessagePart(Flags flags, long modSeq) throws JsonProcessingException { http://git-wip-us.apache.org/repos/asf/james-project/blob/941a7039/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/ElasticSearchSearcher.java ---------------------------------------------------------------------- diff --git a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/ElasticSearchSearcher.java b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/ElasticSearchSearcher.java index 20ab38d..61bca7e 100644 --- a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/ElasticSearchSearcher.java +++ b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/ElasticSearchSearcher.java @@ -34,11 +34,9 @@ import org.apache.james.mailbox.elasticsearch.query.QueryConverter; import org.apache.james.mailbox.elasticsearch.query.SortConverter; import org.apache.james.mailbox.exception.MailboxException; import org.apache.james.mailbox.model.MailboxId; -import org.apache.james.mailbox.model.MailboxId.Factory; import org.apache.james.mailbox.model.MessageId; import org.apache.james.mailbox.model.MultimailboxesSearchQuery; import org.apache.james.mailbox.store.search.MessageSearchIndex; -import org.apache.james.mailbox.store.search.SimpleMessageSearchIndex; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.Client; @@ -57,7 +55,7 @@ public class ElasticSearchSearcher { private final Client client; private final QueryConverter queryConverter; private final int size; - private final Factory mailboxIdFactory; + private final MailboxId.Factory mailboxIdFactory; private final MessageId.Factory messageIdFactory; @Inject @@ -65,7 +63,7 @@ public class ElasticSearchSearcher { this(client, queryConverter, DEFAULT_SIZE, mailboxIdFactory, messageIdFactory); } - public ElasticSearchSearcher(Client client, QueryConverter queryConverter, int size, Factory mailboxIdFactory, MessageId.Factory messageIdFactory) { + public ElasticSearchSearcher(Client client, QueryConverter queryConverter, int size, MailboxId.Factory mailboxIdFactory, MessageId.Factory messageIdFactory) { this.client = client; this.queryConverter = queryConverter; this.size = size; @@ -109,11 +107,11 @@ public class ElasticSearchSearcher { private Optional<MessageSearchIndex.SearchResult> extractContentFromHit(SearchHit hit) { SearchHitField mailboxId = hit.field(JsonMessageConstants.MAILBOX_ID); SearchHitField uid = hit.field(JsonMessageConstants.UID); - SearchHitField id = hit.field(JsonMessageConstants.ID); - if (mailboxId != null && uid != null && id != null) { + Optional<SearchHitField> id = retrieveMessageIdField(hit); + if (mailboxId != null && uid != null) { Number uidAsNumber = uid.getValue(); return Optional.of( - new MessageSearchIndex.SearchResult(messageIdFactory.fromString(id.getValue()), + new MessageSearchIndex.SearchResult(toGuava(id.map(field -> messageIdFactory.fromString(field.getValue()))), mailboxIdFactory.fromString(mailboxId.getValue()), MessageUid.of(uidAsNumber.longValue()))); } else { @@ -122,4 +120,16 @@ public class ElasticSearchSearcher { } } + private Optional<SearchHitField> retrieveMessageIdField(SearchHit hit) { + if (hit.fields().keySet().contains(JsonMessageConstants.MESSAGE_ID)) { + return Optional.ofNullable(hit.field(JsonMessageConstants.MESSAGE_ID)); + } else { + return Optional.empty(); + } + } + + private <T> com.google.common.base.Optional<T> toGuava(Optional<T> optional) { + return com.google.common.base.Optional.fromNullable(optional.orElse(null)); + } + } http://git-wip-us.apache.org/repos/asf/james-project/blob/941a7039/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/ElasticSearchIntegrationTest.java ---------------------------------------------------------------------- diff --git a/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/ElasticSearchIntegrationTest.java b/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/ElasticSearchIntegrationTest.java index 749f11b..fb73266 100644 --- a/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/ElasticSearchIntegrationTest.java +++ b/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/ElasticSearchIntegrationTest.java @@ -40,6 +40,7 @@ import org.apache.james.mailbox.store.MailboxSessionMapperFactory; import org.apache.james.mailbox.store.extractor.DefaultTextExtractor; import org.apache.james.mailbox.store.mail.model.impl.MessageParser; import org.apache.james.mailbox.store.search.AbstractMessageSearchIndexTest; +import org.apache.james.mailbox.store.search.MessageSearchIndex; import org.elasticsearch.client.Client; import org.junit.Rule; import org.junit.rules.RuleChain; @@ -71,7 +72,7 @@ public class ElasticSearchIntegrationTest extends AbstractMessageSearchIndexTest messageSearchIndex = new ElasticSearchListeningMessageSearchIndex(mapperFactory, new ElasticSearchIndexer(client, new DeleteByQueryPerformer(client, Executors.newSingleThreadExecutor(), BATCH_SIZE)), new ElasticSearchSearcher(client, new QueryConverter(new CriterionConverter()), SEARCH_SIZE, new InMemoryId.Factory(), messageIdFactory), - new MessageToElasticSearchJson(new DefaultTextExtractor(), ZoneId.of("Europe/Paris"), IndexAttachments.YES)); + new MessageToElasticSearchJson(new DefaultTextExtractor(), ZoneId.of("Europe/Paris"), IndexAttachments.YES, MessageSearchIndex.IndexMessageId.Required)); storeMailboxManager = new InMemoryMailboxManager( mapperFactory, new FakeAuthenticator(), http://git-wip-us.apache.org/repos/asf/james-project/blob/941a7039/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/json/IndexableMessageTest.java ---------------------------------------------------------------------- diff --git a/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/json/IndexableMessageTest.java b/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/json/IndexableMessageTest.java index 1318c74..55ce95c 100644 --- a/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/json/IndexableMessageTest.java +++ b/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/json/IndexableMessageTest.java @@ -29,6 +29,7 @@ import java.time.ZoneId; import javax.mail.Flags; import org.apache.commons.io.IOUtils; +import org.apache.james.mailbox.MessageUid; import org.apache.james.mailbox.elasticsearch.IndexAttachments; import org.apache.james.mailbox.mock.MockMailboxSession; import org.apache.james.mailbox.model.TestId; @@ -40,6 +41,8 @@ import com.google.common.collect.ImmutableList; public class IndexableMessageTest { + public static final MessageUid MESSAGE_UID = MessageUid.of(154); + @Test public void textShouldBeEmptyWhenNoMatchingHeaders() throws Exception { MailboxMessage mailboxMessage = mock(MailboxMessage.class); @@ -50,6 +53,8 @@ public class IndexableMessageTest { .thenReturn(new ByteArrayInputStream("".getBytes())); when(mailboxMessage.createFlags()) .thenReturn(new Flags()); + when(mailboxMessage.getUid()) + .thenReturn(MESSAGE_UID); IndexableMessage indexableMessage = IndexableMessage.from(mailboxMessage, ImmutableList.of(new MockMailboxSession("username").getUser()), new DefaultTextExtractor(), ZoneId.of("Europe/Paris"), IndexAttachments.NO); @@ -67,6 +72,8 @@ public class IndexableMessageTest { .thenReturn(new ByteArrayInputStream("From: First user <u...@james.org>\nFrom: Second user <us...@james.org>".getBytes())); when(mailboxMessage.createFlags()) .thenReturn(new Flags()); + when(mailboxMessage.getUid()) + .thenReturn(MESSAGE_UID); IndexableMessage indexableMessage = IndexableMessage.from(mailboxMessage, ImmutableList.of(new MockMailboxSession("username").getUser()), new DefaultTextExtractor(), ZoneId.of("Europe/Paris"), IndexAttachments.NO); @@ -84,6 +91,8 @@ public class IndexableMessageTest { .thenReturn(new ByteArrayInputStream("To: First to <u...@james.org>\nTo: Second to <us...@james.org>".getBytes())); when(mailboxMessage.createFlags()) .thenReturn(new Flags()); + when(mailboxMessage.getUid()) + .thenReturn(MESSAGE_UID); IndexableMessage indexableMessage = IndexableMessage.from(mailboxMessage, ImmutableList.of(new MockMailboxSession("username").getUser()), new DefaultTextExtractor(), ZoneId.of("Europe/Paris"), IndexAttachments.NO); @@ -101,6 +110,8 @@ public class IndexableMessageTest { .thenReturn(new ByteArrayInputStream("Cc: First cc <u...@james.org>\nCc: Second cc <us...@james.org>".getBytes())); when(mailboxMessage.createFlags()) .thenReturn(new Flags()); + when(mailboxMessage.getUid()) + .thenReturn(MESSAGE_UID); IndexableMessage indexableMessage = IndexableMessage.from(mailboxMessage, ImmutableList.of(new MockMailboxSession("username").getUser()), new DefaultTextExtractor(), ZoneId.of("Europe/Paris"), IndexAttachments.NO); @@ -114,6 +125,8 @@ public class IndexableMessageTest { TestId mailboxId = TestId.of(1); when(mailboxMessage.getMailboxId()) .thenReturn(mailboxId); + when(mailboxMessage.getUid()) + .thenReturn(MESSAGE_UID); when(mailboxMessage.getFullContent()) .thenReturn(new ByteArrayInputStream("Bcc: First bcc <u...@james.org>\nBcc: Second bcc <us...@james.org>".getBytes())); when(mailboxMessage.createFlags()) @@ -135,6 +148,8 @@ public class IndexableMessageTest { .thenReturn(new ByteArrayInputStream("Subject: subject1\nSubject: subject2".getBytes())); when(mailboxMessage.createFlags()) .thenReturn(new Flags()); + when(mailboxMessage.getUid()) + .thenReturn(MESSAGE_UID); IndexableMessage indexableMessage = IndexableMessage.from(mailboxMessage, ImmutableList.of(new MockMailboxSession("username").getUser()), new DefaultTextExtractor(), ZoneId.of("Europe/Paris"), IndexAttachments.NO); @@ -152,6 +167,8 @@ public class IndexableMessageTest { .thenReturn(new ByteArrayInputStream("\nMy body".getBytes())); when(mailboxMessage.createFlags()) .thenReturn(new Flags()); + when(mailboxMessage.getUid()) + .thenReturn(MESSAGE_UID); IndexableMessage indexableMessage = IndexableMessage.from(mailboxMessage, ImmutableList.of(new MockMailboxSession("username").getUser()), new DefaultTextExtractor(), ZoneId.of("Europe/Paris"), IndexAttachments.NO); @@ -169,6 +186,8 @@ public class IndexableMessageTest { .thenReturn(new ByteArrayInputStream(IOUtils.toByteArray(ClassLoader.getSystemResourceAsStream("eml/mailWithHeaders.eml")))); when(mailboxMessage.createFlags()) .thenReturn(new Flags()); + when(mailboxMessage.getUid()) + .thenReturn(MESSAGE_UID); IndexableMessage indexableMessage = IndexableMessage.from(mailboxMessage, ImmutableList.of(new MockMailboxSession("username").getUser()), new DefaultTextExtractor(), ZoneId.of("Europe/Paris"), IndexAttachments.NO); @@ -195,6 +214,8 @@ public class IndexableMessageTest { .thenReturn(new ByteArrayInputStream(IOUtils.toByteArray(ClassLoader.getSystemResourceAsStream("eml/Toto.eml")))); when(mailboxMessage.createFlags()) .thenReturn(new Flags()); + when(mailboxMessage.getUid()) + .thenReturn(MESSAGE_UID); // When IndexableMessage indexableMessage = IndexableMessage.from(mailboxMessage, ImmutableList.of(new MockMailboxSession("username").getUser()), @@ -215,6 +236,8 @@ public class IndexableMessageTest { .thenReturn(new ByteArrayInputStream(IOUtils.toByteArray(ClassLoader.getSystemResourceAsStream("eml/Toto.eml")))); when(mailboxMessage.createFlags()) .thenReturn(new Flags()); + when(mailboxMessage.getUid()) + .thenReturn(MESSAGE_UID); // When IndexableMessage indexableMessage = IndexableMessage.from(mailboxMessage, ImmutableList.of(new MockMailboxSession("username").getUser()), http://git-wip-us.apache.org/repos/asf/james-project/blob/941a7039/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/json/MailboxMessageToElasticSearchJsonTest.java ---------------------------------------------------------------------- diff --git a/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/json/MailboxMessageToElasticSearchJsonTest.java b/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/json/MailboxMessageToElasticSearchJsonTest.java index 596d774..6fa9d1f 100644 --- a/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/json/MailboxMessageToElasticSearchJsonTest.java +++ b/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/json/MailboxMessageToElasticSearchJsonTest.java @@ -45,6 +45,7 @@ import org.apache.james.mailbox.store.extractor.DefaultTextExtractor; import org.apache.james.mailbox.store.mail.model.MailboxMessage; import org.apache.james.mailbox.store.mail.model.impl.PropertyBuilder; import org.apache.james.mailbox.store.mail.model.impl.SimpleMailboxMessage; +import org.apache.james.mailbox.store.search.MessageSearchIndex; import org.apache.james.mailbox.tika.extractor.TikaTextExtractor; import org.junit.Before; import org.junit.Test; @@ -81,7 +82,8 @@ public class MailboxMessageToElasticSearchJsonTest { public void convertToJsonShouldThrowWhenNoUser() throws Exception { MessageToElasticSearchJson messageToElasticSearchJson = new MessageToElasticSearchJson( new DefaultTextExtractor(), - ZoneId.of("Europe/Paris"), IndexAttachments.YES); + ZoneId.of("Europe/Paris"), IndexAttachments.YES, + MessageSearchIndex.IndexMessageId.Required); MailboxMessage spamMail = new SimpleMailboxMessage(MESSAGE_ID, date, SIZE, @@ -100,7 +102,8 @@ public class MailboxMessageToElasticSearchJsonTest { public void spamEmailShouldBeWellConvertedToJson() throws IOException { MessageToElasticSearchJson messageToElasticSearchJson = new MessageToElasticSearchJson( new DefaultTextExtractor(), - ZoneId.of("Europe/Paris"), IndexAttachments.YES); + ZoneId.of("Europe/Paris"), IndexAttachments.YES, + MessageSearchIndex.IndexMessageId.Required); MailboxMessage spamMail = new SimpleMailboxMessage(MESSAGE_ID, date, SIZE, @@ -120,7 +123,8 @@ public class MailboxMessageToElasticSearchJsonTest { public void htmlEmailShouldBeWellConvertedToJson() throws IOException { MessageToElasticSearchJson messageToElasticSearchJson = new MessageToElasticSearchJson( new DefaultTextExtractor(), - ZoneId.of("Europe/Paris"), IndexAttachments.YES); + ZoneId.of("Europe/Paris"), IndexAttachments.YES, + MessageSearchIndex.IndexMessageId.Required); MailboxMessage htmlMail = new SimpleMailboxMessage(MESSAGE_ID, date, SIZE, @@ -140,7 +144,8 @@ public class MailboxMessageToElasticSearchJsonTest { public void pgpSignedEmailShouldBeWellConvertedToJson() throws IOException { MessageToElasticSearchJson messageToElasticSearchJson = new MessageToElasticSearchJson( new DefaultTextExtractor(), - ZoneId.of("Europe/Paris"), IndexAttachments.YES); + ZoneId.of("Europe/Paris"), IndexAttachments.YES, + MessageSearchIndex.IndexMessageId.Required); MailboxMessage pgpSignedMail = new SimpleMailboxMessage(MESSAGE_ID, date, SIZE, @@ -160,7 +165,8 @@ public class MailboxMessageToElasticSearchJsonTest { public void simpleEmailShouldBeWellConvertedToJson() throws IOException { MessageToElasticSearchJson messageToElasticSearchJson = new MessageToElasticSearchJson( new DefaultTextExtractor(), - ZoneId.of("Europe/Paris"), IndexAttachments.YES); + ZoneId.of("Europe/Paris"), IndexAttachments.YES, + MessageSearchIndex.IndexMessageId.Required); MailboxMessage mail = new SimpleMailboxMessage(MESSAGE_ID, date, SIZE, @@ -181,7 +187,8 @@ public class MailboxMessageToElasticSearchJsonTest { public void recursiveEmailShouldBeWellConvertedToJson() throws IOException { MessageToElasticSearchJson messageToElasticSearchJson = new MessageToElasticSearchJson( new DefaultTextExtractor(), - ZoneId.of("Europe/Paris"), IndexAttachments.YES); + ZoneId.of("Europe/Paris"), IndexAttachments.YES, + MessageSearchIndex.IndexMessageId.Required); MailboxMessage recursiveMail = new SimpleMailboxMessage(MESSAGE_ID, date, SIZE, @@ -201,7 +208,8 @@ public class MailboxMessageToElasticSearchJsonTest { public void emailWithNoInternalDateShouldUseNowDate() throws IOException { MessageToElasticSearchJson messageToElasticSearchJson = new MessageToElasticSearchJson( new DefaultTextExtractor(), - ZoneId.of("Europe/Paris"), IndexAttachments.YES); + ZoneId.of("Europe/Paris"), IndexAttachments.YES, + MessageSearchIndex.IndexMessageId.Required); MailboxMessage mailWithNoInternalDate = new SimpleMailboxMessage(MESSAGE_ID, null, SIZE, @@ -235,7 +243,8 @@ public class MailboxMessageToElasticSearchJsonTest { // When MessageToElasticSearchJson messageToElasticSearchJson = new MessageToElasticSearchJson( new DefaultTextExtractor(), - ZoneId.of("Europe/Paris"), IndexAttachments.YES); + ZoneId.of("Europe/Paris"), IndexAttachments.YES, + MessageSearchIndex.IndexMessageId.Required); String convertToJson = messageToElasticSearchJson.convertToJson(mailWithNoInternalDate, ImmutableList.of(new MockMailboxSession("username").getUser())); // Then @@ -262,7 +271,8 @@ public class MailboxMessageToElasticSearchJsonTest { // When MessageToElasticSearchJson messageToElasticSearchJson = new MessageToElasticSearchJson( new DefaultTextExtractor(), - ZoneId.of("Europe/Paris"), IndexAttachments.NO); + ZoneId.of("Europe/Paris"), IndexAttachments.NO, + MessageSearchIndex.IndexMessageId.Required); String convertToJson = messageToElasticSearchJson.convertToJson(mailWithNoInternalDate, ImmutableList.of(new MockMailboxSession("username").getUser())); // Then @@ -276,7 +286,8 @@ public class MailboxMessageToElasticSearchJsonTest { public void emailWithNoMailboxIdShouldThrow() throws IOException { MessageToElasticSearchJson messageToElasticSearchJson = new MessageToElasticSearchJson( new DefaultTextExtractor(), - ZoneId.of("Europe/Paris"), IndexAttachments.YES); + ZoneId.of("Europe/Paris"), IndexAttachments.YES, + MessageSearchIndex.IndexMessageId.Required); MailboxMessage mailWithNoMailboxId; try { mailWithNoMailboxId = new SimpleMailboxMessage(MESSAGE_ID, date, @@ -298,7 +309,8 @@ public class MailboxMessageToElasticSearchJsonTest { public void getUpdatedJsonMessagePartShouldBehaveWellOnEmptyFlags() throws Exception { MessageToElasticSearchJson messageToElasticSearchJson = new MessageToElasticSearchJson( new DefaultTextExtractor(), - ZoneId.of("Europe/Paris"), IndexAttachments.YES); + ZoneId.of("Europe/Paris"), IndexAttachments.YES, + MessageSearchIndex.IndexMessageId.Required); assertThatJson(messageToElasticSearchJson.getUpdatedJsonMessagePart(new Flags(), MOD_SEQ)) .isEqualTo("{\"modSeq\":42,\"isAnswered\":false,\"isDeleted\":false,\"isDraft\":false,\"isFlagged\":false,\"isRecent\":false,\"userFlags\":[],\"isUnread\":true}"); } @@ -307,7 +319,8 @@ public class MailboxMessageToElasticSearchJsonTest { public void getUpdatedJsonMessagePartShouldBehaveWellOnNonEmptyFlags() throws Exception { MessageToElasticSearchJson messageToElasticSearchJson = new MessageToElasticSearchJson( new DefaultTextExtractor(), - ZoneId.of("Europe/Paris"), IndexAttachments.YES); + ZoneId.of("Europe/Paris"), IndexAttachments.YES, + MessageSearchIndex.IndexMessageId.Required); assertThatJson(messageToElasticSearchJson.getUpdatedJsonMessagePart(new FlagsBuilder().add(Flags.Flag.DELETED, Flags.Flag.FLAGGED).add("user").build(), MOD_SEQ)) .isEqualTo("{\"modSeq\":42,\"isAnswered\":false,\"isDeleted\":true,\"isDraft\":false,\"isFlagged\":true,\"isRecent\":false,\"userFlags\":[\"user\"],\"isUnread\":true}"); } @@ -316,7 +329,8 @@ public class MailboxMessageToElasticSearchJsonTest { public void getUpdatedJsonMessagePartShouldThrowIfFlagsIsNull() throws Exception { MessageToElasticSearchJson messageToElasticSearchJson = new MessageToElasticSearchJson( new DefaultTextExtractor(), - ZoneId.of("Europe/Paris"), IndexAttachments.YES); + ZoneId.of("Europe/Paris"), IndexAttachments.YES, + MessageSearchIndex.IndexMessageId.Required); messageToElasticSearchJson.getUpdatedJsonMessagePart(null, MOD_SEQ); } @@ -324,7 +338,8 @@ public class MailboxMessageToElasticSearchJsonTest { public void spamEmailShouldBeWellConvertedToJsonWithApacheTika() throws IOException { MessageToElasticSearchJson messageToElasticSearchJson = new MessageToElasticSearchJson( new TikaTextExtractor(), - ZoneId.of("Europe/Paris"), IndexAttachments.YES); + ZoneId.of("Europe/Paris"), IndexAttachments.YES, + MessageSearchIndex.IndexMessageId.Required); MailboxMessage spamMail = new SimpleMailboxMessage(MESSAGE_ID, date, SIZE, BODY_START_OCTET, http://git-wip-us.apache.org/repos/asf/james-project/blob/941a7039/mailbox/lucene/src/main/java/org/apache/james/mailbox/lucene/search/LuceneMessageSearchIndex.java ---------------------------------------------------------------------- diff --git a/mailbox/lucene/src/main/java/org/apache/james/mailbox/lucene/search/LuceneMessageSearchIndex.java b/mailbox/lucene/src/main/java/org/apache/james/mailbox/lucene/search/LuceneMessageSearchIndex.java index 7dee3cf..9b3c0c1 100644 --- a/mailbox/lucene/src/main/java/org/apache/james/mailbox/lucene/search/LuceneMessageSearchIndex.java +++ b/mailbox/lucene/src/main/java/org/apache/james/mailbox/lucene/search/LuceneMessageSearchIndex.java @@ -40,13 +40,13 @@ import javax.inject.Inject; import javax.mail.Flags; import javax.mail.Flags.Flag; +import org.apache.james.mailbox.MailboxManager; import org.apache.james.mailbox.MailboxManager.SearchCapabilities; import org.apache.james.mailbox.MailboxSession; import org.apache.james.mailbox.MessageUid; import org.apache.james.mailbox.exception.MailboxException; import org.apache.james.mailbox.exception.UnsupportedSearchException; import org.apache.james.mailbox.model.MailboxId; -import org.apache.james.mailbox.model.MailboxId.Factory; import org.apache.james.mailbox.model.MessageId; import org.apache.james.mailbox.model.MessageRange; import org.apache.james.mailbox.model.MultimailboxesSearchQuery; @@ -68,6 +68,7 @@ import org.apache.james.mailbox.store.mail.MessageMapperFactory; import org.apache.james.mailbox.store.mail.model.Mailbox; import org.apache.james.mailbox.store.mail.model.MailboxMessage; import org.apache.james.mailbox.store.search.ListeningMessageSearchIndex; +import org.apache.james.mailbox.store.search.MessageSearchIndex; import org.apache.james.mailbox.store.search.SearchUtil; import org.apache.james.mime4j.MimeException; import org.apache.james.mime4j.dom.Header; @@ -118,6 +119,7 @@ import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.Version; import com.google.common.base.Function; +import com.google.common.base.Optional; import com.google.common.base.Preconditions; import com.google.common.collect.FluentIterable; import com.google.common.collect.ImmutableList; @@ -346,33 +348,42 @@ public class LuceneMessageSearchIndex extends ListeningMessageSearchIndex { private final static SortField FIRST_FROM_MAILBOX_DISPLAY_SORT = new SortField(FIRST_FROM_MAILBOX_DISPLAY_FIELD, SortField.STRING); private final static SortField FIRST_FROM_MAILBOX_DISPLAY_SORT_REVERSE = new SortField(FIRST_FROM_MAILBOX_DISPLAY_FIELD, SortField.STRING, true); - private final Factory mailboxIdFactory; + private final MailboxId.Factory mailboxIdFactory; private final MessageId.Factory messageIdFactory; private final IndexWriter writer; + private final IndexMessageId shouldIndexMessageId; private int maxQueryResults = DEFAULT_MAX_QUERY_RESULTS; private boolean suffixMatch = false; @Inject - public LuceneMessageSearchIndex(MessageMapperFactory factory, Factory mailboxIdFactory, Directory directory, MessageId.Factory messageIdFactory) throws CorruptIndexException, LockObtainFailedException, IOException { - this(factory, mailboxIdFactory, directory, false, true, messageIdFactory); + public LuceneMessageSearchIndex(MessageMapperFactory factory, MailboxId.Factory mailboxIdFactory, Directory directory, MessageId.Factory messageIdFactory, MailboxManager mailboxManager) throws CorruptIndexException, LockObtainFailedException, IOException { + this(factory, mailboxIdFactory, directory, false, true, messageIdFactory, indexMessageId(mailboxManager)); + } + + private static MessageSearchIndex.IndexMessageId indexMessageId(MailboxManager mailboxManager) { + if (mailboxManager.getSupportedMessageCapabilities().contains(MailboxManager.MessageCapabilities.UniqueID)) { + return MessageSearchIndex.IndexMessageId.Required; + } + return MessageSearchIndex.IndexMessageId.Optional; } - - public LuceneMessageSearchIndex(MessageMapperFactory factory, Factory mailboxIdFactory, Directory directory, boolean dropIndexOnStart, boolean lenient, MessageId.Factory messageIdFactory) throws CorruptIndexException, LockObtainFailedException, IOException { + public LuceneMessageSearchIndex(MessageMapperFactory factory, MailboxId.Factory mailboxIdFactory, Directory directory, boolean dropIndexOnStart, boolean lenient, MessageId.Factory messageIdFactory, IndexMessageId shouldIndexMessageId) throws CorruptIndexException, LockObtainFailedException, IOException { super(factory); this.mailboxIdFactory = mailboxIdFactory; this.messageIdFactory = messageIdFactory; this.writer = new IndexWriter(directory, createConfig(createAnalyzer(lenient), dropIndexOnStart)); + this.shouldIndexMessageId = shouldIndexMessageId; } - public LuceneMessageSearchIndex(MessageMapperFactory factory, Factory mailboxIdFactory, MessageId.Factory messageIdFactory, IndexWriter writer) { + public LuceneMessageSearchIndex(MessageMapperFactory factory, MailboxId.Factory mailboxIdFactory, MessageId.Factory messageIdFactory, IndexWriter writer, IndexMessageId shouldIndexMessageId) { super(factory); this.mailboxIdFactory = mailboxIdFactory; this.messageIdFactory = messageIdFactory; this.writer = writer; + this.shouldIndexMessageId = shouldIndexMessageId; } @Override @@ -461,7 +472,7 @@ public class LuceneMessageSearchIndex extends ListeningMessageSearchIndex { .transform(new Function<SearchResult, MessageId>() { @Override public MessageId apply(SearchResult input) { - return input.getMessageId(); + return input.getMessageId().get(); } }) .limit(Long.valueOf(limit).intValue()) @@ -492,7 +503,7 @@ public class LuceneMessageSearchIndex extends ListeningMessageSearchIndex { Document doc = searcher.doc(sDoc.doc); MessageUid uid = MessageUid.of(Long.valueOf(doc.get(UID_FIELD))); MailboxId mailboxId = mailboxIdFactory.fromString(doc.get(MAILBOX_ID_FIELD)); - MessageId messageId = messageIdFactory.fromString(doc.get(MESSAGE_ID_FIELD)); + Optional<MessageId> messageId = toMessageId(Optional.fromNullable(doc.get(MESSAGE_ID_FIELD))); results.add(new SearchResult(messageId, mailboxId, uid)); } } catch (IOException e) { @@ -508,7 +519,14 @@ public class LuceneMessageSearchIndex extends ListeningMessageSearchIndex { } return results.build(); } - + + private Optional<MessageId> toMessageId(Optional<String> messageIdField) { + if (messageIdField.isPresent()) { + return Optional.of(messageIdFactory.fromString(messageIdField.get())); + } + return Optional.absent(); + } + private Query buildQueryFromMailboxes(ImmutableSet<MailboxId> mailboxIds) { if (mailboxIds.isEmpty()) { return new MatchAllDocsQuery(); @@ -535,8 +553,11 @@ public class LuceneMessageSearchIndex extends ListeningMessageSearchIndex { // TODO: Better handling doc.add(new Field(MAILBOX_ID_FIELD, membership.getMailboxId().serialize().toUpperCase(Locale.ENGLISH), Store.YES, Index.NOT_ANALYZED)); doc.add(new NumericField(UID_FIELD,Store.YES, true).setLongValue(membership.getUid().asLong())); - doc.add(new Field(MESSAGE_ID_FIELD, membership.getMessageId().serialize(), Store.YES, Index.NOT_ANALYZED)); - + + if (shouldIndexMessageId == IndexMessageId.Required) { + doc.add(new Field(MESSAGE_ID_FIELD, membership.getMessageId().serialize(), Store.YES, Index.NOT_ANALYZED)); + } + // create an unqiue key for the document which can be used later on updates to find the document doc.add(new Field(ID_FIELD, membership.getMailboxId().serialize().toUpperCase(Locale.ENGLISH) +"-" + Long.toString(membership.getUid().asLong()), Store.YES, Index.NOT_ANALYZED)); http://git-wip-us.apache.org/repos/asf/james-project/blob/941a7039/mailbox/lucene/src/main/resources/META-INF/spring/mailbox-index-lucene.xml ---------------------------------------------------------------------- diff --git a/mailbox/lucene/src/main/resources/META-INF/spring/mailbox-index-lucene.xml b/mailbox/lucene/src/main/resources/META-INF/spring/mailbox-index-lucene.xml index 1ed21f4..e551478 100644 --- a/mailbox/lucene/src/main/resources/META-INF/spring/mailbox-index-lucene.xml +++ b/mailbox/lucene/src/main/resources/META-INF/spring/mailbox-index-lucene.xml @@ -35,6 +35,7 @@ <constructor-arg index="1" ref="mailboxIdFactory"/> <constructor-arg index="2" type="org.apache.lucene.store.Directory" ref="fsDirectory"/> <constructor-arg index="3" ref="messageIdFactory"/> + <constructor-arg index="4" ref="mailboxmanager"/> <property name="enableSuffixMatch" value="true"/> </bean> <bean id="fsDirectory" class="org.apache.lucene.store.FSDirectory" factory-method="open"> http://git-wip-us.apache.org/repos/asf/james-project/blob/941a7039/mailbox/lucene/src/test/java/org/apache/james/mailbox/lucene/search/LuceneMailboxMessageSearchIndexTest.java ---------------------------------------------------------------------- diff --git a/mailbox/lucene/src/test/java/org/apache/james/mailbox/lucene/search/LuceneMailboxMessageSearchIndexTest.java b/mailbox/lucene/src/test/java/org/apache/james/mailbox/lucene/search/LuceneMailboxMessageSearchIndexTest.java index 82d8cba..054ec1c 100644 --- a/mailbox/lucene/src/test/java/org/apache/james/mailbox/lucene/search/LuceneMailboxMessageSearchIndexTest.java +++ b/mailbox/lucene/src/test/java/org/apache/james/mailbox/lucene/search/LuceneMailboxMessageSearchIndexTest.java @@ -50,6 +50,7 @@ import org.apache.james.mailbox.model.TestMessageId; import org.apache.james.mailbox.store.MessageBuilder; import org.apache.james.mailbox.store.SimpleMailboxMembership; import org.apache.james.mailbox.store.mail.model.Mailbox; +import org.apache.james.mailbox.store.search.MessageSearchIndex; import org.apache.lucene.store.RAMDirectory; import org.junit.Before; import org.junit.Test; @@ -99,7 +100,7 @@ public class LuceneMailboxMessageSearchIndexTest { id3 = factory.generate(); id4 = factory.generate(); id5 = factory.generate(); - index = new LuceneMessageSearchIndex(null, new TestId.Factory(), new RAMDirectory(), true, useLenient(), factory); + index = new LuceneMessageSearchIndex(null, new TestId.Factory(), new RAMDirectory(), true, useLenient(), factory, MessageSearchIndex.IndexMessageId.Required); index.setEnableSuffixMatch(true); Map<String, String> headersSubject = new HashMap<String, String>(); headersSubject.put("Subject", "test (fwd)"); http://git-wip-us.apache.org/repos/asf/james-project/blob/941a7039/mailbox/lucene/src/test/java/org/apache/james/mailbox/lucene/search/LuceneMessageSearchIndexTest.java ---------------------------------------------------------------------- diff --git a/mailbox/lucene/src/test/java/org/apache/james/mailbox/lucene/search/LuceneMessageSearchIndexTest.java b/mailbox/lucene/src/test/java/org/apache/james/mailbox/lucene/search/LuceneMessageSearchIndexTest.java index 864a7a0..dfdaffd 100644 --- a/mailbox/lucene/src/test/java/org/apache/james/mailbox/lucene/search/LuceneMessageSearchIndexTest.java +++ b/mailbox/lucene/src/test/java/org/apache/james/mailbox/lucene/search/LuceneMessageSearchIndexTest.java @@ -45,7 +45,6 @@ public class LuceneMessageSearchIndexTest extends AbstractMessageSearchIndexTest protected void initializeMailboxManager() throws Exception { TestMessageId.Factory messageIdFactory = new TestMessageId.Factory(); MailboxSessionMapperFactory mapperFactory = new InMemoryMailboxSessionMapperFactory(); - messageSearchIndex = new LuceneMessageSearchIndex(mapperFactory, new InMemoryId.Factory(), new RAMDirectory(), messageIdFactory); storeMailboxManager = new InMemoryMailboxManager( mapperFactory, new FakeAuthenticator(), @@ -54,6 +53,7 @@ public class LuceneMessageSearchIndexTest extends AbstractMessageSearchIndexTest new SimpleGroupMembershipResolver(), new MessageParser(), messageIdFactory); + messageSearchIndex = new LuceneMessageSearchIndex(mapperFactory, new InMemoryId.Factory(), new RAMDirectory(), messageIdFactory, storeMailboxManager); storeMailboxManager.setMessageSearchIndex(messageSearchIndex); storeMailboxManager.init(); } http://git-wip-us.apache.org/repos/asf/james-project/blob/941a7039/mailbox/spring/src/test/resources/META-INF/spring/mailbox-index-lucene.xml ---------------------------------------------------------------------- diff --git a/mailbox/spring/src/test/resources/META-INF/spring/mailbox-index-lucene.xml b/mailbox/spring/src/test/resources/META-INF/spring/mailbox-index-lucene.xml index cf51b98..31c5790 100644 --- a/mailbox/spring/src/test/resources/META-INF/spring/mailbox-index-lucene.xml +++ b/mailbox/spring/src/test/resources/META-INF/spring/mailbox-index-lucene.xml @@ -35,10 +35,12 @@ <constructor-arg index="1" ref="mailboxIdFactory"/> <constructor-arg index="2" type="org.apache.lucene.store.Directory" ref="fsDirectory"/> <constructor-arg index="3" ref="messageIdFactory"/> + <constructor-arg index="4" ref="mailboxmanager"/> <property name="enableSuffixMatch" value="true"/> </bean> <bean id="fsDirectory" class="org.apache.lucene.store.FSDirectory" factory-method="open"> <constructor-arg index="0" value="../var/store/lucene"/> </bean> <alias name="jpa-mailboxIdFactory" alias="mailboxIdFactory"/> + <alias name="jpa-mailboxmanager" alias="mailboxmanager"/> </beans> http://git-wip-us.apache.org/repos/asf/james-project/blob/941a7039/mailbox/store/src/main/java/org/apache/james/mailbox/store/search/MessageSearchIndex.java ---------------------------------------------------------------------- diff --git a/mailbox/store/src/main/java/org/apache/james/mailbox/store/search/MessageSearchIndex.java b/mailbox/store/src/main/java/org/apache/james/mailbox/store/search/MessageSearchIndex.java index 3df16bd..162598a 100644 --- a/mailbox/store/src/main/java/org/apache/james/mailbox/store/search/MessageSearchIndex.java +++ b/mailbox/store/src/main/java/org/apache/james/mailbox/store/search/MessageSearchIndex.java @@ -33,6 +33,8 @@ import org.apache.james.mailbox.model.MultimailboxesSearchQuery; import org.apache.james.mailbox.model.SearchQuery; import org.apache.james.mailbox.store.mail.model.Mailbox; +import com.google.common.base.Optional; + /** * An index which can be used to search for MailboxMessage UID's that match a {@link SearchQuery}. * @@ -41,6 +43,11 @@ import org.apache.james.mailbox.store.mail.model.Mailbox; */ public interface MessageSearchIndex { + enum IndexMessageId { + Required, + Optional + } + /** * Return all uids of the previous indexed {@link Mailbox}'s which match the {@link SearchQuery} */ @@ -54,17 +61,17 @@ public interface MessageSearchIndex { EnumSet<MailboxManager.SearchCapabilities> getSupportedCapabilities(); class SearchResult { - private final MessageId messageId; + private final Optional<MessageId> messageId; private final MailboxId mailboxId; private final MessageUid messageUid; - public SearchResult(MessageId messageId, MailboxId mailboxId, MessageUid messageUid) { + public SearchResult(Optional<MessageId> messageId, MailboxId mailboxId, MessageUid messageUid) { this.messageId = messageId; this.mailboxId = mailboxId; this.messageUid = messageUid; } - public MessageId getMessageId() { + public Optional<MessageId> getMessageId() { return messageId; } http://git-wip-us.apache.org/repos/asf/james-project/blob/941a7039/mailbox/store/src/main/java/org/apache/james/mailbox/store/search/MessageSearches.java ---------------------------------------------------------------------- diff --git a/mailbox/store/src/main/java/org/apache/james/mailbox/store/search/MessageSearches.java b/mailbox/store/src/main/java/org/apache/james/mailbox/store/search/MessageSearches.java index 2f69adb..6fe58b5 100644 --- a/mailbox/store/src/main/java/org/apache/james/mailbox/store/search/MessageSearches.java +++ b/mailbox/store/src/main/java/org/apache/james/mailbox/store/search/MessageSearches.java @@ -68,6 +68,7 @@ import org.apache.james.mime4j.message.HeaderImpl; import org.apache.james.mime4j.utils.search.MessageMatcher; import com.google.common.base.Function; +import com.google.common.base.Optional; import com.google.common.collect.FluentIterable; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; @@ -114,7 +115,7 @@ public class MessageSearches implements Iterable<SimpleMessageSearchIndex.Search @Override public SimpleMessageSearchIndex.SearchResult apply(MailboxMessage input) { return new SimpleMessageSearchIndex.SearchResult( - input.getMessageId(), + Optional.of(input.getMessageId()), input.getMailboxId(), input.getUid()); } http://git-wip-us.apache.org/repos/asf/james-project/blob/941a7039/mailbox/store/src/main/java/org/apache/james/mailbox/store/search/SimpleMessageSearchIndex.java ---------------------------------------------------------------------- diff --git a/mailbox/store/src/main/java/org/apache/james/mailbox/store/search/SimpleMessageSearchIndex.java b/mailbox/store/src/main/java/org/apache/james/mailbox/store/search/SimpleMessageSearchIndex.java index 537164e..3d3ee63 100644 --- a/mailbox/store/src/main/java/org/apache/james/mailbox/store/search/SimpleMessageSearchIndex.java +++ b/mailbox/store/src/main/java/org/apache/james/mailbox/store/search/SimpleMessageSearchIndex.java @@ -182,7 +182,7 @@ public class SimpleMessageSearchIndex implements MessageSearchIndex { return new Function<SearchResult, MessageId>() { @Override public MessageId apply(SearchResult input) { - return input.getMessageId(); + return input.getMessageId().get(); } }; } http://git-wip-us.apache.org/repos/asf/james-project/blob/941a7039/mailbox/store/src/test/resources/eml/htmlMail.json ---------------------------------------------------------------------- diff --git a/mailbox/store/src/test/resources/eml/htmlMail.json b/mailbox/store/src/test/resources/eml/htmlMail.json index fdc1465..1655f5e 100644 --- a/mailbox/store/src/test/resources/eml/htmlMail.json +++ b/mailbox/store/src/test/resources/eml/htmlMail.json @@ -1,5 +1,5 @@ { - "id":"184", + "messageId":"184", "uid":25, "mailboxId":"18", "modSeq":42, http://git-wip-us.apache.org/repos/asf/james-project/blob/941a7039/mailbox/store/src/test/resources/eml/mail.json ---------------------------------------------------------------------- diff --git a/mailbox/store/src/test/resources/eml/mail.json b/mailbox/store/src/test/resources/eml/mail.json index c734e45..76068a5 100644 --- a/mailbox/store/src/test/resources/eml/mail.json +++ b/mailbox/store/src/test/resources/eml/mail.json @@ -1,5 +1,5 @@ { - "id":"184", + "messageId":"184", "uid":25, "mailboxId": "18", "modSeq": 42, http://git-wip-us.apache.org/repos/asf/james-project/blob/941a7039/mailbox/store/src/test/resources/eml/nonTextual.json ---------------------------------------------------------------------- diff --git a/mailbox/store/src/test/resources/eml/nonTextual.json b/mailbox/store/src/test/resources/eml/nonTextual.json index f15f412..9d1aa9c 100644 --- a/mailbox/store/src/test/resources/eml/nonTextual.json +++ b/mailbox/store/src/test/resources/eml/nonTextual.json @@ -1,5 +1,5 @@ { - "id":"184", + "messageId":"184", "uid":25, "mailboxId":"18", "modSeq":42, http://git-wip-us.apache.org/repos/asf/james-project/blob/941a7039/mailbox/store/src/test/resources/eml/pgpSignedMail.json ---------------------------------------------------------------------- diff --git a/mailbox/store/src/test/resources/eml/pgpSignedMail.json b/mailbox/store/src/test/resources/eml/pgpSignedMail.json index 91bdc42..0b00109 100644 --- a/mailbox/store/src/test/resources/eml/pgpSignedMail.json +++ b/mailbox/store/src/test/resources/eml/pgpSignedMail.json @@ -1,5 +1,5 @@ { - "id":"184", + "messageId":"184", "uid":25, "mailboxId": "18", "modSeq": 42, http://git-wip-us.apache.org/repos/asf/james-project/blob/941a7039/mailbox/store/src/test/resources/eml/recursiveMail.json ---------------------------------------------------------------------- diff --git a/mailbox/store/src/test/resources/eml/recursiveMail.json b/mailbox/store/src/test/resources/eml/recursiveMail.json index 4ec9942..f0ae82e 100644 --- a/mailbox/store/src/test/resources/eml/recursiveMail.json +++ b/mailbox/store/src/test/resources/eml/recursiveMail.json @@ -1,5 +1,5 @@ { - "id":"184", + "messageId":"184", "uid":25, "mailboxId": "18", "modSeq": 42, http://git-wip-us.apache.org/repos/asf/james-project/blob/941a7039/mailbox/store/src/test/resources/eml/recursiveMailWithoutAttachments.json ---------------------------------------------------------------------- diff --git a/mailbox/store/src/test/resources/eml/recursiveMailWithoutAttachments.json b/mailbox/store/src/test/resources/eml/recursiveMailWithoutAttachments.json index 06935fd..9dfb964 100644 --- a/mailbox/store/src/test/resources/eml/recursiveMailWithoutAttachments.json +++ b/mailbox/store/src/test/resources/eml/recursiveMailWithoutAttachments.json @@ -1,5 +1,5 @@ { - "id":"184", + "messageId":"184", "uid":25, "mailboxId": "18", "modSeq": 42, http://git-wip-us.apache.org/repos/asf/james-project/blob/941a7039/mailbox/store/src/test/resources/eml/spamMail.json ---------------------------------------------------------------------- diff --git a/mailbox/store/src/test/resources/eml/spamMail.json b/mailbox/store/src/test/resources/eml/spamMail.json index d8ede1e..b71a5ce 100644 --- a/mailbox/store/src/test/resources/eml/spamMail.json +++ b/mailbox/store/src/test/resources/eml/spamMail.json @@ -1,5 +1,5 @@ { - "id":"184", + "messageId":"184", "uid":25, "mailboxId": "18", "modSeq": 42, http://git-wip-us.apache.org/repos/asf/james-project/blob/941a7039/mpt/impl/imap-mailbox/elasticsearch/src/test/java/org/apache/james/mpt/imapmailbox/elasticsearch/host/ElasticSearchHostSystem.java ---------------------------------------------------------------------- diff --git a/mpt/impl/imap-mailbox/elasticsearch/src/test/java/org/apache/james/mpt/imapmailbox/elasticsearch/host/ElasticSearchHostSystem.java b/mpt/impl/imap-mailbox/elasticsearch/src/test/java/org/apache/james/mpt/imapmailbox/elasticsearch/host/ElasticSearchHostSystem.java index 4a50a02..e6a09a7 100644 --- a/mpt/impl/imap-mailbox/elasticsearch/src/test/java/org/apache/james/mpt/imapmailbox/elasticsearch/host/ElasticSearchHostSystem.java +++ b/mpt/impl/imap-mailbox/elasticsearch/src/test/java/org/apache/james/mpt/imapmailbox/elasticsearch/host/ElasticSearchHostSystem.java @@ -21,6 +21,7 @@ package org.apache.james.mpt.imapmailbox.elasticsearch.host; import java.nio.file.Files; import java.nio.file.Path; +import java.time.ZoneId; import java.util.concurrent.Executors; import org.apache.commons.io.FileUtils; @@ -57,6 +58,7 @@ import org.apache.james.mailbox.store.extractor.DefaultTextExtractor; import org.apache.james.mailbox.store.mail.model.impl.MessageParser; import org.apache.james.mailbox.store.quota.DefaultQuotaRootResolver; import org.apache.james.mailbox.store.quota.NoQuotaManager; +import org.apache.james.mailbox.store.search.MessageSearchIndex; import org.apache.james.mpt.api.ImapFeatures; import org.apache.james.mpt.api.ImapFeatures.Feature; import org.apache.james.mpt.host.JamesImapHostSystem; @@ -111,7 +113,7 @@ public class ElasticSearchHostSystem extends JamesImapHostSystem { factory, new ElasticSearchIndexer(client, new DeleteByQueryPerformer(client, Executors.newSingleThreadExecutor())), new ElasticSearchSearcher(client, new QueryConverter(new CriterionConverter()), new InMemoryId.Factory(), messageIdFactory), - new MessageToElasticSearchJson(new DefaultTextExtractor(), IndexAttachments.YES)); + new MessageToElasticSearchJson(new DefaultTextExtractor(), ZoneId.systemDefault(), IndexAttachments.YES, MessageSearchIndex.IndexMessageId.Required)); MailboxACLResolver aclResolver = new UnionMailboxACLResolver(); GroupMembershipResolver groupMembershipResolver = new SimpleGroupMembershipResolver(); http://git-wip-us.apache.org/repos/asf/james-project/blob/941a7039/mpt/impl/imap-mailbox/lucenesearch/src/test/java/org/apache/james/mpt/imapmailbox/lucenesearch/host/LuceneSearchHostSystem.java ---------------------------------------------------------------------- diff --git a/mpt/impl/imap-mailbox/lucenesearch/src/test/java/org/apache/james/mpt/imapmailbox/lucenesearch/host/LuceneSearchHostSystem.java b/mpt/impl/imap-mailbox/lucenesearch/src/test/java/org/apache/james/mpt/imapmailbox/lucenesearch/host/LuceneSearchHostSystem.java index 63bd19b..40b6e33 100644 --- a/mpt/impl/imap-mailbox/lucenesearch/src/test/java/org/apache/james/mpt/imapmailbox/lucenesearch/host/LuceneSearchHostSystem.java +++ b/mpt/impl/imap-mailbox/lucenesearch/src/test/java/org/apache/james/mpt/imapmailbox/lucenesearch/host/LuceneSearchHostSystem.java @@ -151,15 +151,16 @@ public class LuceneSearchHostSystem extends JamesImapHostSystem { try { JPAId.Factory mailboxIdFactory = new Factory(); FSDirectory fsDirectory = FSDirectory.open(tempFile); - LuceneMessageSearchIndex searchIndex = new LuceneMessageSearchIndex(factory, mailboxIdFactory, fsDirectory, messageIdFactory); - searchIndex.setEnableSuffixMatch(true); + MessageId.Factory messageIdFactory = new DefaultMessageId.Factory(); MailboxACLResolver aclResolver = new UnionMailboxACLResolver(); GroupMembershipResolver groupMembershipResolver = new SimpleGroupMembershipResolver(); MessageParser messageParser = new MessageParser(); - MessageId.Factory messageIdFactory = new DefaultMessageId.Factory(); mailboxManager = new OpenJPAMailboxManager(factory, userManager, locker, false, aclResolver, groupMembershipResolver, messageParser, messageIdFactory); + + LuceneMessageSearchIndex searchIndex = new LuceneMessageSearchIndex(factory, mailboxIdFactory, fsDirectory, messageIdFactory, mailboxManager); + searchIndex.setEnableSuffixMatch(true); mailboxManager.setMessageSearchIndex(searchIndex); mailboxManager.init(); --------------------------------------------------------------------- To unsubscribe, e-mail: server-dev-unsubscr...@james.apache.org For additional commands, e-mail: server-dev-h...@james.apache.org
