JAMES-2362 MessageID should be indexed in ElasticSearch
Project: http://git-wip-us.apache.org/repos/asf/james-project/repo Commit: http://git-wip-us.apache.org/repos/asf/james-project/commit/93f1d3ea Tree: http://git-wip-us.apache.org/repos/asf/james-project/tree/93f1d3ea Diff: http://git-wip-us.apache.org/repos/asf/james-project/diff/93f1d3ea Branch: refs/heads/master Commit: 93f1d3eac6717fa27c665cea5294a37042d09a74 Parents: d8113c1 Author: benwa <btell...@linagora.com> Authored: Tue Mar 27 09:39:10 2018 +0700 Committer: benwa <btell...@linagora.com> Committed: Tue Mar 27 15:16:32 2018 +0700 ---------------------------------------------------------------------- .../elasticsearch/MailboxMappingFactory.java | 6 ++++ .../elasticsearch/json/HeaderCollection.java | 17 ++++++++-- .../elasticsearch/json/IndexableMessage.java | 14 +++++++-- .../json/JsonMessageConstants.java | 1 + .../json/HeaderCollectionTest.java | 33 ++++++++++++++++++++ ...NonIndexableAttachmentWithoutAttachment.json | 1 + .../store/src/test/resources/eml/htmlMail.json | 1 + mailbox/store/src/test/resources/eml/mail.json | 1 + .../src/test/resources/eml/nonTextual.json | 1 + .../src/test/resources/eml/pgpSignedMail.json | 1 + .../src/test/resources/eml/recursiveMail.json | 1 + .../eml/recursiveMailWithoutAttachments.json | 1 + .../store/src/test/resources/eml/spamMail.json | 1 + 13 files changed, 74 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/james-project/blob/93f1d3ea/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/MailboxMappingFactory.java ---------------------------------------------------------------------- diff --git a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/MailboxMappingFactory.java b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/MailboxMappingFactory.java index 506b73c..ac75a5b 100644 --- a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/MailboxMappingFactory.java +++ b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/MailboxMappingFactory.java @@ -53,6 +53,7 @@ import static org.apache.james.mailbox.elasticsearch.json.JsonMessageConstants.I import static org.apache.james.mailbox.elasticsearch.json.JsonMessageConstants.MAILBOX_ID; import static org.apache.james.mailbox.elasticsearch.json.JsonMessageConstants.MEDIA_TYPE; import static org.apache.james.mailbox.elasticsearch.json.JsonMessageConstants.MESSAGE_ID; +import static org.apache.james.mailbox.elasticsearch.json.JsonMessageConstants.MIME_MESSAGE_ID; import static org.apache.james.mailbox.elasticsearch.json.JsonMessageConstants.MODSEQ; import static org.apache.james.mailbox.elasticsearch.json.JsonMessageConstants.SENT_DATE; import static org.apache.james.mailbox.elasticsearch.json.JsonMessageConstants.SIZE; @@ -278,6 +279,11 @@ public class MailboxMappingFactory { .field(INDEX, NOT_ANALYZED) .endObject() + .startObject(MIME_MESSAGE_ID) + .field(TYPE, STRING) + .field(INDEX, NOT_ANALYZED) + .endObject() + .startObject(USERS) .field(TYPE, STRING) .field(INDEX, NOT_ANALYZED) http://git-wip-us.apache.org/repos/asf/james-project/blob/93f1d3ea/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/HeaderCollection.java ---------------------------------------------------------------------- diff --git a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/HeaderCollection.java b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/HeaderCollection.java index d50b1e2..4782f74 100644 --- a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/HeaderCollection.java +++ b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/HeaderCollection.java @@ -54,6 +54,7 @@ public class HeaderCollection { private final Set<String> subjectSet; private final Multimap<String, String> headers; private Optional<ZonedDateTime> sentDate; + private Optional<String> messageID; private Builder() { toAddressSet = new HashSet<>(); @@ -64,6 +65,7 @@ public class HeaderCollection { subjectSet = new HashSet<>(); headers = ArrayListMultimap.create(); sentDate = Optional.empty(); + messageID = Optional.empty(); } public Builder add(Field field) { @@ -87,7 +89,7 @@ public class HeaderCollection { ImmutableSet.copyOf(replyToAddressSet), ImmutableSet.copyOf(subjectSet), ImmutableMultimap.copyOf(headers), - sentDate); + sentDate, messageID); } private void handleSpecificHeader(String headerName, String headerValue) { @@ -105,6 +107,9 @@ public class HeaderCollection { case DATE: sentDate = SentDateComparator.toISODate(headerValue); break; + case MESSAGE_ID: + messageID = Optional.ofNullable(headerValue); + break; } } @@ -150,6 +155,7 @@ public class HeaderCollection { public static final String REPLY_TO = "reply-to"; public static final String SUBJECT = "subject"; public static final String DATE = "date"; + public static final String MESSAGE_ID = "message-id"; public static Builder builder() { return new Builder(); @@ -163,10 +169,11 @@ public class HeaderCollection { private final ImmutableSet<String> subjectSet; private final ImmutableMultimap<String, String> headers; private final Optional<ZonedDateTime> sentDate; + private final Optional<String> messageID; private HeaderCollection(ImmutableSet<EMailer> toAddressSet, ImmutableSet<EMailer> fromAddressSet, - ImmutableSet<EMailer> ccAddressSet, ImmutableSet<EMailer> bccAddressSet, ImmutableSet<EMailer> replyToAddressSet, ImmutableSet<String> subjectSet, - ImmutableMultimap<String, String> headers, Optional<ZonedDateTime> sentDate) { + ImmutableSet<EMailer> ccAddressSet, ImmutableSet<EMailer> bccAddressSet, ImmutableSet<EMailer> replyToAddressSet, ImmutableSet<String> subjectSet, + ImmutableMultimap<String, String> headers, Optional<ZonedDateTime> sentDate, Optional<String> messageID) { this.toAddressSet = toAddressSet; this.fromAddressSet = fromAddressSet; this.ccAddressSet = ccAddressSet; @@ -175,6 +182,7 @@ public class HeaderCollection { this.subjectSet = subjectSet; this.headers = headers; this.sentDate = sentDate; + this.messageID = messageID; } public Set<EMailer> getToAddressSet() { @@ -209,4 +217,7 @@ public class HeaderCollection { return headers; } + public Optional<String> getMessageID() { + return messageID; + } } http://git-wip-us.apache.org/repos/asf/james-project/blob/93f1d3ea/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/IndexableMessage.java ---------------------------------------------------------------------- diff --git a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/IndexableMessage.java b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/IndexableMessage.java index bd40558..ffcb140 100644 --- a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/IndexableMessage.java +++ b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/IndexableMessage.java @@ -140,6 +140,7 @@ public class IndexableMessage { EMailers cc = EMailers.from(headerCollection.getCcAddressSet()); EMailers bcc = EMailers.from(headerCollection.getBccAddressSet()); String sentDate = DateResolutionFormater.DATE_TIME_FOMATTER.format(headerCollection.getSentDate().orElse(internalDate)); + Optional<String> mimeMessageID = headerCollection.getMessageID(); String text = Stream.of(from.serialize(), to.serialize(), @@ -197,7 +198,8 @@ public class IndexableMessage { to, uid, userFlags, - stringifiedUsers); + stringifiedUsers, + mimeMessageID); } private List<MimePart> setFlattenedAttachments(MimePart parsingResult, IndexAttachments indexAttachments) { @@ -246,6 +248,7 @@ public class IndexableMessage { private final long uid; private final String[] userFlags; private final List<String> users; + private final Optional<String> mimeMessageID; private IndexableMessage( List<MimePart> attachments, @@ -277,7 +280,8 @@ public class IndexableMessage { EMailers to, long uid, String[] userFlags, - List<String> users) { + List<String> users, + Optional<String> mimeMessageID) { this.attachments = attachments; this.bcc = bcc; this.bodyHtml = bodyHtml; @@ -308,6 +312,7 @@ public class IndexableMessage { this.uid = uid; this.userFlags = userFlags; this.users = users; + this.mimeMessageID = mimeMessageID; } @JsonProperty(JsonMessageConstants.ATTACHMENTS) @@ -459,4 +464,9 @@ public class IndexableMessage { public boolean isUnRead() { return isUnRead; } + + @JsonProperty(JsonMessageConstants.MIME_MESSAGE_ID) + public Optional<String> getMimeMessageID() { + return mimeMessageID; + } } http://git-wip-us.apache.org/repos/asf/james-project/blob/93f1d3ea/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/JsonMessageConstants.java ---------------------------------------------------------------------- diff --git a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/JsonMessageConstants.java b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/JsonMessageConstants.java index f931051..d6fa74e 100644 --- a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/JsonMessageConstants.java +++ b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/JsonMessageConstants.java @@ -46,6 +46,7 @@ public interface JsonMessageConstants { String SENT_DATE = "sentDate"; String ATTACHMENTS = "attachments"; String TEXT = "text"; + String MIME_MESSAGE_ID = "mimeMessageID"; /* James properties we can easily get http://git-wip-us.apache.org/repos/asf/james-project/blob/93f1d3ea/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/json/HeaderCollectionTest.java ---------------------------------------------------------------------- diff --git a/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/json/HeaderCollectionTest.java b/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/json/HeaderCollectionTest.java index 52b4002..9d10e7a 100644 --- a/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/json/HeaderCollectionTest.java +++ b/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/json/HeaderCollectionTest.java @@ -272,6 +272,39 @@ public class HeaderCollectionTest { assertThat(headerCollection.getSubjectSet()).containsOnly("A fantastic ElasticSearch module will be available soon for JAMES"); } + @Test + public void getMessageIDShouldReturnMessageIdValue() { + String messageID = "<abc@123>"; + HeaderCollection headerCollection = HeaderCollection.builder() + .add(new FieldImpl("Message-ID", messageID)) + .build(); + + assertThat(headerCollection.getMessageID()) + .contains(messageID); + } + + @Test + public void getMessageIDShouldReturnLatestEncounteredMessageIdValue() { + String messageID = "<abc@123>"; + HeaderCollection headerCollection = HeaderCollection.builder() + .add(new FieldImpl("Message-ID", "<ot...@toto.com>")) + .add(new FieldImpl("Message-ID", messageID)) + .build(); + + assertThat(headerCollection.getMessageID()) + .contains(messageID); + } + + @Test + public void getMessageIDShouldReturnEmptyWhenNoMessageId() { + HeaderCollection headerCollection = HeaderCollection.builder() + .add(new FieldImpl("Other", "value")) + .build(); + + assertThat(headerCollection.getMessageID()) + .isEmpty(); + } + @Test(expected = NullPointerException.class) public void nullFieldShouldThrow() { HeaderCollection.builder().add(null).build(); http://git-wip-us.apache.org/repos/asf/james-project/blob/93f1d3ea/mailbox/store/src/test/resources/eml/emailWithNonIndexableAttachmentWithoutAttachment.json ---------------------------------------------------------------------- diff --git a/mailbox/store/src/test/resources/eml/emailWithNonIndexableAttachmentWithoutAttachment.json b/mailbox/store/src/test/resources/eml/emailWithNonIndexableAttachmentWithoutAttachment.json index deadebc..b960a84 100644 --- a/mailbox/store/src/test/resources/eml/emailWithNonIndexableAttachmentWithoutAttachment.json +++ b/mailbox/store/src/test/resources/eml/emailWithNonIndexableAttachmentWithoutAttachment.json @@ -13,6 +13,7 @@ "mailboxId" : "18", "date" : "2016-12-07T11:51:51+0100", "size" : 25, + "mimeMessageID": "1480502737913", "properties" : [ { "value" : "plain", http://git-wip-us.apache.org/repos/asf/james-project/blob/93f1d3ea/mailbox/store/src/test/resources/eml/htmlMail.json ---------------------------------------------------------------------- diff --git a/mailbox/store/src/test/resources/eml/htmlMail.json b/mailbox/store/src/test/resources/eml/htmlMail.json index a8998ef..04cc4e3 100644 --- a/mailbox/store/src/test/resources/eml/htmlMail.json +++ b/mailbox/store/src/test/resources/eml/htmlMail.json @@ -8,6 +8,7 @@ "mediaType":"plain", "subtype":"text", "userFlags":["social","pocket-money"], + "mimeMessageID": "<556fffe8cac78_7ed0e0fe20445...@i-dee0850e.mail>", "headers":{ "date":[ "Thu, 04 Jun 2015 07:36:08 +0000" http://git-wip-us.apache.org/repos/asf/james-project/blob/93f1d3ea/mailbox/store/src/test/resources/eml/mail.json ---------------------------------------------------------------------- diff --git a/mailbox/store/src/test/resources/eml/mail.json b/mailbox/store/src/test/resources/eml/mail.json index 76068a5..2ad96b7 100644 --- a/mailbox/store/src/test/resources/eml/mail.json +++ b/mailbox/store/src/test/resources/eml/mail.json @@ -11,6 +11,7 @@ "security", "debian" ], + "mimeMessageID": "<can1zdnw5pw7z5msm-tjupouygxwy6z-25a3wcb-c5bvqdup...@mail.gmail.com>", "headers": { "mime-version": [ "1.0" http://git-wip-us.apache.org/repos/asf/james-project/blob/93f1d3ea/mailbox/store/src/test/resources/eml/nonTextual.json ---------------------------------------------------------------------- diff --git a/mailbox/store/src/test/resources/eml/nonTextual.json b/mailbox/store/src/test/resources/eml/nonTextual.json index af831cb..c9f3ed9 100644 --- a/mailbox/store/src/test/resources/eml/nonTextual.json +++ b/mailbox/store/src/test/resources/eml/nonTextual.json @@ -8,6 +8,7 @@ "mediaType":"plain", "subtype":"text", "userFlags":[], + "mimeMessageID": "<5582a0ce.4020...@linagora.com>", "headers":{ "date":[ "Thu, 18 Jun 2015 12:43:26 +0200" http://git-wip-us.apache.org/repos/asf/james-project/blob/93f1d3ea/mailbox/store/src/test/resources/eml/pgpSignedMail.json ---------------------------------------------------------------------- diff --git a/mailbox/store/src/test/resources/eml/pgpSignedMail.json b/mailbox/store/src/test/resources/eml/pgpSignedMail.json index 0b00109..b565d75 100644 --- a/mailbox/store/src/test/resources/eml/pgpSignedMail.json +++ b/mailbox/store/src/test/resources/eml/pgpSignedMail.json @@ -11,6 +11,7 @@ "security", "debian" ], + "mimeMessageID": "<e1z0e7u-0004bc...@master.debian.org>", "headers": { "date": [ "Wed, 03 Jun 2015 19:14:32 +0000" http://git-wip-us.apache.org/repos/asf/james-project/blob/93f1d3ea/mailbox/store/src/test/resources/eml/recursiveMail.json ---------------------------------------------------------------------- diff --git a/mailbox/store/src/test/resources/eml/recursiveMail.json b/mailbox/store/src/test/resources/eml/recursiveMail.json index f0ae82e..ba1ca5c 100644 --- a/mailbox/store/src/test/resources/eml/recursiveMail.json +++ b/mailbox/store/src/test/resources/eml/recursiveMail.json @@ -11,6 +11,7 @@ "security", "debian" ], + "mimeMessageID": "<5577f927.2040...@linagora.com>", "headers": { "date": [ "Wed, 10 Jun 2015 10:45:27 +0200" http://git-wip-us.apache.org/repos/asf/james-project/blob/93f1d3ea/mailbox/store/src/test/resources/eml/recursiveMailWithoutAttachments.json ---------------------------------------------------------------------- diff --git a/mailbox/store/src/test/resources/eml/recursiveMailWithoutAttachments.json b/mailbox/store/src/test/resources/eml/recursiveMailWithoutAttachments.json index 9dfb964..c40110f 100644 --- a/mailbox/store/src/test/resources/eml/recursiveMailWithoutAttachments.json +++ b/mailbox/store/src/test/resources/eml/recursiveMailWithoutAttachments.json @@ -7,6 +7,7 @@ "date": "2015-06-07T00:00:00+02:00", "mediaType": "plain", "subtype": "text", + "mimeMessageID": "<5577f927.2040...@linagora.com>", "userFlags": [ "security", "debian" http://git-wip-us.apache.org/repos/asf/james-project/blob/93f1d3ea/mailbox/store/src/test/resources/eml/spamMail.json ---------------------------------------------------------------------- diff --git a/mailbox/store/src/test/resources/eml/spamMail.json b/mailbox/store/src/test/resources/eml/spamMail.json index 388aa2d..d133cd7 100644 --- a/mailbox/store/src/test/resources/eml/spamMail.json +++ b/mailbox/store/src/test/resources/eml/spamMail.json @@ -7,6 +7,7 @@ "date": "2015-06-07T00:00:00+0200", "mediaType": "plain", "subtype": "text", + "mimeMessageID": "<vass-izaxqm...@spam.minet.net>", "userFlags": [], "headers": { "mime-version": [ --------------------------------------------------------------------- To unsubscribe, e-mail: server-dev-unsubscr...@james.apache.org For additional commands, e-mail: server-dev-h...@james.apache.org
