Repository: james-project Updated Branches: refs/heads/master 3c379ddb4 -> e5fcb2ab5
MAILBOX-330 Stop indexing attachment metadata No added value. Additionally metadata structure is guessed by ES, which leads to some potential non sanitized dates failures. Project: http://git-wip-us.apache.org/repos/asf/james-project/repo Commit: http://git-wip-us.apache.org/repos/asf/james-project/commit/01dd7460 Tree: http://git-wip-us.apache.org/repos/asf/james-project/tree/01dd7460 Diff: http://git-wip-us.apache.org/repos/asf/james-project/diff/01dd7460 Branch: refs/heads/master Commit: 01dd74605fae4c1a2ac88778812301f50693073f Parents: 3c379dd Author: benwa <[email protected]> Authored: Wed Apr 18 11:28:48 2018 +0700 Committer: benwa <[email protected]> Committed: Wed Apr 18 12:27:02 2018 +0700 ---------------------------------------------------------------------- .../json/JsonMessageConstants.java | 1 - .../mailbox/elasticsearch/json/MimePart.java | 21 +--------- .../store/src/test/resources/eml/htmlMail.json | 2 - .../src/test/resources/eml/nonTextual.json | 43 -------------------- .../src/test/resources/eml/recursiveMail.json | 5 --- .../store/src/test/resources/eml/spamMail.json | 3 -- 6 files changed, 2 insertions(+), 73 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/james-project/blob/01dd7460/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/JsonMessageConstants.java ---------------------------------------------------------------------- diff --git a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/JsonMessageConstants.java b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/JsonMessageConstants.java index d6fa74e..91e731b 100644 --- a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/JsonMessageConstants.java +++ b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/JsonMessageConstants.java @@ -72,7 +72,6 @@ public interface JsonMessageConstants { String CONTENT_DISPOSITION = "contentDisposition"; String FILENAME = "fileName"; String FILE_EXTENSION = "fileExtension"; - String FILE_METADATA = "fileMetadata"; } interface Property { http://git-wip-us.apache.org/repos/asf/james-project/blob/01dd7460/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/MimePart.java ---------------------------------------------------------------------- diff --git a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/MimePart.java b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/MimePart.java index 366cdd4..11f246c 100644 --- a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/MimePart.java +++ b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/MimePart.java @@ -35,7 +35,6 @@ import org.slf4j.LoggerFactory; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableMultimap; import com.google.common.collect.Lists; import com.google.common.collect.Multimap; @@ -126,16 +125,7 @@ public class MimePart { fileName, fileExtension, contentDisposition, - children, - parsedContent - .map(x -> x.getMetadata() - .entrySet() - .stream() - .reduce(ImmutableMultimap.<String, String>builder(), - (builder, entry) -> builder.putAll(entry.getKey(), entry.getValue()), - (builder1, builder2) -> builder1.putAll(builder2.build())).build()) - .orElse(ImmutableMultimap.of()) - ); + children); } private Optional<ParsedContent> parseContent(TextExtractor textExtractor) { @@ -175,11 +165,10 @@ public class MimePart { private final Optional<String> fileExtension; private final Optional<String> contentDisposition; private final List<MimePart> attachments; - private final ImmutableMultimap<String, String> metadata; private MimePart(HeaderCollection headerCollection, Optional<String> bodyTextContent, Optional<String> mediaType, Optional<String> subType, Optional<String> fileName, Optional<String> fileExtension, - Optional<String> contentDisposition, List<MimePart> attachments, Multimap<String, String> metadata) { + Optional<String> contentDisposition, List<MimePart> attachments) { this.headerCollection = headerCollection; this.mediaType = mediaType; this.subType = subType; @@ -188,7 +177,6 @@ public class MimePart { this.contentDisposition = contentDisposition; this.attachments = attachments; this.bodyTextContent = bodyTextContent; - this.metadata = ImmutableMultimap.copyOf(metadata); } @JsonIgnore @@ -236,11 +224,6 @@ public class MimePart { return bodyTextContent; } - @JsonProperty(JsonMessageConstants.Attachment.FILE_METADATA) - public ImmutableMultimap<String, String> getMetadata() { - return metadata; - } - @JsonIgnore public Optional<String> locateFirstTextBody() { return firstBody(textAttachments() http://git-wip-us.apache.org/repos/asf/james-project/blob/01dd7460/mailbox/store/src/test/resources/eml/htmlMail.json ---------------------------------------------------------------------- diff --git a/mailbox/store/src/test/resources/eml/htmlMail.json b/mailbox/store/src/test/resources/eml/htmlMail.json index 04cc4e3..30cbbf1 100644 --- a/mailbox/store/src/test/resources/eml/htmlMail.json +++ b/mailbox/store/src/test/resources/eml/htmlMail.json @@ -124,7 +124,6 @@ "subtype":"plain", "fileName":null, "fileExtension":null, - "fileMetadata":{}, "contentDisposition":null, "headers":{ "content-transfer-encoding":[ @@ -140,7 +139,6 @@ "mediaType":"text", "subtype":"html", "fileName":null, - "fileMetadata":{}, "fileExtension":null, "contentDisposition":null, "headers":{ http://git-wip-us.apache.org/repos/asf/james-project/blob/01dd7460/mailbox/store/src/test/resources/eml/nonTextual.json ---------------------------------------------------------------------- diff --git a/mailbox/store/src/test/resources/eml/nonTextual.json b/mailbox/store/src/test/resources/eml/nonTextual.json index c9f3ed9..1170d5a 100644 --- a/mailbox/store/src/test/resources/eml/nonTextual.json +++ b/mailbox/store/src/test/resources/eml/nonTextual.json @@ -85,11 +85,6 @@ "fileName":null, "fileExtension":null, "contentDisposition":null, - "fileMetadata":{ - "X-Parsed-By":["org.apache.tika.parser.DefaultParser","org.apache.tika.parser.txt.TXTParser"], - "Content-Encoding":["windows-1252"], - "Content-Type":["text/plain; charset=windows-1252"] - }, "headers":{ "content-transfer-encoding":["7bit"], "content-type":["text/plain; charset=utf-8"] @@ -102,44 +97,6 @@ "fileName":"toto.odt", "fileExtension":"odt", "contentDisposition":"attachment", - "fileMetadata":{ - "date":["2015-06-18T12:43:07.489893918"], - "meta:paragraph-count":["1"], - "meta:word-count":["7"], - "Table-Count":["0"], - "generator":["LibreOffice/4.4.3.2$Linux_X86_64 LibreOffice_project/40m0$Build-2"], - "Word-Count":["7"], - "dcterms:created":["2015-06-18T12:41:25.197399866"], - "dcterms:modified":["2015-06-18T12:43:07.489893918"], - "Last-Modified":["2015-06-18T12:43:07.489893918"], - "nbPara":["1"], - "Last-Save-Date":["2015-06-18T12:43:07.489893918"], - "meta:object-count":["0"], - "meta:character-count":["47"], - "Paragraph-Count":["1"], - "nbImg":["0"], - "meta:save-date":["2015-06-18T12:43:07.489893918"], - "modified":["2015-06-18T12:43:07.489893918"], - "Edit-Time":["PT1M42S"], - "meta:image-count":["0"], - "Image-Count":["0"], - "nbCharacter":["47"], - "nbObject":["0"], - "nbPage":["1"], - "Object-Count":["0"], - "nbWord":["7"], - "Content-Type":["application/vnd.oasis.opendocument.text"], - "X-Parsed-By":["org.apache.tika.parser.DefaultParser","org.apache.tika.parser.odf.OpenDocumentParser"], - "meta:creation-date":["2015-06-18T12:41:25.197399866"], - "meta:table-count":["0"], - "Creation-Date":["2015-06-18T12:41:25.197399866"], - "xmpTPg:NPages":["1"], - "Character Count":["47"], - "editing-cycles":["2"], - "Page-Count":["1"], - "nbTab":["0"], - "meta:page-count":["1"] - }, "headers":{ "content-transfer-encoding":["base64"], "content-disposition":["attachment; filename=\"toto.odt\""], http://git-wip-us.apache.org/repos/asf/james-project/blob/01dd7460/mailbox/store/src/test/resources/eml/recursiveMail.json ---------------------------------------------------------------------- diff --git a/mailbox/store/src/test/resources/eml/recursiveMail.json b/mailbox/store/src/test/resources/eml/recursiveMail.json index ba1ca5c..7a57fc6 100644 --- a/mailbox/store/src/test/resources/eml/recursiveMail.json +++ b/mailbox/store/src/test/resources/eml/recursiveMail.json @@ -98,7 +98,6 @@ "textContent": "Forward as attachment !\n\n\n", "mediaType": "text", "subtype": "plain", - "fileMetadata":{}, "fileName": null, "fileExtension": null, "contentDisposition": null, @@ -115,7 +114,6 @@ "textContent": null, "mediaType": null, "subtype": null, - "fileMetadata":{}, "fileName": null, "fileExtension": null, "contentDisposition": null, @@ -136,7 +134,6 @@ "mediaType": null, "subtype": null, "fileName": null, - "fileMetadata":{}, "fileExtension": null, "contentDisposition": null, "headers": { @@ -182,7 +179,6 @@ "subtype": "plain", "fileName": null, "fileExtension": null, - "fileMetadata":{}, "contentDisposition": null, "headers": { "content-transfer-encoding": [ @@ -195,7 +191,6 @@ }, { "textContent": null, - "fileMetadata":{}, "mediaType": "application", "subtype": "vnd.oasis.opendocument.spreadsheet", "fileName": "Sprint-2015-05-18.ods", http://git-wip-us.apache.org/repos/asf/james-project/blob/01dd7460/mailbox/store/src/test/resources/eml/spamMail.json ---------------------------------------------------------------------- diff --git a/mailbox/store/src/test/resources/eml/spamMail.json b/mailbox/store/src/test/resources/eml/spamMail.json index d133cd7..e101669 100644 --- a/mailbox/store/src/test/resources/eml/spamMail.json +++ b/mailbox/store/src/test/resources/eml/spamMail.json @@ -130,7 +130,6 @@ "mediaType": "text", "subtype": "plain", "fileName": null, - "fileMetadata":{}, "fileExtension": null, "contentDisposition": "inline", "headers": { @@ -151,7 +150,6 @@ "subtype": "rfc822-headers", "fileName": "header", "fileExtension": "", - "fileMetadata":{}, "contentDisposition": "inline", "headers": { "content-disposition": [ @@ -173,7 +171,6 @@ "mediaType": "text", "subtype": "plain", "fileName": null, - "fileMetadata":{}, "fileExtension": null, "contentDisposition": "inline", "headers": { --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
