Repository: tika
Updated Branches:
  refs/heads/2.x 1e55953bc -> 30e03de89


TIKA-2122: Extract all headers from MSG/RFC822


Project: http://git-wip-us.apache.org/repos/asf/tika/repo
Commit: http://git-wip-us.apache.org/repos/asf/tika/commit/30e03de8
Tree: http://git-wip-us.apache.org/repos/asf/tika/tree/30e03de8
Diff: http://git-wip-us.apache.org/repos/asf/tika/diff/30e03de8

Branch: refs/heads/2.x
Commit: 30e03de89fd4b21cb91917c72aec12eede761be3
Parents: 1e55953
Author: tballison <talli...@mitre.org>
Authored: Mon Oct 17 14:30:33 2016 -0400
Committer: tballison <talli...@mitre.org>
Committed: Mon Oct 17 14:30:33 2016 -0400

----------------------------------------------------------------------
 CHANGES.txt                                     |  2 +
 .../java/org/apache/tika/metadata/Message.java  |  6 ++
 .../src/test/java/org/apache/tika/TikaTest.java |  8 ++
 .../tika-parser-office-bundle/pom.xml           |  3 +-
 tika-parser-modules/pom.xml                     |  1 +
 .../tika-parser-office-module/pom.xml           |  5 ++
 .../tika/parser/microsoft/OutlookExtractor.java | 85 +++++++++++++++++++-
 .../parser/microsoft/OutlookParserTest.java     | 15 ++++
 .../tika-parser-web-module/pom.xml              |  5 +-
 .../tika/parser/mail/MailContentHandler.java    |  5 ++
 .../tika/parser/mail/RFC822ParserTest.java      |  1 +
 11 files changed, 130 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tika/blob/30e03de8/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 226d9c1..48180ed 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -17,6 +17,8 @@ Release 2.0 - ???
 
 Release 1.14 - ???
 
+  * Extract all headers from MSG/RFC822 (TIKA-2122).
+
   * Upgrade metadata extractor to 2.9.1 (TIKA-2113).
 
   * Extract PDF DocInfo metadata into separate keys to prevent

http://git-wip-us.apache.org/repos/asf/tika/blob/30e03de8/tika-core/src/main/java/org/apache/tika/metadata/Message.java
----------------------------------------------------------------------
diff --git a/tika-core/src/main/java/org/apache/tika/metadata/Message.java 
b/tika-core/src/main/java/org/apache/tika/metadata/Message.java
index ffb9413..dad3952 100644
--- a/tika-core/src/main/java/org/apache/tika/metadata/Message.java
+++ b/tika-core/src/main/java/org/apache/tika/metadata/Message.java
@@ -16,10 +16,16 @@
  */
 package org.apache.tika.metadata;
 
+import org.apache.tika.Tika;
+
 /**
  * A collection of Message related property names.
  */
 public interface Message {
+    String MESSAGE_PREFIX = "Message"+ Metadata.NAMESPACE_PREFIX_DELIMITER;
+
+    String MESSAGE_RAW_HEADER_PREFIX = 
MESSAGE_PREFIX+"Raw-Header"+Metadata.NAMESPACE_PREFIX_DELIMITER;
+
     String MESSAGE_RECIPIENT_ADDRESS = "Message-Recipient-Address";
     
     String MESSAGE_FROM = "Message-From";

http://git-wip-us.apache.org/repos/asf/tika/blob/30e03de8/tika-core/src/test/java/org/apache/tika/TikaTest.java
----------------------------------------------------------------------
diff --git a/tika-core/src/test/java/org/apache/tika/TikaTest.java 
b/tika-core/src/test/java/org/apache/tika/TikaTest.java
index 847eb79..cfed800 100644
--- a/tika-core/src/test/java/org/apache/tika/TikaTest.java
+++ b/tika-core/src/test/java/org/apache/tika/TikaTest.java
@@ -322,4 +322,12 @@ public abstract class TikaTest {
             i++;
         }
     }
+
+    public static void debug(Metadata metadata) {
+        for (String n : metadata.names()) {
+            for (String v : metadata.getValues(n)) {
+                System.out.println(n + " : "+v);
+            }
+        }
+    }
 }

http://git-wip-us.apache.org/repos/asf/tika/blob/30e03de8/tika-parser-bundles/tika-parser-office-bundle/pom.xml
----------------------------------------------------------------------
diff --git a/tika-parser-bundles/tika-parser-office-bundle/pom.xml 
b/tika-parser-bundles/tika-parser-office-bundle/pom.xml
index 1529c97..1ef8b05 100644
--- a/tika-parser-bundles/tika-parser-office-bundle/pom.xml
+++ b/tika-parser-bundles/tika-parser-office-bundle/pom.xml
@@ -69,9 +69,10 @@
               poi;inline=true,
               poi-scratchpad;inline=true,
               poi-ooxml;inline=true,
-              poi-ooxml-schemas;inline=true;
+              poi-ooxml-schemas;inline=true,
               jackcess;inline=true,
               jackcess-encrypt;inline=true,
+              apache-mime4j-core;inline=true,
               java-libpst;inline=true,
               curvesapi;inline=true,
               xmlbeans;inline=true,

http://git-wip-us.apache.org/repos/asf/tika/blob/30e03de8/tika-parser-modules/pom.xml
----------------------------------------------------------------------
diff --git a/tika-parser-modules/pom.xml b/tika-parser-modules/pom.xml
index dc3b409..c99cd0f 100644
--- a/tika-parser-modules/pom.xml
+++ b/tika-parser-modules/pom.xml
@@ -43,6 +43,7 @@
     <!-- used by POI, PDFBox and Jackcess ...try to sync -->
     <bouncycastle.version>1.54</bouncycastle.version>
     <commons.exec>1.3</commons.exec>
+    <mime4j.version>0.7.2</mime4j.version>
   </properties>
 
   <modules>

http://git-wip-us.apache.org/repos/asf/tika/blob/30e03de8/tika-parser-modules/tika-parser-office-module/pom.xml
----------------------------------------------------------------------
diff --git a/tika-parser-modules/tika-parser-office-module/pom.xml 
b/tika-parser-modules/tika-parser-office-module/pom.xml
index 9cd6c2f..9c50c31 100644
--- a/tika-parser-modules/tika-parser-office-module/pom.xml
+++ b/tika-parser-modules/tika-parser-office-module/pom.xml
@@ -91,6 +91,11 @@
       <version>0.8.1</version>
     </dependency>
     <dependency>
+      <groupId>org.apache.james</groupId>
+      <artifactId>apache-mime4j-core</artifactId>
+      <version>${mime4j.version}</version>
+    </dependency>
+    <dependency>
       <groupId>${project.groupId}</groupId>
       <artifactId>tika-parser-package-module</artifactId>
       <version>${project.version}</version>

http://git-wip-us.apache.org/repos/asf/tika/blob/30e03de8/tika-parser-modules/tika-parser-office-module/src/main/java/org/apache/tika/parser/microsoft/OutlookExtractor.java
----------------------------------------------------------------------
diff --git 
a/tika-parser-modules/tika-parser-office-module/src/main/java/org/apache/tika/parser/microsoft/OutlookExtractor.java
 
b/tika-parser-modules/tika-parser-office-module/src/main/java/org/apache/tika/parser/microsoft/OutlookExtractor.java
index d8d26aa..17d45d2 100644
--- 
a/tika-parser-modules/tika-parser-office-module/src/main/java/org/apache/tika/parser/microsoft/OutlookExtractor.java
+++ 
b/tika-parser-modules/tika-parser-office-module/src/main/java/org/apache/tika/parser/microsoft/OutlookExtractor.java
@@ -26,13 +26,17 @@ import java.nio.charset.IllegalCharsetNameException;
 import java.nio.charset.UnsupportedCharsetException;
 import java.text.ParseException;
 import java.text.SimpleDateFormat;
+import java.util.ArrayList;
 import java.util.Date;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
+import org.apache.james.mime4j.codec.DecodeMonitor;
+import org.apache.james.mime4j.codec.DecoderUtil;
 import org.apache.poi.hmef.attribute.MAPIRtfAttribute;
 import org.apache.poi.hsmf.MAPIMessage;
 import org.apache.poi.hsmf.datatypes.AttachmentChunks;
@@ -68,6 +72,18 @@ import org.xml.sax.SAXException;
  * Outlook Message Parser.
  */
 public class OutlookExtractor extends AbstractPOIFSExtractor {
+
+
+    private static Pattern HEADER_KEY_PAT =
+            Pattern.compile("\\A([\\x21-\\x39\\x3B-\\x7E]+):(.*?)\\Z");
+    //this according to the spec; in practice, it is probably more likely
+    //that a "split field" fails to start with a space character than
+    //that a real header contains anything but [-_A-Za-z0-9].
+    //e.g.
+    //header: this header goes onto the next line
+    //<mailto:x...@cnn.com...
+
+
     private static final Metadata EMPTY_METADATA = new Metadata();
     private final SimpleDateFormat dateFormat;
     private final EncodingDetector htmlEncodingDetectorProxy;
@@ -124,8 +140,19 @@ public class OutlookExtractor extends 
AbstractPOIFSExtractor {
                 }
             } catch (ChunkNotFoundException he) {
             } // Will be fixed in POI 3.7 Final
+            try {
+                Map<String, String[]> headers = 
normalizeHeaders(msg.getHeaders());
+                for (Map.Entry<String, String[]> e : headers.entrySet()) {
+                    String headerKey = e.getKey();
+                    for (String headerValue : e.getValue()) {
+                        
metadata.add(Metadata.MESSAGE_RAW_HEADER_PREFIX+headerKey, headerValue);
+                    }
+                }
+            } catch (ChunkNotFoundException e) {
+
+            }
 
-            // Date - try two ways to find it
+                    // Date - try two ways to find it
             // First try via the proper chunk
             if (msg.getMessageDate() != null) {
                 metadata.set(TikaCoreProperties.CREATED, 
msg.getMessageDate().getTime());
@@ -270,6 +297,62 @@ public class OutlookExtractor extends 
AbstractPOIFSExtractor {
         }
     }
 
+    //As of 3.15, POI currently returns header[] by splitting on /\r?\n/
+    //this rebuilds headers that are broken up over several lines
+    //this also decodes encoded headers.
+    private Map<String, String[]> normalizeHeaders(String[] rows) {
+        Map<String, String[]> ret = new LinkedHashMap<>();
+        if (rows == null) {
+            return ret;
+        }
+        StringBuilder sb = new StringBuilder();
+        Map<String, List<String>> headers = new LinkedHashMap();
+        Matcher headerKeyMatcher = HEADER_KEY_PAT.matcher("");
+        String lastKey = null;
+        int consec = 0;
+        for (String row : rows) {
+            headerKeyMatcher.reset(row);
+            if (headerKeyMatcher.find()) {
+                if (lastKey != null) {
+                    List<String> vals = headers.get(lastKey);
+                    vals = (vals == null) ? new ArrayList<String>() : vals;
+                    vals.add(decodeHeader(sb.toString()));
+                    headers.put(lastKey, vals);
+                }
+                //reset sb
+                sb.setLength(0);
+                lastKey = headerKeyMatcher.group(1).trim();
+                sb.append(headerKeyMatcher.group(2).trim());
+                consec = 0;
+            } else {
+                if (consec > 0) {
+                    sb.append("\n");
+                }
+                sb.append(row);
+            }
+            consec++;
+        }
+
+        //make sure to add the last value
+        if (sb.length() > 0 && lastKey != null) {
+            List<String> vals = headers.get(lastKey);
+            vals = (vals == null) ? new ArrayList<String>() : vals;
+            vals.add(decodeHeader(sb.toString()));
+            headers.put(lastKey, vals);
+        }
+
+        //convert to array
+        for (Map.Entry<String, List<String>> e : headers.entrySet()) {
+            ret.put(e.getKey(), e.getValue().toArray(new 
String[e.getValue().size()]));
+        }
+        return ret;
+
+    }
+
+    private String decodeHeader(String header) {
+        return DecoderUtil.decodeEncodedWords(header, DecodeMonitor.SILENT);
+    }
+
     private void header(XHTMLContentHandler xhtml, String key, String value)
             throws SAXException {
         if (value != null && value.length() > 0) {

http://git-wip-us.apache.org/repos/asf/tika/blob/30e03de8/tika-parser-modules/tika-parser-office-module/src/test/java/org/apache/tika/parser/microsoft/OutlookParserTest.java
----------------------------------------------------------------------
diff --git 
a/tika-parser-modules/tika-parser-office-module/src/test/java/org/apache/tika/parser/microsoft/OutlookParserTest.java
 
b/tika-parser-modules/tika-parser-office-module/src/test/java/org/apache/tika/parser/microsoft/OutlookParserTest.java
index 8662e65..c15308f 100644
--- 
a/tika-parser-modules/tika-parser-office-module/src/test/java/org/apache/tika/parser/microsoft/OutlookParserTest.java
+++ 
b/tika-parser-modules/tika-parser-office-module/src/test/java/org/apache/tika/parser/microsoft/OutlookParserTest.java
@@ -26,6 +26,7 @@ import javax.xml.transform.sax.TransformerHandler;
 import javax.xml.transform.stream.StreamResult;
 import java.io.InputStream;
 import java.io.StringWriter;
+import java.util.Arrays;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -71,6 +72,12 @@ public class OutlookParserTest extends TikaTest {
                 "L'\u00C9quipe Microsoft Outlook Express",
                 metadata.get(Metadata.AUTHOR));
 
+        //ensure that "raw" header is correctly decoded
+        assertEquals(
+                "L'\u00C9quipe Microsoft Outlook Express <m...@microsoft.com>",
+                metadata.get(Metadata.MESSAGE_RAW_HEADER_PREFIX+"From"));
+
+
         // Stored as Thu, 5 Apr 2007 09:26:06 -0700
         assertEquals(
                 "2007-04-05T16:26:06Z",
@@ -108,6 +115,14 @@ public class OutlookParserTest extends TikaTest {
         Matcher matcher = pattern.matcher(content);
         assertTrue(matcher.find());
         assertFalse(matcher.find());
+
+        //test that last header is added
+        assertContains("29 Jan 2009 19:17:10.0163 (UTC) 
FILETIME=[2ED25E30:01C98246]",
+                
Arrays.asList(metadata.getValues("Message:Raw-Header:X-OriginalArrivalTime")));
+        //confirm next line is added correctly
+        assertContains("from athena.apache.org (HELO athena.apache.org) 
(140.211.11.136)\n" +
+                "    by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 29 Jan 2009 
11:17:08 -0800",
+                
Arrays.asList(metadata.getValues("Message:Raw-Header:Received")));
     }
 
     /**

http://git-wip-us.apache.org/repos/asf/tika/blob/30e03de8/tika-parser-modules/tika-parser-web-module/pom.xml
----------------------------------------------------------------------
diff --git a/tika-parser-modules/tika-parser-web-module/pom.xml 
b/tika-parser-modules/tika-parser-web-module/pom.xml
index ee9e24c..6f11184 100644
--- a/tika-parser-modules/tika-parser-web-module/pom.xml
+++ b/tika-parser-modules/tika-parser-web-module/pom.xml
@@ -22,10 +22,7 @@
   <artifactId>tika-parser-web-module</artifactId>
   <name>Apache Tika parser web module</name>
   <url>http://tika.apache.org/</url>
-  
-  <properties>
-    <mime4j.version>0.7.2</mime4j.version>
-  </properties>
+
   
   <dependencies>
     <dependency>

http://git-wip-us.apache.org/repos/asf/tika/blob/30e03de8/tika-parser-modules/tika-parser-web-module/src/main/java/org/apache/tika/parser/mail/MailContentHandler.java
----------------------------------------------------------------------
diff --git 
a/tika-parser-modules/tika-parser-web-module/src/main/java/org/apache/tika/parser/mail/MailContentHandler.java
 
b/tika-parser-modules/tika-parser-web-module/src/main/java/org/apache/tika/parser/mail/MailContentHandler.java
index b42eb89..458ed01 100644
--- 
a/tika-parser-modules/tika-parser-web-module/src/main/java/org/apache/tika/parser/mail/MailContentHandler.java
+++ 
b/tika-parser-modules/tika-parser-web-module/src/main/java/org/apache/tika/parser/mail/MailContentHandler.java
@@ -51,6 +51,7 @@ import org.apache.tika.config.TikaConfig;
 import org.apache.tika.extractor.EmbeddedDocumentExtractor;
 import org.apache.tika.extractor.ParsingEmbeddedDocumentExtractor;
 import org.apache.tika.io.TikaInputStream;
+import org.apache.tika.metadata.Message;
 import org.apache.tika.metadata.Metadata;
 import org.apache.tika.metadata.TikaCoreProperties;
 import org.apache.tika.parser.AutoDetectParser;
@@ -238,6 +239,7 @@ class MailContentHandler implements ContentHandler {
 
         try {
             String fieldname = field.getName();
+
             ParsedField parsedField = LenientFieldParser.getParser().parse(
                     field, DecodeMonitor.SILENT);
             if (fieldname.equalsIgnoreCase("From")) {
@@ -276,6 +278,9 @@ class MailContentHandler implements ContentHandler {
                     date = tryOtherDateFormats(field.getBody());
                 }
                 metadata.set(TikaCoreProperties.CREATED, date);
+            } else {
+                
metadata.add(Metadata.MESSAGE_RAW_HEADER_PREFIX+parsedField.getName(),
+                        field.getBody());
             }
         } catch (RuntimeException me) {
             if (strictParsing) {

http://git-wip-us.apache.org/repos/asf/tika/blob/30e03de8/tika-parser-modules/tika-parser-web-module/src/test/java/org/apache/tika/parser/mail/RFC822ParserTest.java
----------------------------------------------------------------------
diff --git 
a/tika-parser-modules/tika-parser-web-module/src/test/java/org/apache/tika/parser/mail/RFC822ParserTest.java
 
b/tika-parser-modules/tika-parser-web-module/src/test/java/org/apache/tika/parser/mail/RFC822ParserTest.java
index 3be1edd..4a506be 100644
--- 
a/tika-parser-modules/tika-parser-web-module/src/test/java/org/apache/tika/parser/mail/RFC822ParserTest.java
+++ 
b/tika-parser-modules/tika-parser-web-module/src/test/java/org/apache/tika/parser/mail/RFC822ParserTest.java
@@ -355,6 +355,7 @@ public class RFC822ParserTest extends TikaTest {
         assertContains("TEST DATA FOR TIKA.", handler.toString());
         assertContains("This is text inside an unencrypted zip file", 
handler.toString());
         assertContains("TIKA-1028", handler.toString());
+        assertEquals("<juha.ha...@gmail.com>", 
metadata.get("Message:Raw-Header:Return-Path"));
     }
 
     /**

Reply via email to