This is an automated email from the ASF dual-hosted git repository.
tballison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git
The following commit(s) were added to refs/heads/main by this push:
new 5e655ab108 TIKA-4890: improve robustness on length check (#3210)
5e655ab108 is described below
commit 5e655ab1082953eeab6755dc4c088e69dcf41213
Author: Tim Allison <[email protected]>
AuthorDate: Sat Sep 19 08:43:08 2026 -0400
TIKA-4890: improve robustness on length check (#3210)
---
CHANGES.txt | 2 +-
.../org/apache/tika/parser/mail/RFC822Parser.java | 63 ++++++---
.../java/org/apache/tika/parser/mail/EMLXTest.java | 147 +++++++++++++++++----
3 files changed, 171 insertions(+), 41 deletions(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index 077021aee5..38f867a215 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -3,7 +3,7 @@ Release 4.1.0 - unreleased
* Apple Mail emlx files are detected as message/x-emlx (real files pad the
byte-count line to 10 columns and were falling through to text/html or
message/rfc822), and RFC822Parser now parses them, dropping the byte-count
- line and the trailing plist (TIKA-4890).
+ line and, when the count is intact, the trailing plist (TIKA-4890).
* The tika-server UAT gains an inference pass (TIKA-4913).
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-mail-module/src/main/java/org/apache/tika/parser/mail/RFC822Parser.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-mail-module/src/main/java/org/apache/tika/parser/mail/RFC822Parser.java
index cee6a203b1..391d8a0294 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-mail-module/src/main/java/org/apache/tika/parser/mail/RFC822Parser.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-mail-module/src/main/java/org/apache/tika/parser/mail/RFC822Parser.java
@@ -16,13 +16,18 @@
*/
package org.apache.tika.parser.mail;
+import static java.nio.charset.StandardCharsets.US_ASCII;
+
import java.io.IOException;
import java.io.InputStream;
+import java.nio.ByteBuffer;
+import java.nio.channels.SeekableByteChannel;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.Set;
+import org.apache.commons.io.IOUtils;
import org.apache.james.mime4j.MimeException;
import org.apache.james.mime4j.message.DefaultBodyDescriptorBuilder;
import org.apache.james.mime4j.parser.MimeStreamParser;
@@ -50,7 +55,9 @@ import org.apache.tika.sax.XHTMLContentHandler;
* corresponding parser and displayed within elements.
* <p/>
* Also handles Apple Mail's emlx framing: a first line carrying the message's
- * byte count, and an XML plist after the message. Both are dropped.
+ * byte count, and an XML plist after the message. Both are dropped when the
+ * count checks out; an edited file whose count no longer lands on the plist
+ * is parsed whole rather than cut.
* <p/>
* A {@link MimeConfig} object can be passed in the parsing context
* to better control the parsing process.
@@ -86,6 +93,8 @@ public class RFC822Parser implements Parser {
// digits, then Apple Mail's space padding to 10 columns, then the newline
private static final int EMLX_COUNT_LINE_MAX = 11;
+ private static final byte[] PLIST_START = "<?xml".getBytes(US_ASCII);
+
//rely on the detector to be thread-safe
//built lazily and then reused
private Detector detector;
@@ -145,12 +154,7 @@ public class RFC822Parser implements Parser {
xhtml.startDocument();
checkForZeroByte(tis);//avoid stackoverflow
try {
- InputStream message = tis;
- long emlxLength = readEmlxByteCount(tis);
- if (emlxLength >= 0) {
- message = new BoundedInputStream(emlxLength, tis);
- }
- parser.parse(message);
+ parser.parse(stripEmlxFraming(tis));
} catch (IOException e) {
tis.throwIfCauseOf(e);
throw new TikaException("Failed to parse an email message", e);
@@ -169,30 +173,57 @@ public class RFC822Parser implements Parser {
}
/**
- * Consumes the emlx byte-count line if the stream starts with one.
+ * Consumes the emlx byte-count line if the stream starts with one, and
bounds the
+ * message to that count when the plist really starts there.
*
- * @return the message's length in bytes, or -1 if the stream is not emlx
framed
+ * @return the stream to hand to mime4j
*/
- private static long readEmlxByteCount(TikaInputStream tis) throws
IOException {
- tis.mark(EMLX_COUNT_LINE_MAX);
+ private static InputStream stripEmlxFraming(TikaInputStream tis) throws
IOException {
+ byte[] head = new byte[EMLX_COUNT_LINE_MAX];
+ int n = tis.peek(head);
long count = 0;
int digits = 0;
boolean padding = false;
- for (int i = 0; i < EMLX_COUNT_LINE_MAX; i++) {
- int c = tis.read();
+ int lineLength = -1;
+ for (int i = 0; i < n; i++) {
+ int c = head[i];
if (c >= '0' && c <= '9' && !padding) {
count = count * 10 + (c - '0');
digits++;
} else if (c == ' ' && digits > 0) {
padding = true;
} else if (c == '\n' && digits > 0) {
- return count;
+ lineLength = i + 1;
+ break;
} else {
break;
}
}
- tis.reset();
- return -1;
+ if (lineLength < 0) {
+ return tis;
+ }
+ boolean verified = plistStartsAt(tis, lineLength + count);
+ IOUtils.skipFully(tis, lineLength);
+ return verified ? new BoundedInputStream(count, tis) : tis;
+ }
+
+ private static boolean plistStartsAt(TikaInputStream tis, long offset) {
+ // must run before any byte is consumed: a passthrough source refuses
a seekable
+ // view once its position has moved
+ try (SeekableByteChannel channel = tis.getSeekableByteChannel()) {
+ if (offset + PLIST_START.length > channel.size()) {
+ return false;
+ }
+ ByteBuffer buffer = ByteBuffer.allocate(PLIST_START.length);
+ channel.position(offset);
+ while (buffer.hasRemaining() && channel.read(buffer) != -1) {
+ // fill
+ }
+ return !buffer.hasRemaining() && Arrays.equals(buffer.array(),
PLIST_START);
+ } catch (IOException e) {
+ // unverifiable is not a reason to cut; a real read failure recurs
in mime4j
+ return false;
+ }
}
private void checkForZeroByte(TikaInputStream tstream) throws IOException,
ZeroByteFileException {
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-mail-module/src/test/java/org/apache/tika/parser/mail/EMLXTest.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-mail-module/src/test/java/org/apache/tika/parser/mail/EMLXTest.java
index 33a8fd4a25..c4ce2c37cd 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-mail-module/src/test/java/org/apache/tika/parser/mail/EMLXTest.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-mail-module/src/test/java/org/apache/tika/parser/mail/EMLXTest.java
@@ -16,11 +16,20 @@
*/
package org.apache.tika.parser.mail;
+import static java.nio.charset.StandardCharsets.US_ASCII;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
-import java.nio.charset.StandardCharsets;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.EnumSource;
import org.apache.tika.TikaTest;
import org.apache.tika.io.TikaInputStream;
@@ -37,65 +46,155 @@ import org.apache.tika.parser.Parser;
*/
public class EMLXTest extends TikaTest {
+ private static final String MULTIPART = "testEMLX_multipart_html.emlx";
+ private static final String MULTIPART_TEXT =
+ "The plan for the Foundation is in this message’s HTML part.";
+ private static final String SINGLE_PART = "testEMLX_single_part_html.emlx";
+ private static final String SINGLE_PART_TEXT = "The trial balance is ready
for review.";
+
private static final String PLIST = "<?xml version=\"1.0\"
encoding=\"UTF-8\"?>\n" +
"<!DOCTYPE plist PUBLIC \"-//Apple//DTD PLIST 1.0//EN\" " +
"\"http://www.apple.com/DTDs/PropertyList-1.0.dtd\";>\n" +
"<plist version=\"1.0\">\n<dict>\n\t<key>date-received</key>\n" +
"\t<integer>1698393810</integer>\n</dict>\n</plist>\n";
- @Test
- public void testLongHeaderBlockWithHtmlAlternative() throws Exception {
- XMLResult r = getXML("testEMLX_multipart_html.emlx");
+ /** The three TikaInputStream backings; the count check seeks differently
in each. */
+ enum Source { BYTES, STREAM, FILE }
+
+ @TempDir
+ Path tmp;
+
+ @ParameterizedTest
+ @EnumSource(Source.class)
+ public void testLongHeaderBlockWithHtmlAlternative(Source source) throws
Exception {
+ XMLResult r = parse(fixture(MULTIPART), source);
assertEquals("message/x-emlx",
r.metadata.get(HttpHeaders.CONTENT_TYPE));
assertEquals("Set the Foundation Straight",
r.metadata.get(TikaCoreProperties.TITLE));
assertEquals("[email protected]",
r.metadata.get(Message.MESSAGE_FROM_EMAIL));
assertEquals("2015-09-26T16:30:20Z",
r.metadata.get(TikaCoreProperties.CREATED));
- assertContains("The plan for the Foundation is in this message’s HTML
part.", r.xml);
+ assertContains(MULTIPART_TEXT, r.xml);
assertPlistAndCountDropped(r.xml, "2803");
}
- @Test
- public void testSinglePartHtml() throws Exception {
- XMLResult r = getXML("testEMLX_single_part_html.emlx");
+ @ParameterizedTest
+ @EnumSource(Source.class)
+ public void testSinglePartHtml(Source source) throws Exception {
+ XMLResult r = parse(fixture(SINGLE_PART), source);
assertEquals("message/x-emlx",
r.metadata.get(HttpHeaders.CONTENT_TYPE));
assertEquals("Trial balance",
r.metadata.get(TikaCoreProperties.TITLE));
- assertContains("The trial balance is ready for review.", r.xml);
+ assertContains(SINGLE_PART_TEXT, r.xml);
assertPlistAndCountDropped(r.xml, "446");
}
+ @ParameterizedTest
+ @EnumSource(Source.class)
+ public void testStaleCountIsNotTrusted(Source source) throws Exception {
+ // an edited file whose count no longer lands on the plist must not be
cut
+ for (String fixture : new String[]{MULTIPART, SINGLE_PART}) {
+ byte[] good = fixture(fixture);
+ String text = fixture.equals(MULTIPART) ? MULTIPART_TEXT :
SINGLE_PART_TEXT;
+ for (long bad : new long[]{0, 100, 1500, 99999}) {
+ XMLResult r = parse(withCount(good, bad), source);
+ assertEquals("message/x-emlx",
r.metadata.get(HttpHeaders.CONTENT_TYPE),
+ fixture + " count=" + bad);
+ assertContains(text, r.xml);
+ assertNotContained(paddedCount(bad), body(r.xml));
+ }
+ }
+ }
+
@Test
public void testUnpaddedByteCount() throws Exception {
String message = "Subject: Unpadded\nFrom: [email protected]\nTo:
[email protected]\n\n" +
"the body\n";
- String emlx = message.length() + "\n" + message + PLIST;
- try (TikaInputStream tis =
TikaInputStream.get(emlx.getBytes(StandardCharsets.US_ASCII))) {
- XMLResult r = getXML(tis, new AutoDetectParser(), new Metadata());
- assertEquals("message/x-emlx",
r.metadata.get(HttpHeaders.CONTENT_TYPE));
- assertEquals("Unpadded", r.metadata.get(TikaCoreProperties.TITLE));
- assertContains("the body", r.xml);
- assertPlistAndCountDropped(r.xml,
Integer.toString(message.length()));
- }
+ byte[] emlx = (message.length() + "\n" + message +
PLIST).getBytes(US_ASCII);
+ XMLResult r = parse(emlx, Source.BYTES);
+ assertEquals("message/x-emlx",
r.metadata.get(HttpHeaders.CONTENT_TYPE));
+ assertEquals("Unpadded", r.metadata.get(TikaCoreProperties.TITLE));
+ assertContains("the body", r.xml);
+ assertPlistAndCountDropped(r.xml, Integer.toString(message.length()));
}
@Test
public void testForcedTypeStillStripsFraming() throws Exception {
// an emlx handed to the parser as message/rfc822 must not leak the
plist either
String message = "Subject: Forced\nFrom: [email protected]\n\nthe
body\n";
- String emlx = (message.length() + " \n") + message + PLIST;
+ byte[] emlx = ((message.length() + " \n") + message +
PLIST).getBytes(US_ASCII);
+ XMLResult r = parseAsRfc822(emlx);
+ assertEquals("Forced", r.metadata.get(TikaCoreProperties.TITLE));
+ assertContains("the body", r.xml);
+ assertPlistAndCountDropped(r.xml, Integer.toString(message.length()));
+ }
+
+ @Test
+ public void testCrlfIsNotFraming() throws Exception {
+ // a CRLF-converted file has a stale count and a "\r\n" count line:
left alone
+ byte[] crlf = new String(fixture(SINGLE_PART), US_ASCII).replace("\n",
"\r\n")
+ .getBytes(US_ASCII);
+ XMLResult r = parseAsRfc822(crlf);
+ assertEquals("Trial balance",
r.metadata.get(TikaCoreProperties.TITLE));
+ assertContains(SINGLE_PART_TEXT, r.xml);
+ }
+
+ private XMLResult parse(byte[] emlx, Source source) throws Exception {
+ try (TikaInputStream tis = open(emlx, source)) {
+ return getXML(tis, new AutoDetectParser(), new Metadata());
+ }
+ }
+
+ private XMLResult parseAsRfc822(byte[] bytes) throws Exception {
Metadata metadata = new Metadata();
metadata.set(HttpHeaders.CONTENT_TYPE, "message/rfc822");
ParseContext context = new ParseContext();
context.set(Parser.class, new AutoDetectParser());
- try (TikaInputStream tis =
TikaInputStream.get(emlx.getBytes(StandardCharsets.US_ASCII))) {
- XMLResult r = getXML(tis, new RFC822Parser(), metadata, context);
- assertEquals("Forced", r.metadata.get(TikaCoreProperties.TITLE));
- assertContains("the body", r.xml);
- assertPlistAndCountDropped(r.xml,
Integer.toString(message.length()));
+ try (TikaInputStream tis = TikaInputStream.get(bytes)) {
+ return getXML(tis, new RFC822Parser(), metadata, context);
+ }
+ }
+
+ private TikaInputStream open(byte[] bytes, Source source) throws
IOException {
+ switch (source) {
+ case BYTES:
+ return TikaInputStream.get(bytes);
+ case STREAM:
+ return TikaInputStream.get(new ByteArrayInputStream(bytes));
+ default:
+ Path p = Files.createTempFile(tmp, "emlx", ".emlx");
+ Files.write(p, bytes);
+ return TikaInputStream.get(p);
+ }
+ }
+
+ private static byte[] fixture(String name) throws IOException {
+ try (InputStream in =
EMLXTest.class.getResourceAsStream("/test-documents/" + name)) {
+ assertNotNull(in, name);
+ return in.readAllBytes();
+ }
+ }
+
+ /** Replaces the fixture's 10-column count line. */
+ private static byte[] withCount(byte[] emlx, long count) {
+ byte[] line = (paddedCount(count) + "\n").getBytes(US_ASCII);
+ byte[] out = new byte[emlx.length];
+ System.arraycopy(line, 0, out, 0, 11);
+ System.arraycopy(emlx, 11, out, 11, emlx.length - 11);
+ return out;
+ }
+
+ private static String paddedCount(long count) {
+ StringBuilder sb = new StringBuilder(Long.toString(count));
+ while (sb.length() < 10) {
+ sb.append(' ');
}
+ return sb.toString();
+ }
+
+ private static String body(String xml) {
+ return xml.substring(xml.indexOf("<body"));
}
private static void assertPlistAndCountDropped(String xml, String
byteCount) {
- String body = xml.substring(xml.indexOf("<body"));
+ String body = body(xml);
assertNotContained(byteCount, body);
assertNotContained("date-received", body);
assertNotContained("plist", body);
