This is an automated email from the ASF dual-hosted git repository.
tballison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git
The following commit(s) were added to refs/heads/main by this push:
new 847b2a2bfa TIKA-5051: skip MP4 boxes over 2 GB instead of stopping the
walk (#3547)
847b2a2bfa is described below
commit 847b2a2bfad028469a3180efaa26edc2b36a15b3
Author: Dominik Schmidt <[email protected]>
AuthorDate: Fri Nov 6 20:42:20 2026 +0100
TIKA-5051: skip MP4 boxes over 2 GB instead of stopping the walk (#3547)
TikaMp4Reader stopped at any box declaring more than Integer.MAX_VALUE
bytes, before deciding whether to read or skip it. An mdat over 2 GB in
front of moov therefore lost all metadata. Only boxes that are read into
memory need the limit, and those are already capped by maxBoxSize.
Co-authored-by: Tim Allison <[email protected]>
---
CHANGES.txt | 3 ++
.../org/apache/tika/parser/mp4/TikaMp4Reader.java | 6 +--
.../org/apache/tika/parser/mp4/MP4ParserTest.java | 44 ++++++++++++++++++++++
3 files changed, 48 insertions(+), 5 deletions(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index 866d6b1430..eee8b6110b 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -89,6 +89,9 @@ Release 4.3.0 - ???
EXTRACTION AND FORMATS
+ * MP4 and QuickTime files with a box over 2 GB, typically an mdat in front
of
+ moov, keep their metadata: the walk skips such a box instead of stopping
(TIKA-5051).
+
* WOFF and WOFF2 fonts (font/woff, font/woff2), SquashFS images
(application/vnd.squashfs), Adobe Swatch Exchange files, PalmDOC e-books
and binary
OpenPGP signatures (application/pgp-signature) are detected by content;
they were
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/main/java/org/apache/tika/parser/mp4/TikaMp4Reader.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/main/java/org/apache/tika/parser/mp4/TikaMp4Reader.java
index 9ae6aada6b..e113f72cfe 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/main/java/org/apache/tika/parser/mp4/TikaMp4Reader.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/main/java/org/apache/tika/parser/mp4/TikaMp4Reader.java
@@ -85,10 +85,6 @@ final class TikaMp4Reader {
if (headerSize == 16) {
boxSize = reader.getInt64();
}
- if (boxSize > Integer.MAX_VALUE) {
- handler.addError("Box size too large.");
- break;
- }
if (boxSize < headerSize) {
handler.addError("Box size too small.");
break;
@@ -106,7 +102,7 @@ final class TikaMp4Reader {
//deliberate: unlike the TikaMemoryLimitException other
parsers throw,
//this keeps the remaining boxes' metadata; the skip is
recorded as a
//warning via the directory's error list.
- boolean tooLarge = payloadLength > maxBoxSize;
+ boolean tooLarge = payloadLength > Math.min(maxBoxSize,
Integer.MAX_VALUE);
boolean beyondInput = inputLength >= 0
&& reader.getPosition() + payloadLength >
inputLength;
if (tooLarge || beyondInput) {
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/test/java/org/apache/tika/parser/mp4/MP4ParserTest.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/test/java/org/apache/tika/parser/mp4/MP4ParserTest.java
index 3a78d71e53..935cd5f1fe 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/test/java/org/apache/tika/parser/mp4/MP4ParserTest.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/test/java/org/apache/tika/parser/mp4/MP4ParserTest.java
@@ -25,6 +25,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.InputStream;
+import java.io.SequenceInputStream;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
@@ -651,6 +652,49 @@ public class MP4ParserTest extends TikaTest {
assertEquals("isom", majorBrand(bos.toByteArray(), 1000L));
}
+ @Test
+ public void testBoxOver2GbDoesNotStopTheWalk() throws Exception {
+ long mdatSize = 3L * 1024 * 1024 * 1024 + 16;
+ ByteBuffer mdat = ByteBuffer.allocate(16);
+
mdat.putInt(1).put("mdat".getBytes(StandardCharsets.ISO_8859_1)).putLong(mdatSize);
+ byte[] ftyp = ftypBox();
+ long length = mdatSize + ftyp.length;
+ InputStream stream = new
SequenceInputStream(Collections.enumeration(List.of(
+ new ByteArrayInputStream(mdat.array()), new
ZeroInputStream(mdatSize - 16),
+ new ByteArrayInputStream(ftyp))));
+ com.drew.metadata.Metadata mp4Metadata = new
com.drew.metadata.Metadata();
+ Metadata tikaMetadata = new Metadata();
+ TikaMp4BoxHandler handler = new TikaMp4BoxHandler(mp4Metadata,
tikaMetadata,
+ new XHTMLContentHandler(new DefaultHandler(), tikaMetadata),
new ParseContext());
+ TikaMp4Reader.extract(stream, handler, 1000L, length, tikaMetadata,
new ParseContext());
+ Mp4Directory dir =
mp4Metadata.getFirstDirectoryOfType(Mp4Directory.class);
+ assertEquals("isom", dir == null ? null :
dir.getString(Mp4Directory.TAG_MAJOR_BRAND));
+ }
+
+ private static final class ZeroInputStream extends InputStream {
+ private long remaining;
+
+ ZeroInputStream(long length) {
+ remaining = length;
+ }
+
+ @Override
+ public int read() {
+ if (remaining == 0) {
+ return -1;
+ }
+ remaining--;
+ return 0;
+ }
+
+ @Override
+ public long skip(long n) {
+ long skipped = Math.min(n, remaining);
+ remaining -= skipped;
+ return skipped;
+ }
+ }
+
private static String majorBrand(byte[] boxes, long maxBoxSize) throws
Exception {
com.drew.metadata.Metadata mp4Metadata = new
com.drew.metadata.Metadata();
Metadata tikaMetadata = new Metadata();
