This is an automated email from the ASF dual-hosted git repository.
tballison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git
The following commit(s) were added to refs/heads/main by this push:
new da7e4d80a2 TIKA-4878: zip and friends (#3134)
da7e4d80a2 is described below
commit da7e4d80a200f7902485e30014bc36aef8e99fc1
Author: Tim Allison <[email protected]>
AuthorDate: Fri Sep 4 15:09:10 2026 -0400
TIKA-4878: zip and friends (#3134)
---
CHANGES.txt | 12 +-
.../tika/extractor/RewindRecordingExtractor.java | 85 ++++++++++++++
.../tika/parser/apple/AppleSingleFileParser.java | 58 +++++++---
.../parser/iwork/iwana/IWork13PackageParser.java | 3 +-
.../parser/iwork/iwana/IWork18PackageParser.java | 9 +-
.../apple/AppleSingleFileParserNoTempFileTest.java | 100 ++++++++++++++++
.../org/apache/tika/parser/mbox/MboxParser.java | 13 ++-
.../tika/parser/mbox/MboxParserNoTempFileTest.java | 62 ++++++++++
.../org/apache/tika/parser/epub/EpubParser.java | 20 ++--
.../tika/parser/geogebra/GeoGebraParser.java | 12 +-
.../apache/tika/parser/odf/OpenDocumentParser.java | 12 +-
.../parser/odf/EmbeddedEntriesNoTempFileTest.java | 128 +++++++++++++++++++++
.../org/apache/tika/parser/wacz/WACZParser.java | 16 ++-
.../org/apache/tika/parser/xliff/XLZParser.java | 4 +-
14 files changed, 488 insertions(+), 46 deletions(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index f1a0032763..f15a629c07 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,5 +1,14 @@
Release 4.1.0 - unreleased
+ * Entries of ODF, EPUB, GeoGebra, WACZ, XLZ and iWork containers, mbox
+ messages and the AppleSingle data fork are re-opened from their
+ container on rewind instead of cached: digesting rewinds every
+ embedded document, and the cached copy cost heap for the whole entry
+ and, past the cache budget or the 1 MB floor, a temp file. AppleSingle
+ no longer spools its data fork to a temp file on every parse, and
+ GeoGebra no longer spools every embedded picture to detect it
+ (TIKA-4878).
+
* A declared Content-Length is no longer treated as a measurement: the
zip-bomb ratio counts only measured input bytes (a container-declared
size on an embedded document could inflate its denominator), and a
@@ -7,6 +16,7 @@ Release 4.1.0 - unreleased
from the declared length (a lying one could push a small payload to
disk or churn the shared budget). Neither is in a release: the
exposure arrived with TIKA-4868 and TIKA-4873 (TIKA-4878).
+
* Embedded objects in Office documents are re-opened from their container
instead of cached: every OOXML part (pictures, media, attachments), the
OLE 2.0 package inside an OOXML part, the CONTENTS entry of an OLE 2.0
@@ -15,7 +25,7 @@ Release 4.1.0 - unreleased
the cached copy that made possible cost heap for the whole object and,
past the cache budget or the 1 MB floor, a temp file. The container
hands the bytes back on demand, so neither is needed (TIKA-4878).
-
+
* PDF attachments, PDF XMP packets, 3D on-instantiate scripts and PST
attachments are re-opened from their document instead of cached when
the embedded-document extractor rewinds them (digesting does, for every
diff --git
a/tika-core/src/test/java/org/apache/tika/extractor/RewindRecordingExtractor.java
b/tika-core/src/test/java/org/apache/tika/extractor/RewindRecordingExtractor.java
new file mode 100644
index 0000000000..0f27865bf9
--- /dev/null
+++
b/tika-core/src/test/java/org/apache/tika/extractor/RewindRecordingExtractor.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.extractor;
+
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.xml.sax.ContentHandler;
+
+import org.apache.tika.io.TikaInputStream;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.parser.ParseContext;
+
+/**
+ * Test double for parsers that hand embedded documents over. Rewinds each
+ * embedded stream the way a digester does, then records whether that left the
+ * stream backed by a temp file and how many bytes it still yields.
+ * <p>
+ * Assert on what this extractor was handed, not on a watched temp directory:
+ * a parser owns each child's {@code TemporaryResources}, so a directory watch
+ * passes whatever the parser did.
+ */
+public class RewindRecordingExtractor implements EmbeddedDocumentExtractor {
+
+ private final List<Boolean> spooled = new ArrayList<>();
+ private final List<Integer> lengths = new ArrayList<>();
+
+ @Override
+ public boolean shouldParseEmbedded(Metadata metadata, ParseContext
context) {
+ return true;
+ }
+
+ @Override
+ public void parseEmbedded(TikaInputStream stream, ContentHandler handler,
Metadata metadata,
+ ParseContext context, boolean outputHtml) throws
IOException {
+ stream.enableRewind();
+ stream.readAllBytes();
+ stream.rewind();
+ spooled.add(stream.hasFile());
+ lengths.add(stream.readAllBytes().length);
+ }
+
+ /** Byte counts of the embedded documents seen, in order. */
+ public List<Integer> lengths() {
+ return Collections.unmodifiableList(lengths);
+ }
+
+ public void assertSawLength(int length) {
+ assertTrue(lengths.contains(length),
+ "an embedded document of " + length + " bytes reached the
extractor; saw "
+ + lengths);
+ }
+
+ public void assertSawLengthAtLeast(int length) {
+ assertTrue(lengths.stream().anyMatch(l -> l >= length),
+ "an embedded document of at least " + length + " bytes reached
the extractor; saw "
+ + lengths);
+ }
+
+ public void assertNothingSpooled() {
+ for (int i = 0; i < spooled.size(); i++) {
+ assertFalse(spooled.get(i), "embedded stream " + i + " (" +
lengths.get(i)
+ + " bytes) was spooled to disk to rewind instead of
re-opened");
+ }
+ }
+}
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-apple-module/src/main/java/org/apache/tika/parser/apple/AppleSingleFileParser.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-apple-module/src/main/java/org/apache/tika/parser/apple/AppleSingleFileParser.java
index 4628a47726..8808b40822 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-apple-module/src/main/java/org/apache/tika/parser/apple/AppleSingleFileParser.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-apple-module/src/main/java/org/apache/tika/parser/apple/AppleSingleFileParser.java
@@ -18,6 +18,8 @@ package org.apache.tika.parser.apple;
import java.io.IOException;
import java.io.InputStream;
+import java.nio.channels.Channels;
+import java.nio.channels.SeekableByteChannel;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collections;
@@ -35,7 +37,9 @@ import org.apache.tika.exception.TikaException;
import org.apache.tika.exception.TikaMemoryLimitException;
import org.apache.tika.extractor.EmbeddedDocumentExtractor;
import org.apache.tika.extractor.EmbeddedDocumentUtil;
+import org.apache.tika.io.CacheMemoryBudget;
import org.apache.tika.io.EndianUtils;
+import org.apache.tika.io.TemporaryResources;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.metadata.TikaCoreProperties;
@@ -86,36 +90,54 @@ public class AppleSingleFileParser implements Parser {
EmbeddedDocumentExtractor ex =
EmbeddedDocumentUtil.getEmbeddedDocumentExtractor(context);
+ //the data fork is handed over as a region of this stream, which needs
a
+ //seekable view after the header has been read sequentially
+ tis.enableRewind(context.get(CacheMemoryBudget.class));
short numEntries = readThroughNumEntries(tis);
- long bytesRead = 26;
List<FieldInfo> fieldInfoList = getSortedFieldInfoList(tis,
numEntries);
- bytesRead += 12 * numEntries;
Metadata embeddedMetadata = Metadata.newInstance(context);
- bytesRead = processFieldEntries(tis, fieldInfoList, embeddedMetadata,
bytesRead);
+ processFieldEntries(tis, fieldInfoList, embeddedMetadata, 26 + 12L *
numEntries);
FieldInfo contentFieldInfo = getContentFieldInfo(fieldInfoList);
XHTMLContentHandler xhtml = new XHTMLContentHandler(handler, metadata,
context);
xhtml.startDocument();
- if (contentFieldInfo != null) {
- long diff = contentFieldInfo.offset - bytesRead;
- IOUtils.skipFully(tis, diff);
- if (ex.shouldParseEmbedded(embeddedMetadata, context)) {
- // Use BoundedInputStream to limit bytes read, then spool to
temp file
- // for complete isolation from parent stream (reset() goes to
embedded start)
- BoundedInputStream bounded =
- BoundedInputStream.builder()
- .setInputStream(tis)
- .setMaxCount(contentFieldInfo.length)
- .get();
- try (TikaInputStream inner = TikaInputStream.get(bounded)) {
- inner.getPath();
- ex.parseEmbedded(inner, xhtml, embeddedMetadata, context,
true);
- }
+ if (contentFieldInfo != null &&
ex.shouldParseEmbedded(embeddedMetadata, context)) {
+ //re-opened from the channel on rewind: a digest re-reads the fork
in place
+ //instead of the copy-and-spool that getPath() used to force on
every parse
+ long offset = contentFieldInfo.offset;
+ long length = contentFieldInfo.length;
+ try (TikaInputStream inner = TikaInputStream.get(() -> region(tis,
offset, length),
+ new TemporaryResources(), null)) {
+ ex.parseEmbedded(inner, xhtml, embeddedMetadata, context,
true);
}
}
xhtml.endDocument();
}
+ /**
+ * The data fork as a fresh stream over the parent's seekable channel: in
memory
+ * when the parent is, from its file when it has one. The offset and
length are
+ * the file's own claims; a region past the end simply reads as empty.
+ */
+ private static InputStream region(TikaInputStream tis, long offset, long
length)
+ throws IOException {
+ if (offset < 0 || length < 0) {
+ throw new IOException("AppleSingle data fork out of range:
offset=" + offset +
+ " length=" + length);
+ }
+ SeekableByteChannel channel = tis.getSeekableByteChannel();
+ try {
+ channel.position(offset);
+ return BoundedInputStream.builder()
+ .setInputStream(Channels.newInputStream(channel))
+ .setMaxCount(length)
+ .get();
+ } catch (IOException e) {
+ channel.close();
+ throw e;
+ }
+ }
+
private FieldInfo getContentFieldInfo(List<FieldInfo> fieldInfoList) {
for (FieldInfo fieldInfo : fieldInfoList) {
if (fieldInfo.entryId == 1) {
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-apple-module/src/main/java/org/apache/tika/parser/iwork/iwana/IWork13PackageParser.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-apple-module/src/main/java/org/apache/tika/parser/iwork/iwana/IWork13PackageParser.java
index 61c273d281..37cbef7a4e 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-apple-module/src/main/java/org/apache/tika/parser/iwork/iwana/IWork13PackageParser.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-apple-module/src/main/java/org/apache/tika/parser/iwork/iwana/IWork13PackageParser.java
@@ -155,7 +155,8 @@ public class IWork13PackageParser implements Parser {
if (type == null) {
type = IWork13DocumentType.detectIfPossible(entry);
}
- try (TikaInputStream tis =
TikaInputStream.get(zipFile.getInputStream(entry))) {
+ try (TikaInputStream tis = TikaInputStream.get(() ->
zipFile.getInputStream(entry),
+ new TemporaryResources(), null)) {
processZipEntry(entry, tis, metadata, xhtml, parseContext,
embeddedDocumentExtractor);
} catch (SecurityException e) {
throw e;
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-apple-module/src/main/java/org/apache/tika/parser/iwork/iwana/IWork18PackageParser.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-apple-module/src/main/java/org/apache/tika/parser/iwork/iwana/IWork18PackageParser.java
index a98566855d..b6a860d2fc 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-apple-module/src/main/java/org/apache/tika/parser/iwork/iwana/IWork18PackageParser.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-apple-module/src/main/java/org/apache/tika/parser/iwork/iwana/IWork18PackageParser.java
@@ -35,6 +35,7 @@ import org.apache.tika.annotation.TikaComponent;
import org.apache.tika.exception.TikaException;
import org.apache.tika.extractor.EmbeddedDocumentExtractor;
import org.apache.tika.extractor.EmbeddedDocumentUtil;
+import org.apache.tika.io.TemporaryResources;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.HttpHeaders;
import org.apache.tika.metadata.Metadata;
@@ -92,8 +93,7 @@ public class IWork18PackageParser implements Parser {
type = IWork18DocumentType.detectIfPossible(entry);
}
if (isPreview(entry) && zipFile.canReadEntryData(entry)) {
- try (TikaInputStream previewStream =
-
TikaInputStream.get(zipFile.getInputStream(entry))) {
+ try (TikaInputStream previewStream = entryStream(zipFile,
entry)) {
handleThumbnail(entry, previewStream, xhtml, context);
}
}
@@ -127,6 +127,11 @@ public class IWork18PackageParser implements Parser {
String name = entry.getName();
return name.equals("preview.jpg") || name.endsWith("/preview.jpg");
}
+ /** Re-opened from the zip on rewind rather than cached: the entry is in
the container already. */
+ private static TikaInputStream entryStream(ZipFile zipFile,
ZipArchiveEntry entry) {
+ return TikaInputStream.get(() -> zipFile.getInputStream(entry), new
TemporaryResources(), null);
+ }
+
private static void handleThumbnail(ZipEntry entry, TikaInputStream
previewStream,
XHTMLContentHandler xhtml,
ParseContext context)
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-apple-module/src/test/java/org/apache/tika/parser/apple/AppleSingleFileParserNoTempFileTest.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-apple-module/src/test/java/org/apache/tika/parser/apple/AppleSingleFileParserNoTempFileTest.java
new file mode 100644
index 0000000000..6a91971164
--- /dev/null
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-apple-module/src/test/java/org/apache/tika/parser/apple/AppleSingleFileParserNoTempFileTest.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.apple;
+
+import java.io.ByteArrayOutputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Random;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.xml.sax.helpers.DefaultHandler;
+
+import org.apache.tika.extractor.EmbeddedDocumentExtractor;
+import org.apache.tika.extractor.RewindRecordingExtractor;
+import org.apache.tika.io.TikaInputStream;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.parser.ParseContext;
+
+/**
+ * The data fork of an AppleSingle file is a region of the file itself. The
+ * stream handed to the embedded-document extractor must re-read that region on
+ * rewind, from memory when the parent is in memory and from the file when it
+ * has one, rather than copy it to a temp file. The parser used to force that
+ * copy on every parse, digester or not.
+ */
+public class AppleSingleFileParserNoTempFileTest {
+
+ private static final int PAYLOAD_LENGTH = 2 * 1024 * 1024;
+ private static final int HEADER_LENGTH = 26 + 12;
+
+ @TempDir
+ Path tempDir;
+
+ @Test
+ public void testDataForkFromFileIsNotSpooled() throws Exception {
+ Path file = tempDir.resolve("fork.as");
+ Files.write(file, appleSingle());
+ try (TikaInputStream tis = TikaInputStream.get(file, new Metadata())) {
+ RewindRecordingExtractor extractor = parse(tis);
+ extractor.assertSawLength(PAYLOAD_LENGTH);
+ extractor.assertNothingSpooled();
+ }
+ }
+
+ @Test
+ public void testDataForkFromMemoryIsNotSpooled() throws Exception {
+ try (TikaInputStream tis = TikaInputStream.get(appleSingle())) {
+ RewindRecordingExtractor extractor = parse(tis);
+ extractor.assertSawLength(PAYLOAD_LENGTH);
+ extractor.assertNothingSpooled();
+ }
+ }
+
+ private static RewindRecordingExtractor parse(TikaInputStream tis) throws
Exception {
+ RewindRecordingExtractor extractor = new RewindRecordingExtractor();
+ ParseContext context = new ParseContext();
+ context.set(EmbeddedDocumentExtractor.class, extractor);
+ new AppleSingleFileParser().parse(tis, new DefaultHandler(), new
Metadata(), context);
+ return extractor;
+ }
+
+ /** Header, one entry (the data fork, id 1) and the fork itself. */
+ private static byte[] appleSingle() {
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ int32(out, 0x00051600);
+ int32(out, 0x00020000);
+ out.writeBytes(new byte[16]);
+ out.write(0);
+ out.write(1);
+ int32(out, 1);
+ int32(out, HEADER_LENGTH);
+ int32(out, PAYLOAD_LENGTH);
+ byte[] payload = new byte[PAYLOAD_LENGTH];
+ new Random(4878).nextBytes(payload);
+ out.writeBytes(payload);
+ return out.toByteArray();
+ }
+
+ private static void int32(ByteArrayOutputStream out, int v) {
+ out.write((v >>> 24) & 0xFF);
+ out.write((v >>> 16) & 0xFF);
+ out.write((v >>> 8) & 0xFF);
+ out.write(v & 0xFF);
+ }
+}
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-mail-module/src/main/java/org/apache/tika/parser/mbox/MboxParser.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-mail-module/src/main/java/org/apache/tika/parser/mbox/MboxParser.java
index 0aea3dcd50..da909baac9 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-mail-module/src/main/java/org/apache/tika/parser/mbox/MboxParser.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-mail-module/src/main/java/org/apache/tika/parser/mbox/MboxParser.java
@@ -40,6 +40,7 @@ import org.apache.tika.annotation.TikaComponent;
import org.apache.tika.exception.TikaException;
import org.apache.tika.extractor.EmbeddedDocumentExtractor;
import org.apache.tika.extractor.EmbeddedDocumentUtil;
+import org.apache.tika.io.TemporaryResources;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.HttpHeaders;
import org.apache.tika.metadata.KeyPrefix;
@@ -144,11 +145,13 @@ public class MboxParser implements Parser {
saveHeaderInMetadata(mailMetadata, item);
}
- TikaInputStream msgStream =
TikaInputStream.get(message.toInputStream());
- message = null;
-
- if (extractor.shouldParseEmbedded(mailMetadata, context)) {
- extractor.parseEmbedded(msgStream, xhtml,
mailMetadata, context, true);
+ //re-opened over the buffer on rewind rather than cached:
the
+ //message is in memory already, a digest must not copy it
again
+ try (TikaInputStream msgStream =
TikaInputStream.get(message::toInputStream,
+ new TemporaryResources(), null)) {
+ if (extractor.shouldParseEmbedded(mailMetadata,
context)) {
+ extractor.parseEmbedded(msgStream, xhtml,
mailMetadata, context, true);
+ }
}
if (tracking) {
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-mail-module/src/test/java/org/apache/tika/parser/mbox/MboxParserNoTempFileTest.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-mail-module/src/test/java/org/apache/tika/parser/mbox/MboxParserNoTempFileTest.java
new file mode 100644
index 0000000000..9960bc4804
--- /dev/null
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-mail-module/src/test/java/org/apache/tika/parser/mbox/MboxParserNoTempFileTest.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.mbox;
+
+import java.nio.charset.StandardCharsets;
+
+import org.junit.jupiter.api.Test;
+import org.xml.sax.helpers.DefaultHandler;
+
+import org.apache.tika.extractor.EmbeddedDocumentExtractor;
+import org.apache.tika.extractor.RewindRecordingExtractor;
+import org.apache.tika.io.TikaInputStream;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.parser.ParseContext;
+
+/**
+ * An mbox message is buffered in memory before it is handed over. Rewinding
the
+ * stream the embedded-document extractor gets -- which digesting does for
every
+ * embedded document -- must re-read that buffer, not copy it and spill the
copy
+ * to a temp file. The message is over the 1 MB a cache keeps in memory, so the
+ * difference is observable.
+ */
+public class MboxParserNoTempFileTest {
+
+ private static final int BODY_LENGTH = 2 * 1024 * 1024;
+
+ @Test
+ public void testMessageIsNotSpooled() throws Exception {
+ StringBuilder mbox = new StringBuilder(BODY_LENGTH + 200);
+ mbox.append("From [email protected] Thu Sep 4 10:00:00 2026\n")
+ .append("From: [email protected]\n")
+ .append("Subject: big\n\n");
+ String line = "x".repeat(99) + "\n";
+ while (mbox.length() < BODY_LENGTH) {
+ mbox.append(line);
+ }
+ RewindRecordingExtractor extractor = new RewindRecordingExtractor();
+ ParseContext context = new ParseContext();
+ context.set(EmbeddedDocumentExtractor.class, extractor);
+ try (TikaInputStream tis =
+
TikaInputStream.get(mbox.toString().getBytes(StandardCharsets.US_ASCII))) {
+ new MboxParser().parse(tis, new DefaultHandler(), new Metadata(),
context);
+ }
+ //the parser keeps everything but the divider line, so a little under
the input
+ extractor.assertSawLengthAtLeast(BODY_LENGTH - 100);
+ extractor.assertNothingSpooled();
+ }
+}
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/epub/EpubParser.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/epub/EpubParser.java
index 7428a5b6ae..ad0ad4ee90 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/epub/EpubParser.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/epub/EpubParser.java
@@ -51,6 +51,7 @@ import org.apache.tika.exception.TikaException;
import org.apache.tika.exception.WriteLimitReachedException;
import org.apache.tika.extractor.EmbeddedDocumentExtractor;
import org.apache.tika.extractor.EmbeddedDocumentUtil;
+import org.apache.tika.io.TemporaryResources;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.HttpHeaders;
import org.apache.tika.metadata.Metadata;
@@ -184,7 +185,7 @@ public class EpubParser implements Parser {
return fallbackParseAllHtmlEntries(zipFile, bodyHandler,
normalizer, metadata, context,
"OPF entry missing or unreadable in (possibly truncated)
container");
}
- try (TikaInputStream tis =
TikaInputStream.get(zipFile.getInputStream(zae))) {
+ try (TikaInputStream tis = entryStream(zipFile, zae)) {
opf.parse(tis, new DefaultHandler(), metadata, context);
}
@@ -234,7 +235,7 @@ public class EpubParser implements Parser {
}
zae = zipFile.getEntry(relativePath + hRefMediaPair.href);
if (zae != null) {
- try (TikaInputStream tis =
TikaInputStream.get(zipFile.getInputStream(zae))) {
+ try (TikaInputStream tis = entryStream(zipFile, zae)) {
content.parse(tis, bodyHandler, metadata, context);
spineParsed++;
} catch (SAXException e) {
@@ -345,7 +346,7 @@ public class EpubParser implements Parser {
if (!zipFile.canReadEntryData(entry)) {
continue;
}
- try (TikaInputStream tis =
TikaInputStream.get(zipFile.getInputStream(entry))) {
+ try (TikaInputStream tis = entryStream(zipFile, entry)) {
content.parse(tis, bodyHandler, metadata, context);
parsed++;
} catch (SAXException e) {
@@ -428,6 +429,11 @@ public class EpubParser implements Parser {
* @param cover whether this is the publication's cover image, emitted as
* the {@link
TikaCoreProperties.EmbeddedResourceType#THUMBNAIL}
*/
+ /** Re-opened from the zip on rewind rather than cached: the entry is in
the container already. */
+ private static TikaInputStream entryStream(ZipFile zipFile,
ZipArchiveEntry entry) {
+ return TikaInputStream.get(() -> zipFile.getInputStream(entry), new
TemporaryResources(), null);
+ }
+
private void handleEmbedded(ZipFile zipFile, String relativePath,
HRefMediaPair hRefMediaPair,
boolean cover,
EmbeddedDocumentExtractor
embeddedDocumentExtractor,
@@ -456,14 +462,14 @@ public class EpubParser implements Parser {
return;
}
- TikaInputStream tis = null;
+ //open once now so a broken entry is recorded here, not mid-parse
try {
- tis = TikaInputStream.get(zipFile.getInputStream(ze));
+ zipFile.getInputStream(ze).close();
} catch (IOException e) {
- //store this exception in the parent's metadata
EmbeddedDocumentUtil.recordEmbeddedStreamException(e,
parentMetadata, context);
return;
}
+ TikaInputStream tis = entryStream(zipFile, ze);
xhtml.startElement("div", "class", "embedded");
try {
@@ -491,7 +497,7 @@ public class EpubParser implements Parser {
}
zae = zipFile.getEntry("metadata.xml");
if (zae != null && zipFile.canReadEntryData(zae)) {
- try (TikaInputStream tis =
TikaInputStream.get(zipFile.getInputStream(zae))) {
+ try (TikaInputStream tis = entryStream(zipFile, zae)) {
meta.parse(tis, new DefaultHandler(), metadata, context);
}
}
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/geogebra/GeoGebraParser.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/geogebra/GeoGebraParser.java
index 4b725006e1..e8033ec742 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/geogebra/GeoGebraParser.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/geogebra/GeoGebraParser.java
@@ -44,6 +44,8 @@ import org.apache.tika.exception.WriteLimitReachedException;
import org.apache.tika.extractor.EmbeddedDocumentExtractor;
import org.apache.tika.extractor.EmbeddedDocumentUtil;
import org.apache.tika.io.BoundedInputStream;
+import org.apache.tika.io.CacheMemoryBudget;
+import org.apache.tika.io.TemporaryResources;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.HttpHeaders;
import org.apache.tika.metadata.Metadata;
@@ -429,13 +431,15 @@ public class GeoGebraParser implements Parser {
if (page != null) {
PageAnchoring.applyPageMetadata(embeddedMetadata,
Collections.singleton(page));
}
- try (TikaInputStream tisZip =
TikaInputStream.get(zipFile.getInputStream(entry))) {
+ //re-opened from the zip on rewind rather than cached or spooled: the
entry
+ //is in the container already, so detection and a digest re-read it in
place
+ try (TikaInputStream tisZip = TikaInputStream.get(() ->
zipFile.getInputStream(entry),
+ new TemporaryResources(), null)) {
if (type == null) {
- //spool so the stream can be rewound after detection
- tisZip.getFile();
+ tisZip.enableRewind(context.get(CacheMemoryBudget.class));
MediaType mediaType = EmbeddedDocumentUtil.getDetector(context)
.detect(tisZip, embeddedMetadata, context);
- tisZip.reset();
+ tisZip.rewind();
if (mediaType != null) {
embeddedMetadata.set(HttpHeaders.CONTENT_TYPE,
mediaType.toString());
}
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/OpenDocumentParser.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/OpenDocumentParser.java
index 5cc0f37a1c..9c4124b9fe 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/OpenDocumentParser.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/OpenDocumentParser.java
@@ -46,6 +46,7 @@ import org.apache.tika.exception.TikaException;
import org.apache.tika.extractor.EmbeddedDocumentExtractor;
import org.apache.tika.extractor.EmbeddedDocumentUtil;
import org.apache.tika.io.CacheMemoryBudget;
+import org.apache.tika.io.TemporaryResources;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.HttpHeaders;
import org.apache.tika.metadata.Metadata;
@@ -208,6 +209,11 @@ public class OpenDocumentParser implements Parser {
return extractMacros;
}
+ /** Re-opened from the zip on rewind rather than cached: the entry is in
the container already. */
+ private static TikaInputStream entryStream(ZipFile zipFile,
ZipArchiveEntry entry) {
+ return TikaInputStream.get(() -> zipFile.getInputStream(entry), new
TemporaryResources(), null);
+ }
+
private void handleZipFile(ZipFile zipFile, Metadata metadata,
ParseContext context,
EndDocumentShieldingContentHandler handler,
EmbeddedDocumentExtractor
embeddedDocumentExtractor)
@@ -226,7 +232,7 @@ public class OpenDocumentParser implements Parser {
ZipArchiveEntry entry = zipFile.getEntry(MANIFEST_NAME);
if (entry != null) {
- try (TikaInputStream tisZip =
TikaInputStream.get(zipFile.getInputStream(entry))) {
+ try (TikaInputStream tisZip = entryStream(zipFile, entry)) {
handleZipArchiveEntry(entry, tisZip, metadata, context,
handler,
embeddedDocumentExtractor, picturePages);
}
@@ -234,7 +240,7 @@ public class OpenDocumentParser implements Parser {
entry = zipFile.getEntry(META_NAME);
if (entry != null) {
- try (TikaInputStream tisZip =
TikaInputStream.get(zipFile.getInputStream(entry))) {
+ try (TikaInputStream tisZip = entryStream(zipFile, entry)) {
handleZipArchiveEntry(entry, tisZip, metadata, context,
handler,
embeddedDocumentExtractor, picturePages);
}
@@ -244,7 +250,7 @@ public class OpenDocumentParser implements Parser {
while (entries.hasMoreElements()) {
entry = entries.nextElement();
if (!META_NAME.equals(entry.getName())) {
- try (TikaInputStream tis =
TikaInputStream.get(zipFile.getInputStream(entry))) {
+ try (TikaInputStream tis = entryStream(zipFile, entry)) {
handleZipArchiveEntry(entry, tis, metadata, context,
handler,
embeddedDocumentExtractor, picturePages);
}
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/test/java/org/apache/tika/parser/odf/EmbeddedEntriesNoTempFileTest.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/test/java/org/apache/tika/parser/odf/EmbeddedEntriesNoTempFileTest.java
new file mode 100644
index 0000000000..e39aaa3de4
--- /dev/null
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/test/java/org/apache/tika/parser/odf/EmbeddedEntriesNoTempFileTest.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.odf;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Enumeration;
+import java.util.Random;
+
+import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
+import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream;
+import org.apache.commons.compress.archivers.zip.ZipFile;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.xml.sax.helpers.DefaultHandler;
+
+import org.apache.tika.TikaTest;
+import org.apache.tika.extractor.EmbeddedDocumentExtractor;
+import org.apache.tika.extractor.RewindRecordingExtractor;
+import org.apache.tika.io.TikaInputStream;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.parser.ParseContext;
+import org.apache.tika.parser.Parser;
+import org.apache.tika.parser.epub.EpubParser;
+
+/**
+ * An entry of an ODF or EPUB container is in the zip already. Rewinding the
+ * stream handed to the embedded-document extractor -- which digesting does for
+ * every embedded document, and which the ODF parser itself does to detect a
+ * picture -- must re-open the entry, not cache a copy and spill it to a temp
+ * file. The payload is over the 1 MB a cache keeps in memory, so the
difference
+ * is observable; it is built at test time by rewriting one entry of a fixture.
+ */
+public class EmbeddedEntriesNoTempFileTest extends TikaTest {
+
+ private static final int PAYLOAD_LENGTH = 2 * 1024 * 1024;
+
+ @TempDir
+ Path tempDir;
+
+ @Test
+ public void testOdfPictureIsNotSpooled() throws Exception {
+ Path odt = withEntryReplaced("testODTEmbeddedImageLink.odt",
+ "Pictures/10000201000001240000006457F5B1D1243E0671.png",
"picture.odt");
+ RewindRecordingExtractor extractor = parse(odt, new
OpenDocumentParser());
+ extractor.assertSawLength(PAYLOAD_LENGTH);
+ extractor.assertNothingSpooled();
+ }
+
+ @Test
+ public void testEpubResourceIsNotSpooled() throws Exception {
+ Path epub = withEntryReplaced("testEPUB.epub", "OPS/CoverDesign.jpg",
"cover.epub");
+ RewindRecordingExtractor extractor = parse(epub, new EpubParser());
+ extractor.assertSawLength(PAYLOAD_LENGTH);
+ extractor.assertNothingSpooled();
+ }
+
+ private RewindRecordingExtractor parse(Path file, Parser parser) throws
Exception {
+ RewindRecordingExtractor extractor = new RewindRecordingExtractor();
+ ParseContext context = new ParseContext();
+ context.set(EmbeddedDocumentExtractor.class, extractor);
+ Metadata metadata = new Metadata();
+ try (TikaInputStream tis = TikaInputStream.get(file, metadata)) {
+ parser.parse(tis, new DefaultHandler(), metadata, context);
+ }
+ return extractor;
+ }
+
+ /**
+ * A copy of the fixture with one entry's bytes replaced by the payload;
every
+ * other entry is copied raw, so the stored {@code mimetype} entry stays
stored.
+ */
+ private Path withEntryReplaced(String fixture, String entryName, String
name)
+ throws IOException {
+ Path original = tempDir.resolve("original-" + name);
+ try (InputStream is = getResourceAsStream("/test-documents/" +
fixture)) {
+ Files.copy(is, original);
+ }
+ Path copy = tempDir.resolve(name);
+ boolean replaced = false;
+ try (ZipFile in = ZipFile.builder().setPath(original).get();
+ ZipArchiveOutputStream out = new
ZipArchiveOutputStream(copy.toFile())) {
+ Enumeration<ZipArchiveEntry> entries = in.getEntries();
+ while (entries.hasMoreElements()) {
+ ZipArchiveEntry entry = entries.nextElement();
+ if (entry.getName().equals(entryName)) {
+ ZipArchiveEntry big = new ZipArchiveEntry(entryName);
+ big.setMethod(ZipArchiveEntry.DEFLATED);
+ out.putArchiveEntry(big);
+ out.write(payload());
+ out.closeArchiveEntry();
+ replaced = true;
+ } else {
+ try (InputStream raw = in.getRawInputStream(entry)) {
+ out.addRawArchiveEntry(entry, raw);
+ }
+ }
+ }
+ }
+ if (!replaced) {
+ throw new IOException(entryName + " is not in " + fixture);
+ }
+ return copy;
+ }
+
+ /** Incompressible filler, so DEFLATE keeps it at full size. */
+ private static byte[] payload() {
+ byte[] bytes = new byte[PAYLOAD_LENGTH];
+ new Random(4878).nextBytes(bytes);
+ return bytes;
+ }
+}
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-webarchive-module/src/main/java/org/apache/tika/parser/wacz/WACZParser.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-webarchive-module/src/main/java/org/apache/tika/parser/wacz/WACZParser.java
index e165b1c74e..fd96a6a33f 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-webarchive-module/src/main/java/org/apache/tika/parser/wacz/WACZParser.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-webarchive-module/src/main/java/org/apache/tika/parser/wacz/WACZParser.java
@@ -37,6 +37,7 @@ import org.apache.tika.annotation.TikaComponent;
import org.apache.tika.exception.TikaException;
import org.apache.tika.extractor.EmbeddedDocumentExtractor;
import org.apache.tika.extractor.EmbeddedDocumentUtil;
+import org.apache.tika.io.TemporaryResources;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.HttpHeaders;
import org.apache.tika.metadata.Metadata;
@@ -140,6 +141,11 @@ public class WACZParser implements Parser {
}
}
+ /** Re-opened from the zip on rewind rather than cached: the entry is in
the container already. */
+ private static TikaInputStream entryStream(ZipFile zipFile,
ZipArchiveEntry entry) {
+ return TikaInputStream.get(() -> zipFile.getInputStream(entry), new
TemporaryResources(), null);
+ }
+
private void processZip(ZipFile zip, XHTMLContentHandler xhtml, Metadata
metadata,
EmbeddedDocumentExtractor ex, ParseContext
context) throws IOException, SAXException {
@@ -149,12 +155,14 @@ public class WACZParser implements Parser {
String name = zae.getName();
if (name.startsWith("archive/")) {
name = name.substring(8);
- processWARC(TikaInputStream.get(zip.getInputStream(zae)), zae,
name, xhtml,
- metadata, ex, context);
+ try (TikaInputStream tis = entryStream(zip, zae)) {
+ processWARC(tis, zae, name, xhtml, metadata, ex, context);
+ }
} else if ("datapackage.json".equals(name)) {
//no-op
-
processDataPackage(TikaInputStream.get(zip.getInputStream(zae)), zae, xhtml,
- metadata);
+ try (TikaInputStream tis = entryStream(zip, zae)) {
+ processDataPackage(tis, zae, xhtml, metadata);
+ }
}
}
}
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-xml-module/src/main/java/org/apache/tika/parser/xliff/XLZParser.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-xml-module/src/main/java/org/apache/tika/parser/xliff/XLZParser.java
index 8d66c937a9..3ff0445666 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-xml-module/src/main/java/org/apache/tika/parser/xliff/XLZParser.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-xml-module/src/main/java/org/apache/tika/parser/xliff/XLZParser.java
@@ -30,6 +30,7 @@ import org.xml.sax.SAXException;
import org.apache.tika.annotation.TikaComponent;
import org.apache.tika.exception.TikaException;
+import org.apache.tika.io.TemporaryResources;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.mime.MediaType;
@@ -137,7 +138,8 @@ public class XLZParser implements Parser {
while (entries.hasMoreElements()) {
ZipEntry entry = entries.nextElement();
if (entry.getName().contains(XLF)) {
- try (TikaInputStream tisZip =
TikaInputStream.get(zipFile.getInputStream(entry))) {
+ try (TikaInputStream tisZip = TikaInputStream.get(
+ () -> zipFile.getInputStream(entry), new
TemporaryResources(), null)) {
xliffParser.parse(tisZip, handler, metadata, context);
}
}
