This is an automated email from the ASF dual-hosted git repository. tballison pushed a commit to branch TIKA-4835-spill-less in repository https://gitbox.apache.org/repos/asf/tika.git
commit 1645e95d9937cb26942b2b823c4508180e1875b5 Author: tallison <[email protected]> AuthorDate: Mon Aug 24 20:39:12 2026 -0400 TIKA-4835-spill-less --- CHANGES.txt | 17 +++++ .../java/org/apache/tika/digest/DigestHelper.java | 30 ++++++-- .../org/apache/tika/digest/TranslatedBytes.java | 85 +++++++++++++++++++++ .../apache/tika/digest/TranslatedBytesTest.java | 86 ++++++++++++++++++++++ .../tika/parser/image/ImageMetadataExtractor.java | 32 ++++++-- .../org/apache/tika/parser/image/ImageXmp.java | 16 ++-- .../org/apache/tika/parser/image/JpegParser.java | 9 ++- .../org/apache/tika/parser/image/TiffParser.java | 13 +++- .../org/apache/tika/parser/image/WebPParser.java | 8 +- .../parser/image/ImageParsersNoTempFileTest.java | 86 ++++++++++++++++++++++ .../detect/microsoft/POIFSContainerDetector.java | 57 +++++++++++++- .../POIFSContainerDetectorNoTempFileTest.java | 71 ++++++++++++++++++ .../apache/tika/parser/odf/OpenDocumentParser.java | 7 +- .../odf/OpenDocumentParserNoTempFileTest.java | 68 +++++++++++++++++ .../java/org/apache/tika/parser/pdf/PDFParser.java | 11 ++- .../tika/parser/pdf/PDFParserNoTempFileTest.java | 71 ++++++++++++++++++ 16 files changed, 632 insertions(+), 35 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index eea5ad329b..92f6a8e060 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,5 +1,22 @@ Release 4.1.0 - unreleased + * The JPEG, TIFF and WebP parsers no longer spool in-memory input to a temp + file to read metadata, and the OpenDocument parser no longer spools each + inline picture before detecting it: both now rewind the stream (governed + by the CacheMemoryBudget when one is set) instead of calling getFile(). + On a 20k-file corpus sample these sites accounted for ~70% of the temp + bytes 4.x wrote, the dominant driver of 4.x's batch slowdown vs 3.x on + spinning disks. ImageMetadataExtractor gains InputStream overloads of + parseJpeg/parseTiff/parseWebP; ImageXmp.extractJpeg/extractWebp now take + an InputStream. POIFSContainerDetector opens in-memory OLE2 objects from + memory (under the CacheMemoryBudget, falling back to the file path for + anything POI's stream loader rejects) instead of spooling every embedded + OLE2 object to read its entry names, and the digest of translated + embedded streams (DigestHelper) buffers the translated bytes in memory + up to the budget before spilling. PDFParser's incremental-update xref + scan reads in-memory input from memory instead of spooling it + (TIKA-4835). + * The Kafka pipes iterator no longer stops at the first empty poll. A newly subscribed consumer spends its first poll(s) joining the group and returns empty even when the topic has a backlog, so the iterator could enqueue zero diff --git a/tika-core/src/main/java/org/apache/tika/digest/DigestHelper.java b/tika-core/src/main/java/org/apache/tika/digest/DigestHelper.java index 359ebf9f27..27ba26db1b 100644 --- a/tika-core/src/main/java/org/apache/tika/digest/DigestHelper.java +++ b/tika-core/src/main/java/org/apache/tika/digest/DigestHelper.java @@ -18,8 +18,6 @@ package org.apache.tika.digest; import java.io.IOException; import java.io.OutputStream; -import java.nio.file.Files; -import java.nio.file.Path; import org.apache.tika.extractor.DefaultEmbeddedStreamTranslator; import org.apache.tika.extractor.EmbeddedStreamTranslator; @@ -59,6 +57,9 @@ public class DigestHelper { * @param context parse context (should contain DigesterFactory, may contain SkipContainerDocumentDigest marker) * @throws IOException if an I/O error occurs */ + // Same per-object threshold as StreamCache when no budget is in the context. + private static final long DEFAULT_TRANSLATED_MEMORY_THRESHOLD = 1024 * 1024; + public static void maybeDigest(TikaInputStream tis, Metadata metadata, ParseContext context) throws IOException { @@ -84,16 +85,31 @@ public class DigestHelper { // The translator consumes `tis` (e.g. OLE2), so enableRewind() before and rewind() // after -- otherwise the caller would see an exhausted stream. if (EMBEDDED_STREAM_TRANSLATOR.shouldTranslate(tis, metadata)) { - tis.enableRewind(context.get(CacheMemoryBudget.class)); + CacheMemoryBudget budget = context.get(CacheMemoryBudget.class); + tis.enableRewind(budget); + // Translated size is unknown up front; the source length bounds it, so reserve + // that from the budget (or use the per-object default) as the in-memory threshold. + long threshold = DEFAULT_TRANSLATED_MEMORY_THRESHOLD; + long reserved = 0; + if (budget != null && tis.hasLength()) { + long len = tis.getLength(); + if (len > 0 && budget.tryReserve(len) > 0) { + reserved = len; + threshold = len; + } + } try (TemporaryResources tmp = new TemporaryResources()) { - Path tmpBytes = tmp.createTempFile(); - try (OutputStream os = Files.newOutputStream(tmpBytes)) { + TranslatedBytes translated = new TranslatedBytes(tmp, threshold); + try (OutputStream os = translated) { EMBEDDED_STREAM_TRANSLATOR.translate(tis, metadata, os); } - try (TikaInputStream translated = TikaInputStream.get(tmpBytes)) { - digester.digest(translated, metadata, context); + try (TikaInputStream translatedStream = translated.toTikaInputStream()) { + digester.digest(translatedStream, metadata, context); } } finally { + if (reserved > 0) { + budget.release(reserved); + } tis.rewind(); } } else { diff --git a/tika-core/src/main/java/org/apache/tika/digest/TranslatedBytes.java b/tika-core/src/main/java/org/apache/tika/digest/TranslatedBytes.java new file mode 100644 index 0000000000..2cf7ff4a63 --- /dev/null +++ b/tika-core/src/main/java/org/apache/tika/digest/TranslatedBytes.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.digest; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; + +import org.apache.commons.io.output.UnsynchronizedByteArrayOutputStream; + +import org.apache.tika.io.TemporaryResources; +import org.apache.tika.io.TikaInputStream; + +/** + * Sink for a translated embedded stream: bytes stay in memory up to {@code threshold} + * and spill to a temp file (owned by {@code tmp}) past it, so the common small object is + * digested without touching disk. + */ +class TranslatedBytes extends OutputStream { + + private final TemporaryResources tmp; + private final long threshold; + private UnsynchronizedByteArrayOutputStream memory = + UnsynchronizedByteArrayOutputStream.builder().get(); + private long size; + private Path spillFile; + private OutputStream spill; + + TranslatedBytes(TemporaryResources tmp, long threshold) { + this.tmp = tmp; + this.threshold = threshold; + } + + @Override + public void write(int b) throws IOException { + write(new byte[]{(byte) b}, 0, 1); + } + + @Override + public void write(byte[] b, int off, int len) throws IOException { + if (spill == null && size + len > threshold) { + spillFile = tmp.createTempFile(); + spill = Files.newOutputStream(spillFile); + memory.writeTo(spill); + memory = null; + } + if (spill != null) { + spill.write(b, off, len); + } else { + memory.write(b, off, len); + } + size += len; + } + + @Override + public void close() throws IOException { + if (spill != null) { + spill.close(); + } + } + + boolean isInMemory() { + return spill == null; + } + + /** The translated content; the caller closes it. */ + TikaInputStream toTikaInputStream() throws IOException { + return spill == null ? TikaInputStream.get(memory.toByteArray()) : TikaInputStream.get(spillFile); + } +} diff --git a/tika-core/src/test/java/org/apache/tika/digest/TranslatedBytesTest.java b/tika-core/src/test/java/org/apache/tika/digest/TranslatedBytesTest.java new file mode 100644 index 0000000000..82677164fc --- /dev/null +++ b/tika-core/src/test/java/org/apache/tika/digest/TranslatedBytesTest.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.digest; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.stream.Stream; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import org.apache.tika.io.TemporaryResources; +import org.apache.tika.io.TikaInputStream; + +public class TranslatedBytesTest { + + @TempDir + Path tempDir; + + @Test + public void testStaysInMemoryUnderThreshold() throws Exception { + byte[] data = new byte[1000]; + for (int i = 0; i < data.length; i++) { + data[i] = (byte) i; + } + try (TemporaryResources tmp = new TemporaryResources()) { + tmp.setTemporaryFileDirectory(tempDir); + TranslatedBytes sink = new TranslatedBytes(tmp, 1000); + sink.write(data, 0, 600); + sink.write(data, 600, 400); + sink.close(); + assertTrue(sink.isInMemory()); + try (TikaInputStream tis = sink.toTikaInputStream()) { + assertArrayEquals(data, tis.readAllBytes()); + } + try (Stream<Path> files = Files.list(tempDir)) { + assertEquals(0, files.count()); + } + } + } + + @Test + public void testSpillsPastThreshold() throws Exception { + byte[] data = new byte[5000]; + for (int i = 0; i < data.length; i++) { + data[i] = (byte) (i * 7); + } + try (TemporaryResources tmp = new TemporaryResources()) { + tmp.setTemporaryFileDirectory(tempDir); + TranslatedBytes sink = new TranslatedBytes(tmp, 1000); + sink.write(data, 0, 800); // in memory + sink.write(data, 800, 4200); // crosses the threshold: memory flushed to the file + sink.close(); + assertFalse(sink.isInMemory()); + try (Stream<Path> files = Files.list(tempDir)) { + assertEquals(1, files.count()); + } + try (TikaInputStream tis = sink.toTikaInputStream()) { + assertArrayEquals(data, tis.readAllBytes()); + } + } + // the temp file belongs to tmp and is gone once it closes + try (Stream<Path> files = Files.list(tempDir)) { + assertEquals(0, files.count()); + } + } +} diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/ImageMetadataExtractor.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/ImageMetadataExtractor.java index 3272f8d72f..6511fd7f3f 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/ImageMetadataExtractor.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/ImageMetadataExtractor.java @@ -145,6 +145,16 @@ public class ImageMetadataExtractor { } } + public void parseJpeg(InputStream stream) throws IOException, SAXException, TikaException { + try { + com.drew.metadata.Metadata jpegMetadata = + JpegMetadataReader.readMetadata(stream, JPEG_READERS_NO_XMP); + handle(jpegMetadata); + } catch (JpegProcessingException | MetadataException e) { + throw new TikaException("Can't read JPEG metadata", e); + } + } + public void parseTiff(File file) throws IOException, SAXException, TikaException { try { com.drew.metadata.Metadata tiffMetadata = TiffMetadataReader.readMetadata(file); @@ -154,14 +164,26 @@ public class ImageMetadataExtractor { } } + public void parseTiff(InputStream stream) throws IOException, SAXException, TikaException { + try { + com.drew.metadata.Metadata tiffMetadata = TiffMetadataReader.readMetadata(stream); + handle(tiffMetadata); + } catch (MetadataException | TiffProcessingException e) { + throw new TikaException("Can't read TIFF metadata", e); + } + } + public void parseWebP(File file) throws IOException, TikaException { + try { + handle(WebpMetadataReader.readMetadata(file)); + } catch (RiffProcessingException | MetadataException e) { + throw new TikaException("Can't process Riff data", e); + } + } + public void parseWebP(InputStream stream) throws IOException, TikaException { try { - com.drew.metadata.Metadata webPMetadata = new com.drew.metadata.Metadata(); - webPMetadata = WebpMetadataReader.readMetadata(file); - handle(webPMetadata); - } catch (IOException e) { - throw e; + handle(WebpMetadataReader.readMetadata(stream)); } catch (RiffProcessingException | MetadataException e) { throw new TikaException("Can't process Riff data", e); } diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/ImageXmp.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/ImageXmp.java index af88c13659..179df5c708 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/ImageXmp.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/ImageXmp.java @@ -17,8 +17,6 @@ package org.apache.tika.parser.image; import java.io.BufferedInputStream; -import java.io.File; -import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; @@ -28,6 +26,7 @@ import com.drew.imaging.jpeg.JpegProcessingException; import com.drew.imaging.jpeg.JpegSegmentData; import com.drew.imaging.jpeg.JpegSegmentReader; import com.drew.imaging.jpeg.JpegSegmentType; +import com.drew.lang.StreamReader; import org.apache.commons.io.IOUtils; import org.apache.commons.io.output.UnsynchronizedByteArrayOutputStream; import org.xml.sax.SAXException; @@ -82,11 +81,11 @@ final class ImageXmp { } /** JPEG: read APP1 segments, reassemble Extended XMP, parse each resulting packet. */ - static void extractJpeg(File file, Metadata metadata, ParseContext context) { + static void extractJpeg(InputStream stream, Metadata metadata, ParseContext context) { try { Iterable<byte[]> app1; try { - JpegSegmentData data = JpegSegmentReader.readSegments(file, + JpegSegmentData data = JpegSegmentReader.readSegments(new StreamReader(stream), Collections.singletonList(JpegSegmentType.APP1)); app1 = data.getSegments(JpegSegmentType.APP1); } catch (JpegProcessingException e) { @@ -106,9 +105,9 @@ final class ImageXmp { } /** WebP: pull the raw packet out of the RIFF {@code "XMP "} chunk and parse it. */ - static void extractWebp(File file, Metadata metadata, ParseContext context) { + static void extractWebp(InputStream stream, Metadata metadata, ParseContext context) { try { - byte[] xmp = readRiffChunk(file, "XMP "); + byte[] xmp = readRiffChunk(stream, "XMP "); if (xmp != null) { new XmpExtractor().extract(xmp, metadata, context); } @@ -123,8 +122,9 @@ final class ImageXmp { private static final long MAX_CHUNK = 64L * 1024 * 1024; /** Return the payload of the first top-level RIFF chunk with the given FourCC, or null. */ - private static byte[] readRiffChunk(File file, String fourCC) throws IOException { - try (InputStream in = new BufferedInputStream(new FileInputStream(file))) { + private static byte[] readRiffChunk(InputStream stream, String fourCC) throws IOException { + InputStream in = stream instanceof BufferedInputStream ? stream : new BufferedInputStream(stream); + { byte[] head = new byte[12]; if (IOUtils.read(in, head, 0, 12) < 12 || head[0] != 'R' || head[1] != 'I' || head[2] != 'F' || head[3] != 'F' || head[8] != 'W' || head[9] != 'E' || diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/JpegParser.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/JpegParser.java index a96311857f..22a7b0e082 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/JpegParser.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/JpegParser.java @@ -26,6 +26,7 @@ import org.xml.sax.SAXException; import org.apache.tika.annotation.TikaComponent; import org.apache.tika.exception.TikaException; +import org.apache.tika.io.CacheMemoryBudget; import org.apache.tika.io.TemporaryResources; import org.apache.tika.io.TikaInputStream; import org.apache.tika.metadata.Metadata; @@ -54,9 +55,13 @@ public class JpegParser extends AbstractImageParser { TemporaryResources tmp = new TemporaryResources(); try { TikaInputStream tis = TikaInputStream.get(stream, tmp, metadata); + // Two sequential passes over the bytes; rewind instead of spooling to a file so + // in-memory embedded images never touch disk. + tis.enableRewind(parseContext.get(CacheMemoryBudget.class)); // XMP first so it is canonical; the metadata-extractor handlers (IPTC/EXIF) fill gaps. - ImageXmp.extractJpeg(tis.getFile(), metadata, parseContext); - new ImageMetadataExtractor(metadata).parseJpeg(tis.getFile()); + ImageXmp.extractJpeg(tis, metadata, parseContext); + tis.rewind(); + new ImageMetadataExtractor(metadata).parseJpeg(tis); } finally { tmp.dispose(); } diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/TiffParser.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/TiffParser.java index 65664bff10..a1c194954c 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/TiffParser.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/TiffParser.java @@ -26,6 +26,7 @@ import org.xml.sax.SAXException; import org.apache.tika.annotation.TikaComponent; import org.apache.tika.exception.TikaException; +import org.apache.tika.io.CacheMemoryBudget; import org.apache.tika.io.TemporaryResources; import org.apache.tika.io.TikaInputStream; import org.apache.tika.metadata.Metadata; @@ -54,10 +55,18 @@ public class TiffParser extends AbstractImageParser { TemporaryResources tmp = new TemporaryResources(); try { TikaInputStream tis = TikaInputStream.get(stream, tmp, metadata); - tis.getFile(); // spool so tis is fully re-readable below + tis.enableRewind(parseContext.get(CacheMemoryBudget.class)); // XMP first so it is canonical; metadata-extractor (IPTC/EXIF) fills gaps. ImageXmp.scanAndExtract(tis, metadata, parseContext); - new ImageMetadataExtractor(metadata).parseTiff(tis.getFile()); + tis.rewind(); + ImageMetadataExtractor extractor = new ImageMetadataExtractor(metadata); + // File-backed input keeps drewnoakes' random-access file reader; in-memory input + // is read from the stream rather than spooled to disk. + if (tis.hasFile()) { + extractor.parseTiff(tis.getFile()); + } else { + extractor.parseTiff(tis); + } } finally { tmp.dispose(); } diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/WebPParser.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/WebPParser.java index 1af3b5e5a3..a426d1c492 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/WebPParser.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/WebPParser.java @@ -25,6 +25,7 @@ import org.xml.sax.SAXException; import org.apache.tika.annotation.TikaComponent; import org.apache.tika.exception.TikaException; +import org.apache.tika.io.CacheMemoryBudget; import org.apache.tika.io.TikaInputStream; import org.apache.tika.metadata.Metadata; import org.apache.tika.mime.MediaType; @@ -50,8 +51,11 @@ public class WebPParser implements Parser { public void parse(TikaInputStream tis, ContentHandler handler, Metadata metadata, ParseContext context) throws IOException, SAXException, TikaException { // XMP first (canonical), then EXIF/etc. from metadata-extractor as fallback. - ImageXmp.extractWebp(tis.getFile(), metadata, context); - new ImageMetadataExtractor(metadata).parseWebP(tis.getFile()); + // Rewind between the two passes instead of spooling to a file. + tis.enableRewind(context.get(CacheMemoryBudget.class)); + ImageXmp.extractWebp(tis, metadata, context); + tis.rewind(); + new ImageMetadataExtractor(metadata).parseWebP(tis); XHTMLContentHandler xhtml = new XHTMLContentHandler(handler, metadata, context); xhtml.startDocument(); diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/test/java/org/apache/tika/parser/image/ImageParsersNoTempFileTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/test/java/org/apache/tika/parser/image/ImageParsersNoTempFileTest.java new file mode 100644 index 0000000000..031e9a9b1b --- /dev/null +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/test/java/org/apache/tika/parser/image/ImageParsersNoTempFileTest.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.parser.image; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.stream.Stream; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.xml.sax.helpers.DefaultHandler; + +import org.apache.tika.TikaTest; +import org.apache.tika.io.CacheMemoryBudget; +import org.apache.tika.io.TemporaryResources; +import org.apache.tika.io.TikaInputStream; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.metadata.TIFF; +import org.apache.tika.parser.ParseContext; +import org.apache.tika.parser.Parser; + +/** + * Embedded images usually arrive already in memory; the image parsers must not spool + * them to disk just to read metadata (they used to call getFile() for every pass). + */ +public class ImageParsersNoTempFileTest extends TikaTest { + + @TempDir + Path tempDir; + + @Test + public void testJpeg() throws Exception { + assertNoTempFile(new JpegParser(), "/test-documents/testJPEG_EXIF.jpg", TIFF.IMAGE_WIDTH.getName()); + } + + @Test + public void testTiff() throws Exception { + assertNoTempFile(new TiffParser(), "/test-documents/testTIFF.tif", TIFF.IMAGE_WIDTH.getName()); + } + + @Test + public void testWebP() throws Exception { + assertNoTempFile(new WebPParser(), "/test-documents/testWebp_Alpha_Lossless.webp", + ImageMetadataExtractor.UNKNOWN_IMG_NS + "Image Width"); + } + + private void assertNoTempFile(Parser parser, String resource, String widthKey) throws Exception { + byte[] bytes; + try (InputStream is = getResourceAsStream(resource)) { + bytes = is.readAllBytes(); + } + ParseContext context = new ParseContext(); + context.set(CacheMemoryBudget.class, new CacheMemoryBudget(64L * 1024 * 1024)); + Metadata metadata = new Metadata(); + try (TemporaryResources tmp = new TemporaryResources()) { + tmp.setTemporaryFileDirectory(tempDir); + // a stream-backed, non-file TikaInputStream whose only spill target is tempDir + TikaInputStream tis = TikaInputStream.get(new ByteArrayInputStream(bytes), tmp, metadata); + parser.parse(tis, new DefaultHandler(), metadata, context); + // temp files live until tmp closes, so any spool would be visible right here + try (Stream<Path> files = Files.list(tempDir)) { + assertEquals(0, files.count(), "parser spooled an in-memory image to disk"); + } + } + assertNotNull(metadata.get(widthKey), "metadata was extracted"); + } +} diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/detect/microsoft/POIFSContainerDetector.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/detect/microsoft/POIFSContainerDetector.java index 356dcba2f1..007289db6b 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/detect/microsoft/POIFSContainerDetector.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/detect/microsoft/POIFSContainerDetector.java @@ -22,6 +22,8 @@ import static org.apache.tika.mime.MediaType.image; import java.io.IOException; import java.io.InputStream; +import java.nio.channels.Channels; +import java.nio.channels.SeekableByteChannel; import java.nio.charset.StandardCharsets; import java.nio.file.Path; import java.util.Collections; @@ -44,6 +46,7 @@ import org.slf4j.LoggerFactory; import org.apache.tika.annotation.TikaComponent; import org.apache.tika.detect.Detector; +import org.apache.tika.io.CacheMemoryBudget; import org.apache.tika.io.TikaInputStream; import org.apache.tika.metadata.Metadata; import org.apache.tika.mime.MediaType; @@ -554,7 +557,18 @@ public class POIFSContainerDetector implements Detector { } - private Set<String> getTopLevelNames(TikaInputStream stream) throws IOException { + // In-memory POIFS copies the whole object into heap; above this, spool as before. + private static final long MAX_IN_MEMORY_POIFS = 64L * 1024 * 1024; + private static final long DEFAULT_IN_MEMORY_POIFS = 1024 * 1024; + + private Set<String> getTopLevelNames(TikaInputStream stream, ParseContext context) + throws IOException { + if (!stream.hasFile()) { + Set<String> names = getTopLevelNamesInMemory(stream, context); + if (names != null) { + return names; + } + } // Force the document stream to a (possibly temporary) file // so we don't modify the current position of the stream. Path file = stream.getPath(); @@ -583,16 +597,51 @@ public class POIFSContainerDetector implements Detector { } } + /** + * Opens the OLE2 container from memory instead of spooling it to a temp file. Returns + * null when the object is too large for the in-memory limit or POI cannot load it from a + * stream (the caller then spools, as before). + */ + private Set<String> getTopLevelNamesInMemory(TikaInputStream stream, ParseContext context) + throws IOException { + CacheMemoryBudget budget = context == null ? null : context.get(CacheMemoryBudget.class); + long limit = budget == null ? DEFAULT_IN_MEMORY_POIFS : + Math.min(MAX_IN_MEMORY_POIFS, budget.getMaxBytes()); + // the channel is served from memory while the content fits the cache/budget + try (SeekableByteChannel channel = stream.getSeekableByteChannel()) { + long size = channel.size(); + if (size > limit) { + return null; + } + if (budget != null) { + if (budget.tryReserve(size) == 0) { + return null; + } + stream.addCloseableResource(() -> budget.release(size)); + } + POIFSFileSystem fs = new POIFSFileSystem(Channels.newInputStream(channel)); + stream.setOpenContainer(fs); + return getTopLevelNames(fs.getRoot()); + } catch (SecurityException e) { + throw e; + } catch (IOException | RuntimeException e) { + // POI's stream loader is stricter than its file loader (e.g. truncated objects); + // fall back to the file path so behavior is unchanged for anything it rejects + return null; + } + } + public MediaType detect(TikaInputStream tis, Metadata metadata, ParseContext parseContext) throws IOException { // Check if we have access to the document if (tis == null) { return MediaType.OCTET_STREAM; } - return handleTikaStream(tis, metadata); + return handleTikaStream(tis, metadata, parseContext); } - private MediaType handleTikaStream(TikaInputStream tis, Metadata metadata) throws IOException { + private MediaType handleTikaStream(TikaInputStream tis, Metadata metadata, ParseContext context) + throws IOException { //try for an open container Set<String> names = tryOpenContainerOnTikaInputStream(tis, metadata); @@ -604,7 +653,7 @@ public class POIFSContainerDetector implements Detector { // If OLE, spool to disk if (names == null) { // spool to disk and try detection - names = getTopLevelNames(tis); + names = getTopLevelNames(tis, context); } // Detect based on the names (as available) diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/detect/microsoft/POIFSContainerDetectorNoTempFileTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/detect/microsoft/POIFSContainerDetectorNoTempFileTest.java new file mode 100644 index 0000000000..b00f2e57e7 --- /dev/null +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/detect/microsoft/POIFSContainerDetectorNoTempFileTest.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.detect.microsoft; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.stream.Stream; + +import org.apache.poi.poifs.filesystem.POIFSFileSystem; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import org.apache.tika.TikaTest; +import org.apache.tika.io.CacheMemoryBudget; +import org.apache.tika.io.TemporaryResources; +import org.apache.tika.io.TikaInputStream; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.mime.MediaType; +import org.apache.tika.parser.ParseContext; + +/** + * Detection used to spool every in-memory OLE2 object to a temp file to read its + * top-level entry names; it must now open the container from memory. + */ +public class POIFSContainerDetectorNoTempFileTest extends TikaTest { + + @TempDir + Path tempDir; + + @Test + public void testNoTempFileForInMemoryInput() throws Exception { + byte[] bytes; + try (InputStream is = getResourceAsStream("/test-documents/testWORD.doc")) { + bytes = is.readAllBytes(); + } + ParseContext context = new ParseContext(); + context.set(CacheMemoryBudget.class, new CacheMemoryBudget(64L * 1024 * 1024)); + Metadata metadata = new Metadata(); + try (TemporaryResources tmp = new TemporaryResources()) { + tmp.setTemporaryFileDirectory(tempDir); + TikaInputStream tis = TikaInputStream.get(new ByteArrayInputStream(bytes), tmp, metadata); + MediaType type = new POIFSContainerDetector().detect(tis, metadata, context); + assertEquals(MediaType.application("msword"), type); + try (Stream<Path> files = Files.list(tempDir)) { + assertEquals(0, files.count(), "detector spooled an in-memory OLE2 object to disk"); + } + assertTrue(tis.getOpenContainer() instanceof POIFSFileSystem, "open container kept for the parser"); + assertEquals(0, tis.getPosition(), "detection must not move the stream"); + assertEquals(0xd0, tis.read(), "stream still readable from the start"); + } + } +} diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/OpenDocumentParser.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/OpenDocumentParser.java index c500689ab4..f3db7c4af6 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/OpenDocumentParser.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/OpenDocumentParser.java @@ -45,6 +45,7 @@ import org.apache.tika.exception.EncryptedDocumentException; import org.apache.tika.exception.TikaException; import org.apache.tika.extractor.EmbeddedDocumentExtractor; import org.apache.tika.extractor.EmbeddedDocumentUtil; +import org.apache.tika.io.CacheMemoryBudget; import org.apache.tika.io.TikaInputStream; import org.apache.tika.metadata.HttpHeaders; import org.apache.tika.metadata.Metadata; @@ -320,14 +321,14 @@ public class OpenDocumentParser implements Parser { if (embeddedName.contains("Pictures/")) { embeddedMetadata.set(TikaCoreProperties.EMBEDDED_RESOURCE_TYPE, TikaCoreProperties.EmbeddedResourceType.INLINE.toString()); - //spool - tisZip.getFile(); + // rewind (not spool) so detection leaves the entry re-readable in memory + tisZip.enableRewind(context.get(CacheMemoryBudget.class)); MediaType embeddedMimeType = EmbeddedDocumentUtil.getDetector(context) .detect(tisZip, embeddedMetadata, context); if (embeddedMimeType != null) { embeddedMetadata.set(HttpHeaders.CONTENT_TYPE, embeddedMimeType.toString()); } - tisZip.reset(); + tisZip.rewind(); // Tag the picture with the draw:page indices it // appears on (set populated by scanPicturePages). // A null lookup means "not referenced by any diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/test/java/org/apache/tika/parser/odf/OpenDocumentParserNoTempFileTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/test/java/org/apache/tika/parser/odf/OpenDocumentParserNoTempFileTest.java new file mode 100644 index 0000000000..78c56e5e9e --- /dev/null +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/test/java/org/apache/tika/parser/odf/OpenDocumentParserNoTempFileTest.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.parser.odf; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.stream.Stream; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import org.apache.tika.TikaTest; +import org.apache.tika.io.CacheMemoryBudget; +import org.apache.tika.io.TemporaryResources; +import org.apache.tika.io.TikaInputStream; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.parser.ParseContext; +import org.apache.tika.sax.BodyContentHandler; + +/** + * Inline pictures used to be spooled to a temp file before detection; with rewind + * support the whole document parses from memory. + */ +public class OpenDocumentParserNoTempFileTest extends TikaTest { + + @TempDir + Path tempDir; + + @Test + public void testNoTempFileForInMemoryInput() throws Exception { + byte[] bytes; + try (InputStream is = getResourceAsStream("/test-documents/testODTEmbedded.odt")) { + bytes = is.readAllBytes(); + } + ParseContext context = new ParseContext(); + context.set(CacheMemoryBudget.class, new CacheMemoryBudget(64L * 1024 * 1024)); + Metadata metadata = new Metadata(); + BodyContentHandler handler = new BodyContentHandler(); + try (TemporaryResources tmp = new TemporaryResources()) { + tmp.setTemporaryFileDirectory(tempDir); + TikaInputStream tis = TikaInputStream.get(new ByteArrayInputStream(bytes), tmp, metadata); + new OpenDocumentParser().parse(tis, handler, metadata, context); + try (Stream<Path> files = Files.list(tempDir)) { + assertEquals(0, files.count(), "ODF parse spooled an in-memory entry to disk"); + } + } + assertTrue(handler.toString().length() > 0, "content was extracted"); + } +} diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFParser.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFParser.java index c61f33cc45..d9451b0f5f 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFParser.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFParser.java @@ -20,6 +20,8 @@ import static org.apache.tika.metadata.PDF.OCR_PAGE_COUNT; import java.io.IOException; import java.io.InputStream; +import java.nio.channels.Channels; +import java.nio.channels.SeekableByteChannel; import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; @@ -315,8 +317,13 @@ public class PDFParser implements Parser, RenderingParser { List<StartXRefOffset> xRefOffsets = new ArrayList<>(); //TODO -- can we use the PDFBox parser's RandomAccessRead //so that we don't have to reopen from file? - try (RandomAccessRead ra = - new RandomAccessReadBufferedFile(tikaInputStream.getFile())) { + // In-memory input is scanned from memory (as the main parse already does) rather + // than spooled to a file just for this pass. + try (SeekableByteChannel channel = tikaInputStream.hasFile() ? null : + tikaInputStream.getSeekableByteChannel(); + RandomAccessRead ra = channel == null ? + new RandomAccessReadBufferedFile(tikaInputStream.getFile()) : + new RandomAccessReadBuffer(Channels.newInputStream(channel))) { StartXRefScanner xRefScanner = new StartXRefScanner(ra); xRefOffsets.addAll(xRefScanner.scan()); } catch (IOException e) { diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/test/java/org/apache/tika/parser/pdf/PDFParserNoTempFileTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/test/java/org/apache/tika/parser/pdf/PDFParserNoTempFileTest.java new file mode 100644 index 0000000000..9529fee4d4 --- /dev/null +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/test/java/org/apache/tika/parser/pdf/PDFParserNoTempFileTest.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.parser.pdf; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.stream.Stream; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.xml.sax.helpers.DefaultHandler; + +import org.apache.tika.TikaTest; +import org.apache.tika.io.CacheMemoryBudget; +import org.apache.tika.io.TemporaryResources; +import org.apache.tika.io.TikaInputStream; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.metadata.PDF; +import org.apache.tika.parser.ParseContext; + +/** + * The incremental-update xref scan used to spool in-memory PDFs to a file; it must scan + * from memory like the main parse does. + */ +public class PDFParserNoTempFileTest extends TikaTest { + + @TempDir + Path tempDir; + + @Test + public void testXRefScanNoTempFileForInMemoryInput() throws Exception { + byte[] bytes; + try (InputStream is = getResourceAsStream("/test-documents/testPDF_incrementalUpdates.pdf")) { + bytes = is.readAllBytes(); + } + PDFParserConfig config = new PDFParserConfig(); + config.setExtractIncrementalUpdateInfo(true); + ParseContext context = new ParseContext(); + context.set(PDFParserConfig.class, config); + context.set(CacheMemoryBudget.class, new CacheMemoryBudget(64L * 1024 * 1024)); + Metadata metadata = new Metadata(); + try (TemporaryResources tmp = new TemporaryResources()) { + tmp.setTemporaryFileDirectory(tempDir); + TikaInputStream tis = TikaInputStream.get(new ByteArrayInputStream(bytes), tmp, metadata); + new PDFParser().parse(tis, new DefaultHandler(), metadata, context); + try (Stream<Path> files = Files.list(tempDir)) { + assertEquals(0, files.count(), "xref scan spooled an in-memory PDF to disk"); + } + } + assertNotNull(metadata.get(PDF.PDF_INCREMENTAL_UPDATE_COUNT), "incremental update info was extracted"); + } +}
