This is an automated email from the ASF dual-hosted git repository. tballison pushed a commit to branch TIKA-4835-spill-sites in repository https://gitbox.apache.org/repos/asf/tika.git
commit 58e33ebfa3448febb676fcacde54858f85a9eb92 Author: tallison <[email protected]> AuthorDate: Wed Aug 26 10:06:22 2026 -0400 TIKA-4835 -- read in-memory input in place instead of spooling it --- CHANGES.txt | 22 ++++ .../tika/parser/microsoft/rtf/RTFParserTest.java | 4 +- .../apache/tika/parser/image/ByteBufferReader.java | 74 +++++++++++ .../tika/parser/image/ImageMetadataExtractor.java | 80 ++++++++++-- .../org/apache/tika/parser/image/ImageXmp.java | 55 ++++---- .../org/apache/tika/parser/image/JpegParser.java | 8 +- .../org/apache/tika/parser/image/TiffParser.java | 6 +- .../org/apache/tika/parser/image/WebPParser.java | 7 +- .../parser/image/ImageParsersNoTempFileTest.java | 138 +++++++++++++++++++++ .../detect/microsoft/POIFSContainerDetector.java | 108 ++++++++++++++-- .../POIFSContainerDetectorNoTempFileTest.java | 71 +++++++++++ .../detect/microsoft/POIFSDeclaredSizeTest.java | 115 +++++++++++++++++ .../apache/tika/parser/odf/OpenDocumentParser.java | 6 +- .../odf/OpenDocumentParserNoTempFileTest.java | 104 ++++++++++++++++ .../java/org/apache/tika/parser/pdf/PDFParser.java | 43 ++++--- .../apache/tika/parser/pdf/PDFRandomAccess.java | 66 ++++++++++ .../tika/renderer/pdf/pdfbox/PDFBoxRenderer.java | 6 +- .../tika/parser/pdf/PDFParserNoTempFileTest.java | 71 +++++++++++ .../tika/parser/pdf/PDFPerPageRenderTest.java | 95 ++++++++++++++ 19 files changed, 1007 insertions(+), 72 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 402927e239..2c0ebe5208 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,5 +1,27 @@ Release 4.1.0 - unreleased + * Parsers and detectors no longer spool in-memory input to a temp file just + to read it back with random access: the JPEG, TIFF and WebP parsers, the + OpenDocument parser's inline pictures, the OLE2 container detector's + entry-name read, and PDFParser (the incremental-update scan and the main + load). Together these were the bulk of the temp bytes behind 4.0.0's + batch slowdown on spinning disks (see docs/.../pipes/performance.adoc). + One rule everywhere: content goes through the stream cache (bounded by + the pipes cache memory budget, 1MB per object without one); what stays + in memory is read in place through a zero-copy view, what spills is + read from the file. PDFBox and metadata-extractor no longer receive a + second heap copy of the document. The PDF renderer now re-opens the + document from byte 0 for every render, which fixes per-page rendering + (RENDER_PAGES_AT_PAGE_END) silently producing only the first page -- + for file-backed input too. PDFParser.getPDDocumentFromStream is + deprecated and no longer called; override the new + getPDDocument(RandomAccessRead, ...) instead. ImageMetadataExtractor + gains TikaInputStream and InputStream overloads. One visible metadata + change: JPEG/TIFF/WebP images read from memory no longer carry + metadata-extractor's file-system tags (img:File Name, img:File Size, + img:File Modified Date), which described whatever file was read rather + than the image; input backed by a file still carries them (TIKA-4835). + * tika-pipes: the cache memory budget (how much rewindable content a forked worker keeps in memory before spilling to disk) now defaults to a quarter of the fork's heap, so raising -Xmx raises it; it was a fixed 256MB. It is diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-integration-tests/src/test/java/org/apache/tika/parser/microsoft/rtf/RTFParserTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-integration-tests/src/test/java/org/apache/tika/parser/microsoft/rtf/RTFParserTest.java index 96ba2b3c7b..9b4fe1b5ea 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-integration-tests/src/test/java/org/apache/tika/parser/microsoft/rtf/RTFParserTest.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-integration-tests/src/test/java/org/apache/tika/parser/microsoft/rtf/RTFParserTest.java @@ -145,7 +145,9 @@ public class RTFParserTest extends TikaTest { //need flexibility for if tesseract is installed or not //TODO -- fix this test. It is too fragile. - assertTrue(meta_jpg.names().length >= 52 && meta_jpg.names().length <= 60); + // in-memory embedded images no longer carry metadata-extractor's temp-file + // name/size/date tags (TIKA-4835), hence the lower bound + assertTrue(meta_jpg.names().length >= 49 && meta_jpg.names().length <= 60); assertTrue(meta_jpg_exif.names().length >= 100 && meta_jpg_exif.names().length <= 130); } diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/ByteBufferReader.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/ByteBufferReader.java new file mode 100644 index 0000000000..43da7a2840 --- /dev/null +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/ByteBufferReader.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.parser.image; + +import java.nio.ByteBuffer; + +import com.drew.lang.BufferBoundsException; +import com.drew.lang.RandomAccessReader; + +/** + * drewnoakes random-access reader over a {@link ByteBuffer}, for content already in memory. + * The library's own stream reader retains every chunk it reads and its TIFF reader asks for + * the length up front, which reads the whole stream; this reads the buffer in place. + */ +final class ByteBufferReader extends RandomAccessReader { + + private final ByteBuffer buffer; + + /** Reads {@code buffer} from position 0 to its limit; the buffer's position is not used. */ + ByteBufferReader(ByteBuffer buffer) { + this.buffer = buffer; + } + + @Override + public int toUnshiftedOffset(int localOffset) { + return localOffset; + } + + @Override + public long getLength() { + return buffer.limit(); + } + + @Override + public byte getByte(int index) throws java.io.IOException { + validateIndex(index, 1); + return buffer.get(index); + } + + @Override + public byte[] getBytes(int index, int count) throws java.io.IOException { + validateIndex(index, count); + byte[] bytes = new byte[count]; + buffer.get(index, bytes); + return bytes; + } + + @Override + protected boolean isValidIndex(int index, int bytesRequested) { + return bytesRequested >= 0 && index >= 0 && + (long) index + bytesRequested - 1L < buffer.limit(); + } + + @Override + protected void validateIndex(int index, int bytesRequested) throws java.io.IOException { + if (!isValidIndex(index, bytesRequested)) { + throw new BufferBoundsException(index, bytesRequested, buffer.limit()); + } + } +} diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/ImageMetadataExtractor.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/ImageMetadataExtractor.java index 3272f8d72f..41f2b4bca3 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/ImageMetadataExtractor.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/ImageMetadataExtractor.java @@ -19,6 +19,8 @@ package org.apache.tika.parser.image; import java.io.File; import java.io.IOException; import java.io.InputStream; +import java.nio.ByteBuffer; +import java.nio.channels.SeekableByteChannel; import java.text.DecimalFormat; import java.text.DecimalFormatSymbols; import java.text.SimpleDateFormat; @@ -41,6 +43,7 @@ import com.drew.imaging.tiff.TiffProcessingException; import com.drew.imaging.webp.WebpMetadataReader; import com.drew.lang.ByteArrayReader; import com.drew.lang.GeoLocation; +import com.drew.lang.RandomAccessReader; import com.drew.lang.Rational; import com.drew.metadata.Directory; import com.drew.metadata.MetadataException; @@ -70,6 +73,7 @@ import org.apache.commons.io.IOUtils; import org.xml.sax.SAXException; import org.apache.tika.exception.TikaException; +import org.apache.tika.io.TikaInputStream; import org.apache.tika.metadata.Geographic; import org.apache.tika.metadata.IPTC; import org.apache.tika.metadata.KeyPrefix; @@ -135,6 +139,50 @@ public class ImageMetadataExtractor { new ExifReader(), new IccReader(), new PhotoshopReader(), new DuckyReader(), new IptcReader(), new AdobeJpegReader(), new JpegDhtReader(), new JpegDnlReader()); + /** + * Reads from the file when one already exists (that path also yields metadata-extractor's + * file-system tags: img:File Name/Size/Modified Date, which describe the file, not the + * image), and from the stream otherwise. JPEG is read sequentially, so the stream costs + * no extra memory. + */ + public void parseJpeg(TikaInputStream tis) throws IOException, SAXException, TikaException { + if (tis.hasFile()) { + parseJpeg(tis.getFile()); + } else { + parseJpeg((InputStream) tis); + } + } + + /** + * TIFF needs random access. In-memory content is read in place through a zero-copy view; + * anything else is read from the file. There is deliberately no InputStream overload: + * metadata-extractor's stream reader retains everything it reads. + */ + public void parseTiff(TikaInputStream tis) throws IOException, SAXException, TikaException { + if (tis.hasFile()) { + parseTiff(tis.getFile()); + return; + } + try (SeekableByteChannel channel = tis.getSeekableByteChannel()) { + ByteBuffer view = TikaInputStream.inMemoryContent(channel); + if (view != null) { + parseTiff(new ByteBufferReader(view)); + return; + } + } + // the drain spilled: the content is on disk now + parseTiff(tis.getFile()); + } + + /** See {@link #parseJpeg(TikaInputStream)}; WebP is read sequentially too. */ + public void parseWebP(TikaInputStream tis) throws IOException, TikaException { + if (tis.hasFile()) { + parseWebP(tis.getFile()); + } else { + parseWebP((InputStream) tis); + } + } + public void parseJpeg(File file) throws IOException, SAXException, TikaException { try { com.drew.metadata.Metadata jpegMetadata = @@ -145,23 +193,41 @@ public class ImageMetadataExtractor { } } + public void parseJpeg(InputStream stream) throws IOException, SAXException, TikaException { + try { + handle(JpegMetadataReader.readMetadata(stream, JPEG_READERS_NO_XMP)); + } catch (JpegProcessingException | MetadataException e) { + throw new TikaException("Can't read JPEG metadata", e); + } + } + public void parseTiff(File file) throws IOException, SAXException, TikaException { try { - com.drew.metadata.Metadata tiffMetadata = TiffMetadataReader.readMetadata(file); - handle(tiffMetadata); + handle(TiffMetadataReader.readMetadata(file)); + } catch (MetadataException | TiffProcessingException e) { + throw new TikaException("Can't read TIFF metadata", e); + } + } + + private void parseTiff(RandomAccessReader reader) throws IOException, SAXException, TikaException { + try { + handle(TiffMetadataReader.readMetadata(reader)); } catch (MetadataException | TiffProcessingException e) { throw new TikaException("Can't read TIFF metadata", e); } } public void parseWebP(File file) throws IOException, TikaException { + try { + handle(WebpMetadataReader.readMetadata(file)); + } catch (RiffProcessingException | MetadataException e) { + throw new TikaException("Can't process Riff data", e); + } + } + public void parseWebP(InputStream stream) throws IOException, TikaException { try { - com.drew.metadata.Metadata webPMetadata = new com.drew.metadata.Metadata(); - webPMetadata = WebpMetadataReader.readMetadata(file); - handle(webPMetadata); - } catch (IOException e) { - throw e; + handle(WebpMetadataReader.readMetadata(stream)); } catch (RiffProcessingException | MetadataException e) { throw new TikaException("Can't process Riff data", e); } diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/ImageXmp.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/ImageXmp.java index af88c13659..8f540148de 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/ImageXmp.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/ImageXmp.java @@ -17,8 +17,6 @@ package org.apache.tika.parser.image; import java.io.BufferedInputStream; -import java.io.File; -import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; @@ -28,6 +26,7 @@ import com.drew.imaging.jpeg.JpegProcessingException; import com.drew.imaging.jpeg.JpegSegmentData; import com.drew.imaging.jpeg.JpegSegmentReader; import com.drew.imaging.jpeg.JpegSegmentType; +import com.drew.lang.StreamReader; import org.apache.commons.io.IOUtils; import org.apache.commons.io.output.UnsynchronizedByteArrayOutputStream; import org.xml.sax.SAXException; @@ -81,12 +80,12 @@ final class ImageXmp { } } - /** JPEG: read APP1 segments, reassemble Extended XMP, parse each resulting packet. */ - static void extractJpeg(File file, Metadata metadata, ParseContext context) { + /** JPEG: read APP1 segments, reassemble Extended XMP, parse each resulting packet. Consumes the stream. */ + static void extractJpeg(InputStream stream, Metadata metadata, ParseContext context) { try { Iterable<byte[]> app1; try { - JpegSegmentData data = JpegSegmentReader.readSegments(file, + JpegSegmentData data = JpegSegmentReader.readSegments(new StreamReader(stream), Collections.singletonList(JpegSegmentType.APP1)); app1 = data.getSegments(JpegSegmentType.APP1); } catch (JpegProcessingException e) { @@ -105,10 +104,10 @@ final class ImageXmp { } } - /** WebP: pull the raw packet out of the RIFF {@code "XMP "} chunk and parse it. */ - static void extractWebp(File file, Metadata metadata, ParseContext context) { + /** WebP: pull the raw packet out of the RIFF {@code "XMP "} chunk and parse it. Consumes the stream. */ + static void extractWebp(InputStream stream, Metadata metadata, ParseContext context) { try { - byte[] xmp = readRiffChunk(file, "XMP "); + byte[] xmp = readRiffChunk(stream, "XMP "); if (xmp != null) { new XmpExtractor().extract(xmp, metadata, context); } @@ -123,28 +122,28 @@ final class ImageXmp { private static final long MAX_CHUNK = 64L * 1024 * 1024; /** Return the payload of the first top-level RIFF chunk with the given FourCC, or null. */ - private static byte[] readRiffChunk(File file, String fourCC) throws IOException { - try (InputStream in = new BufferedInputStream(new FileInputStream(file))) { - byte[] head = new byte[12]; - if (IOUtils.read(in, head, 0, 12) < 12 || head[0] != 'R' || head[1] != 'I' || - head[2] != 'F' || head[3] != 'F' || head[8] != 'W' || head[9] != 'E' || - head[10] != 'B' || head[11] != 'P') { - return null; - } - byte[] ch = new byte[8]; - while (IOUtils.read(in, ch, 0, 8) == 8) { - long size = (ch[4] & 0xffL) | (ch[5] & 0xffL) << 8 | - (ch[6] & 0xffL) << 16 | (ch[7] & 0xffL) << 24; - if (fourCC.equals(new String(ch, 0, 4, StandardCharsets.US_ASCII))) { - if (size > MAX_CHUNK) { - return null; // target chunk too large to allocate - } - byte[] data = new byte[(int) size]; - return IOUtils.read(in, data, 0, data.length) == data.length ? data : null; + private static byte[] readRiffChunk(InputStream stream, String fourCC) throws IOException { + // not closed: the stream is the caller's, who rewinds it for the next pass + InputStream in = new BufferedInputStream(stream); + byte[] head = new byte[12]; + if (IOUtils.read(in, head, 0, 12) < 12 || head[0] != 'R' || head[1] != 'I' || + head[2] != 'F' || head[3] != 'F' || head[8] != 'W' || head[9] != 'E' || + head[10] != 'B' || head[11] != 'P') { + return null; + } + byte[] ch = new byte[8]; + while (IOUtils.read(in, ch, 0, 8) == 8) { + long size = (ch[4] & 0xffL) | (ch[5] & 0xffL) << 8 | + (ch[6] & 0xffL) << 16 | (ch[7] & 0xffL) << 24; + if (fourCC.equals(new String(ch, 0, 4, StandardCharsets.US_ASCII))) { + if (size > MAX_CHUNK) { + return null; // target chunk too large to allocate } - // a large foreign chunk before "XMP " must be skipped, not abort the scan - IOUtils.skipFully(in, size + (size & 1L)); // RIFF pads chunks to even length + byte[] data = new byte[(int) size]; + return IOUtils.read(in, data, 0, data.length) == data.length ? data : null; } + // a large foreign chunk before "XMP " must be skipped, not abort the scan + IOUtils.skipFully(in, size + (size & 1L)); // RIFF pads chunks to even length } return null; } diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/JpegParser.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/JpegParser.java index a96311857f..bf49883b5a 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/JpegParser.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/JpegParser.java @@ -26,6 +26,7 @@ import org.xml.sax.SAXException; import org.apache.tika.annotation.TikaComponent; import org.apache.tika.exception.TikaException; +import org.apache.tika.io.CacheMemoryBudget; import org.apache.tika.io.TemporaryResources; import org.apache.tika.io.TikaInputStream; import org.apache.tika.metadata.Metadata; @@ -54,9 +55,12 @@ public class JpegParser extends AbstractImageParser { TemporaryResources tmp = new TemporaryResources(); try { TikaInputStream tis = TikaInputStream.get(stream, tmp, metadata); + // two sequential passes; rewind rather than spool between them + tis.enableRewind(parseContext.get(CacheMemoryBudget.class)); // XMP first so it is canonical; the metadata-extractor handlers (IPTC/EXIF) fill gaps. - ImageXmp.extractJpeg(tis.getFile(), metadata, parseContext); - new ImageMetadataExtractor(metadata).parseJpeg(tis.getFile()); + ImageXmp.extractJpeg(tis, metadata, parseContext); + tis.rewind(); + new ImageMetadataExtractor(metadata).parseJpeg(tis); } finally { tmp.dispose(); } diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/TiffParser.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/TiffParser.java index 65664bff10..4a5cad5f48 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/TiffParser.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/TiffParser.java @@ -26,6 +26,7 @@ import org.xml.sax.SAXException; import org.apache.tika.annotation.TikaComponent; import org.apache.tika.exception.TikaException; +import org.apache.tika.io.CacheMemoryBudget; import org.apache.tika.io.TemporaryResources; import org.apache.tika.io.TikaInputStream; import org.apache.tika.metadata.Metadata; @@ -54,10 +55,11 @@ public class TiffParser extends AbstractImageParser { TemporaryResources tmp = new TemporaryResources(); try { TikaInputStream tis = TikaInputStream.get(stream, tmp, metadata); - tis.getFile(); // spool so tis is fully re-readable below + tis.enableRewind(parseContext.get(CacheMemoryBudget.class)); // XMP first so it is canonical; metadata-extractor (IPTC/EXIF) fills gaps. ImageXmp.scanAndExtract(tis, metadata, parseContext); - new ImageMetadataExtractor(metadata).parseTiff(tis.getFile()); + tis.rewind(); + new ImageMetadataExtractor(metadata).parseTiff(tis); } finally { tmp.dispose(); } diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/WebPParser.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/WebPParser.java index 1af3b5e5a3..459aa07def 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/WebPParser.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/WebPParser.java @@ -25,6 +25,7 @@ import org.xml.sax.SAXException; import org.apache.tika.annotation.TikaComponent; import org.apache.tika.exception.TikaException; +import org.apache.tika.io.CacheMemoryBudget; import org.apache.tika.io.TikaInputStream; import org.apache.tika.metadata.Metadata; import org.apache.tika.mime.MediaType; @@ -50,8 +51,10 @@ public class WebPParser implements Parser { public void parse(TikaInputStream tis, ContentHandler handler, Metadata metadata, ParseContext context) throws IOException, SAXException, TikaException { // XMP first (canonical), then EXIF/etc. from metadata-extractor as fallback. - ImageXmp.extractWebp(tis.getFile(), metadata, context); - new ImageMetadataExtractor(metadata).parseWebP(tis.getFile()); + tis.enableRewind(context.get(CacheMemoryBudget.class)); + ImageXmp.extractWebp(tis, metadata, context); + tis.rewind(); + new ImageMetadataExtractor(metadata).parseWebP(tis); XHTMLContentHandler xhtml = new XHTMLContentHandler(handler, metadata, context); xhtml.startDocument(); diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/test/java/org/apache/tika/parser/image/ImageParsersNoTempFileTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/test/java/org/apache/tika/parser/image/ImageParsersNoTempFileTest.java new file mode 100644 index 0000000000..7a72e536a6 --- /dev/null +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/test/java/org/apache/tika/parser/image/ImageParsersNoTempFileTest.java @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.parser.image; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; + +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.stream.Stream; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.xml.sax.helpers.DefaultHandler; + +import org.apache.tika.TikaTest; +import org.apache.tika.io.CacheMemoryBudget; +import org.apache.tika.io.TemporaryResources; +import org.apache.tika.io.TikaInputStream; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.metadata.TIFF; +import org.apache.tika.parser.ParseContext; +import org.apache.tika.parser.Parser; + +/** + * Embedded images usually arrive already in memory; the image parsers must not spool them to + * disk just to read metadata. The file-system tags are the visible consequence: they describe + * whatever file was read, so in-memory input must not carry them and real files must. + */ +public class ImageParsersNoTempFileTest extends TikaTest { + + private static final String FILE_NAME_KEY = ImageMetadataExtractor.UNKNOWN_IMG_NS + "File Name"; + private static final String FILE_SIZE_KEY = ImageMetadataExtractor.UNKNOWN_IMG_NS + "File Size"; + private static final String FILE_MODIFIED_KEY = + ImageMetadataExtractor.UNKNOWN_IMG_NS + "File Modified Date"; + + @TempDir + Path tempDir; + + private static final String JPEG = "/test-documents/testJPEG_EXIF.jpg"; + private static final String TIFF_RES = "/test-documents/testTIFF.tif"; + private static final String WEBP = "/test-documents/testWebp_Alpha_Lossless.webp"; + private static final String WEBP_WIDTH = ImageMetadataExtractor.UNKNOWN_IMG_NS + "Image Width"; + + private static ParseContext context() { + ParseContext context = new ParseContext(); + context.set(CacheMemoryBudget.class, new CacheMemoryBudget(64L * 1024 * 1024)); + return context; + } + + private byte[] bytes(String resource) throws Exception { + try (InputStream is = getResourceAsStream(resource)) { + return is.readAllBytes(); + } + } + + @Test + public void testJpegInMemory() throws Exception { + assertNotSpooled(new JpegParser(), JPEG, TIFF.IMAGE_WIDTH.getName()); + } + + @Test + public void testTiffInMemory() throws Exception { + assertNotSpooled(new TiffParser(), TIFF_RES, TIFF.IMAGE_WIDTH.getName()); + } + + @Test + public void testWebPInMemory() throws Exception { + assertNotSpooled(new WebPParser(), WEBP, WEBP_WIDTH); + } + + @Test + public void testJpegFromFile() throws Exception { + assertKeepsFileTags(new JpegParser(), JPEG, TIFF.IMAGE_WIDTH.getName(), "jpg"); + } + + @Test + public void testTiffFromFile() throws Exception { + assertKeepsFileTags(new TiffParser(), TIFF_RES, TIFF.IMAGE_WIDTH.getName(), "tif"); + } + + @Test + public void testWebPFromFile() throws Exception { + assertKeepsFileTags(new WebPParser(), WEBP, WEBP_WIDTH, "webp"); + } + + private void assertNotSpooled(Parser parser, String resource, String widthKey) + throws Exception { + byte[] bytes = bytes(resource); + Metadata metadata = new Metadata(); + try (TemporaryResources tmp = new TemporaryResources()) { + tmp.setTemporaryFileDirectory(tempDir); + // a stream-backed, non-file TikaInputStream whose only spill target is tempDir + TikaInputStream tis = TikaInputStream.get(new ByteArrayInputStream(bytes), tmp, metadata); + parser.parse(tis, new DefaultHandler(), metadata, context()); + // temp files live until tmp closes, so any spool would be visible right here + try (Stream<Path> files = Files.list(tempDir)) { + assertEquals(0, files.count(), "parser spooled an in-memory image to disk"); + } + } + assertNotNull(metadata.get(widthKey), "metadata was extracted"); + assertNull(metadata.get(FILE_NAME_KEY), "in-memory input must not carry file tags"); + assertNull(metadata.get(FILE_SIZE_KEY), "in-memory input must not carry file tags"); + assertNull(metadata.get(FILE_MODIFIED_KEY), "in-memory input must not carry file tags"); + } + + private void assertKeepsFileTags(Parser parser, String resource, String widthKey, String ext) + throws Exception { + byte[] bytes = bytes(resource); + Path image = tempDir.resolve("image." + ext); + Files.write(image, bytes); + Metadata metadata = new Metadata(); + try (TikaInputStream tis = TikaInputStream.get(image, metadata)) { + parser.parse(tis, new DefaultHandler(), metadata, context()); + } + assertNotNull(metadata.get(widthKey), "metadata was extracted"); + assertEquals(image.getFileName().toString(), metadata.get(FILE_NAME_KEY), + "a real file keeps metadata-extractor's file-system tags"); + assertNotNull(metadata.get(FILE_SIZE_KEY), "a real file keeps its size tag"); + } +} diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/detect/microsoft/POIFSContainerDetector.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/detect/microsoft/POIFSContainerDetector.java index 356dcba2f1..4b5f470815 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/detect/microsoft/POIFSContainerDetector.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/detect/microsoft/POIFSContainerDetector.java @@ -22,6 +22,9 @@ import static org.apache.tika.mime.MediaType.image; import java.io.IOException; import java.io.InputStream; +import java.nio.ByteBuffer; +import java.nio.channels.Channels; +import java.nio.channels.SeekableByteChannel; import java.nio.charset.StandardCharsets; import java.nio.file.Path; import java.util.Collections; @@ -32,18 +35,23 @@ import java.util.Set; import java.util.regex.Pattern; import org.apache.commons.io.IOUtils; +import org.apache.commons.io.input.UnsynchronizedByteArrayInputStream; import org.apache.poi.hssf.model.InternalWorkbook; +import org.apache.poi.poifs.common.POIFSConstants; import org.apache.poi.poifs.filesystem.DirectoryEntry; import org.apache.poi.poifs.filesystem.DirectoryNode; import org.apache.poi.poifs.filesystem.DocumentInputStream; import org.apache.poi.poifs.filesystem.DocumentNode; import org.apache.poi.poifs.filesystem.Entry; import org.apache.poi.poifs.filesystem.POIFSFileSystem; +import org.apache.poi.poifs.storage.BATBlock; +import org.apache.poi.poifs.storage.HeaderBlock; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.tika.annotation.TikaComponent; import org.apache.tika.detect.Detector; +import org.apache.tika.io.CacheMemoryBudget; import org.apache.tika.io.TikaInputStream; import org.apache.tika.metadata.Metadata; import org.apache.tika.mime.MediaType; @@ -554,9 +562,22 @@ public class POIFSContainerDetector implements Detector { } - private Set<String> getTopLevelNames(TikaInputStream stream) throws IOException { - // Force the document stream to a (possibly temporary) file - // so we don't modify the current position of the stream. + /** + * Largest object opened from memory. POI's stream loader copies the whole object into a + * heap array, so this bounds that copy; the cache underneath is already budget-bounded. + * Embedded OLE2 objects are typically a few hundred KB. + */ + private static final long MAX_IN_MEMORY_POIFS = 8L * 1024 * 1024; + + private Set<String> getTopLevelNames(TikaInputStream stream, ParseContext context) + throws IOException { + if (!stream.hasFile()) { + Set<String> names = getTopLevelNamesInMemory(stream, context); + if (names != null) { + return names; + } + } + // random access over content that is not in memory: use the file Path file = stream.getPath(); if (file == null) { @@ -583,16 +604,89 @@ public class POIFSContainerDetector implements Detector { } } + /** + * Opens the container from memory. Returns null -- and the caller uses the file -- when + * the object is too large, its header would make POI allocate too much, or POI's stream + * loader rejects it (it is stricter than the file loader on truncated objects). + */ + private Set<String> getTopLevelNamesInMemory(TikaInputStream stream, ParseContext context) + throws IOException { + if (stream.hasLength() && stream.getLength() > MAX_IN_MEMORY_POIFS) { + return null; + } + // the budget decides how much the drain below keeps in memory + stream.enableRewind(context == null ? null : context.get(CacheMemoryBudget.class)); + POIFSFileSystem fs = null; + try (SeekableByteChannel channel = stream.getSeekableByteChannel()) { + if (TikaInputStream.inMemoryContent(channel) == null || + channel.size() > MAX_IN_MEMORY_POIFS || + declaredInMemorySize(channel) > MAX_IN_MEMORY_POIFS) { + return null; + } + fs = new POIFSFileSystem(Channels.newInputStream(channel)); + Set<String> names = getTopLevelNames(fs.getRoot()); + stream.setOpenContainer(fs); + fs = null; + return names; + } catch (SecurityException e) { + throw e; + } catch (IOException | RuntimeException e) { + return null; + } finally { + if (fs != null) { + closeQuietly(fs); + } + } + } + + /** + * The heap POI would allocate for this object if opened from a stream. It sizes that buffer + * from the header's declared BAT count rather than the actual length, so a 512-byte object + * can demand hundreds of MB; the file loader reads block by block and never allocates it. + * Returns 0 when the header cannot be read, leaving the rejection to POI. + */ + static long declaredInMemorySize(SeekableByteChannel channel) throws IOException { + byte[] header = new byte[POIFSConstants.SMALLER_BIG_BLOCK_SIZE]; + long start = channel.position(); + try { + channel.position(0); + ByteBuffer buffer = ByteBuffer.wrap(header); + while (buffer.hasRemaining() && channel.read(buffer) > 0) { + // fill the header block + } + if (buffer.hasRemaining()) { + return 0; + } + } finally { + channel.position(start); + } + try { + return BATBlock.calculateMaximumSize( + new HeaderBlock(UnsynchronizedByteArrayInputStream.builder().setByteArray(header).get())); + } catch (IOException | RuntimeException e) { + return 0; + } + } + + private static void closeQuietly(POIFSFileSystem fs) { + try { + fs.close(); + } catch (IOException e) { + LOG.debug("failed to close abandoned POIFSFileSystem", e); + } + } + public MediaType detect(TikaInputStream tis, Metadata metadata, ParseContext parseContext) throws IOException { // Check if we have access to the document if (tis == null) { return MediaType.OCTET_STREAM; } - return handleTikaStream(tis, metadata); + return handleTikaStream(tis, metadata, parseContext); } - private MediaType handleTikaStream(TikaInputStream tis, Metadata metadata) throws IOException { + private MediaType handleTikaStream(TikaInputStream tis, Metadata metadata, ParseContext context) + throws IOException { //try for an open container Set<String> names = tryOpenContainerOnTikaInputStream(tis, metadata); @@ -601,10 +695,8 @@ public class POIFSContainerDetector implements Detector { return OCTET_STREAM; } - // If OLE, spool to disk if (names == null) { - // spool to disk and try detection - names = getTopLevelNames(tis); + names = getTopLevelNames(tis, context); } // Detect based on the names (as available) diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/detect/microsoft/POIFSContainerDetectorNoTempFileTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/detect/microsoft/POIFSContainerDetectorNoTempFileTest.java new file mode 100644 index 0000000000..b00f2e57e7 --- /dev/null +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/detect/microsoft/POIFSContainerDetectorNoTempFileTest.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.detect.microsoft; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.stream.Stream; + +import org.apache.poi.poifs.filesystem.POIFSFileSystem; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import org.apache.tika.TikaTest; +import org.apache.tika.io.CacheMemoryBudget; +import org.apache.tika.io.TemporaryResources; +import org.apache.tika.io.TikaInputStream; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.mime.MediaType; +import org.apache.tika.parser.ParseContext; + +/** + * Detection used to spool every in-memory OLE2 object to a temp file to read its + * top-level entry names; it must now open the container from memory. + */ +public class POIFSContainerDetectorNoTempFileTest extends TikaTest { + + @TempDir + Path tempDir; + + @Test + public void testNoTempFileForInMemoryInput() throws Exception { + byte[] bytes; + try (InputStream is = getResourceAsStream("/test-documents/testWORD.doc")) { + bytes = is.readAllBytes(); + } + ParseContext context = new ParseContext(); + context.set(CacheMemoryBudget.class, new CacheMemoryBudget(64L * 1024 * 1024)); + Metadata metadata = new Metadata(); + try (TemporaryResources tmp = new TemporaryResources()) { + tmp.setTemporaryFileDirectory(tempDir); + TikaInputStream tis = TikaInputStream.get(new ByteArrayInputStream(bytes), tmp, metadata); + MediaType type = new POIFSContainerDetector().detect(tis, metadata, context); + assertEquals(MediaType.application("msword"), type); + try (Stream<Path> files = Files.list(tempDir)) { + assertEquals(0, files.count(), "detector spooled an in-memory OLE2 object to disk"); + } + assertTrue(tis.getOpenContainer() instanceof POIFSFileSystem, "open container kept for the parser"); + assertEquals(0, tis.getPosition(), "detection must not move the stream"); + assertEquals(0xd0, tis.read(), "stream still readable from the start"); + } + } +} diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/detect/microsoft/POIFSDeclaredSizeTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/detect/microsoft/POIFSDeclaredSizeTest.java new file mode 100644 index 0000000000..348655d6e5 --- /dev/null +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/detect/microsoft/POIFSDeclaredSizeTest.java @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.detect.microsoft; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.ByteArrayInputStream; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.channels.SeekableByteChannel; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import org.apache.tika.TikaTest; +import org.apache.tika.io.CacheMemoryBudget; +import org.apache.tika.io.TemporaryResources; +import org.apache.tika.io.TikaInputStream; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.parser.ParseContext; + +/** + * POI sizes its in-memory OLE2 buffer from the header's declared BAT count rather than the + * actual length, so a 512-byte object can demand hundreds of MB. The in-memory detection path + * must reject those on the declared size, not on the real one. + */ +public class POIFSDeclaredSizeTest extends TikaTest { + + private static final int BAT_COUNT_OFFSET = 0x2C; + private static final int SECTOR_SHIFT_OFFSET = 0x1E; + + @TempDir + Path tempDir; + + /** A bare 512-byte OLE2 header declaring {@code batCount} BAT blocks and nothing else. */ + private static byte[] header(int batCount) { + byte[] data = new byte[512]; + byte[] magic = {(byte) 0xd0, (byte) 0xcf, 0x11, (byte) 0xe0, + (byte) 0xa1, (byte) 0xb1, 0x1a, (byte) 0xe1}; + System.arraycopy(magic, 0, data, 0, magic.length); + data[SECTOR_SHIFT_OFFSET] = 9; // 2^9 = 512-byte blocks + ByteBuffer.wrap(data).order(ByteOrder.LITTLE_ENDIAN).putInt(BAT_COUNT_OFFSET, batCount); + return data; + } + + private SeekableByteChannel channelFor(byte[] bytes, String name) throws Exception { + Path p = tempDir.resolve(name); + Files.write(p, bytes); + return Files.newByteChannel(p, StandardOpenOption.READ); + } + + @Test + public void testDeclaredSizeFollowsHeaderNotLength() throws Exception { + // (1 + 3813 * 128) * 512 == 249_889_280, just under POI's 250MB allocation ceiling + try (SeekableByteChannel channel = channelFor(header(3813), "hostile.ole")) { + long declared = POIFSContainerDetector.declaredInMemorySize(channel); + assertEquals(249_889_280L, declared, + "a 512-byte object declares a ~238MB in-memory buffer"); + assertEquals(0, channel.position(), "the size probe must not move the channel"); + } + } + + @Test + public void testModestHeaderIsNotRejected() throws Exception { + try (SeekableByteChannel channel = channelFor(header(1), "modest.ole")) { + assertEquals((1 + 128) * 512L, POIFSContainerDetector.declaredInMemorySize(channel)); + } + } + + @Test + public void testTooShortForAHeaderReportsZero() throws Exception { + try (SeekableByteChannel channel = channelFor(new byte[16], "short.bin")) { + assertEquals(0, POIFSContainerDetector.declaredInMemorySize(channel), + "no header to read; leave the rejection to POI"); + } + } + + /** + * End to end: the crafted object must not be opened in memory, so no POIFSFileSystem is + * retained. Before the declared-size check this allocated ~238MB first. + */ + @Test + public void testHostileHeaderDoesNotBecomeAnOpenContainer() throws Exception { + ParseContext context = new ParseContext(); + context.set(CacheMemoryBudget.class, new CacheMemoryBudget(64L * 1024 * 1024)); + Metadata metadata = new Metadata(); + try (TemporaryResources tmp = new TemporaryResources()) { + TikaInputStream tis = TikaInputStream.get( + new ByteArrayInputStream(header(3813)), tmp, metadata); + POIFSContainerDetector detector = new POIFSContainerDetector(); + assertTrue(detector.detect(tis, metadata, context) != null); + assertNull(tis.getOpenContainer(), + "a header-only object must not be opened from memory"); + } + } +} diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/OpenDocumentParser.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/OpenDocumentParser.java index c500689ab4..5cc0f37a1c 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/OpenDocumentParser.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/OpenDocumentParser.java @@ -45,6 +45,7 @@ import org.apache.tika.exception.EncryptedDocumentException; import org.apache.tika.exception.TikaException; import org.apache.tika.extractor.EmbeddedDocumentExtractor; import org.apache.tika.extractor.EmbeddedDocumentUtil; +import org.apache.tika.io.CacheMemoryBudget; import org.apache.tika.io.TikaInputStream; import org.apache.tika.metadata.HttpHeaders; import org.apache.tika.metadata.Metadata; @@ -320,14 +321,13 @@ public class OpenDocumentParser implements Parser { if (embeddedName.contains("Pictures/")) { embeddedMetadata.set(TikaCoreProperties.EMBEDDED_RESOURCE_TYPE, TikaCoreProperties.EmbeddedResourceType.INLINE.toString()); - //spool - tisZip.getFile(); + tisZip.enableRewind(context.get(CacheMemoryBudget.class)); MediaType embeddedMimeType = EmbeddedDocumentUtil.getDetector(context) .detect(tisZip, embeddedMetadata, context); if (embeddedMimeType != null) { embeddedMetadata.set(HttpHeaders.CONTENT_TYPE, embeddedMimeType.toString()); } - tisZip.reset(); + tisZip.rewind(); // Tag the picture with the draw:page indices it // appears on (set populated by scanPicturePages). // A null lookup means "not referenced by any diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/test/java/org/apache/tika/parser/odf/OpenDocumentParserNoTempFileTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/test/java/org/apache/tika/parser/odf/OpenDocumentParserNoTempFileTest.java new file mode 100644 index 0000000000..fd93ebb635 --- /dev/null +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/test/java/org/apache/tika/parser/odf/OpenDocumentParserNoTempFileTest.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.parser.odf; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; + +import org.junit.jupiter.api.Test; + +import org.apache.tika.TikaTest; +import org.apache.tika.detect.DefaultDetector; +import org.apache.tika.detect.Detector; +import org.apache.tika.io.CacheMemoryBudget; +import org.apache.tika.io.TemporaryResources; +import org.apache.tika.io.TikaInputStream; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.metadata.TikaCoreProperties; +import org.apache.tika.mime.MediaType; +import org.apache.tika.parser.ParseContext; +import org.apache.tika.sax.BodyContentHandler; + +/** + * Inline pictures were spooled to a temp file before detection; with rewind support the + * entry stays in memory. Watching the temp directory cannot see this -- each entry gets its + * own TemporaryResources -- so the assertion is on the stream the detector is handed. + */ +public class OpenDocumentParserNoTempFileTest extends TikaTest { + + /** Records whether each inline picture reached detection backed by a file. */ + private static class SpyDetector implements Detector { + private final Detector delegate = new DefaultDetector(); + private final List<Boolean> pictureHadFile = new ArrayList<>(); + + @Override + public MediaType detect(TikaInputStream input, Metadata metadata, ParseContext context) + throws IOException { + String type = metadata.get(TikaCoreProperties.EMBEDDED_RESOURCE_TYPE); + if (TikaCoreProperties.EmbeddedResourceType.INLINE.toString().equals(type)) { + pictureHadFile.add(input.hasFile()); + } + return delegate.detect(input, metadata, context); + } + } + + @Test + public void testInlinePicturesAreDetectedFromMemory() throws Exception { + byte[] bytes; + try (InputStream is = getResourceAsStream("/test-documents/testODTEmbedded.odt")) { + bytes = is.readAllBytes(); + } + SpyDetector spy = new SpyDetector(); + ParseContext context = new ParseContext(); + context.set(Detector.class, spy); + context.set(CacheMemoryBudget.class, new CacheMemoryBudget(64L * 1024 * 1024)); + Metadata metadata = new Metadata(); + BodyContentHandler handler = new BodyContentHandler(); + try (TemporaryResources tmp = new TemporaryResources()) { + TikaInputStream tis = TikaInputStream.get(new ByteArrayInputStream(bytes), tmp, metadata); + new OpenDocumentParser().parse(tis, handler, metadata, context); + } + assertFalse(spy.pictureHadFile.isEmpty(), "no inline picture reached detection"); + for (Boolean hadFile : spy.pictureHadFile) { + assertFalse(hadFile, "inline picture was spooled to disk before detection"); + } + assertTrue(handler.toString().length() > 0, "content was extracted"); + } + + /** The picture must be re-readable after detection, or the embedded parse sees nothing. */ + @Test + public void testEmbeddedPicturesStillParseAfterRewind() throws Exception { + List<Metadata> metadataList = getRecursiveMetadata("testODTEmbedded.odt"); + boolean sawImage = false; + for (Metadata m : metadataList) { + assertEquals(null, m.get(TikaCoreProperties.EMBEDDED_EXCEPTION), + "embedded exception for " + m.get(TikaCoreProperties.RESOURCE_NAME_KEY)); + String type = m.get(TikaCoreProperties.EMBEDDED_RESOURCE_TYPE); + if (TikaCoreProperties.EmbeddedResourceType.INLINE.toString().equals(type)) { + sawImage = true; + } + } + assertTrue(sawImage, "no inline picture in the recursive metadata"); + } +} diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFParser.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFParser.java index c61f33cc45..bfdf538101 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFParser.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFParser.java @@ -40,7 +40,6 @@ import org.apache.pdfbox.cos.COSString; import org.apache.pdfbox.io.MemoryUsageSetting; import org.apache.pdfbox.io.RandomAccessRead; import org.apache.pdfbox.io.RandomAccessReadBuffer; -import org.apache.pdfbox.io.RandomAccessReadBufferedFile; import org.apache.pdfbox.io.RandomAccessStreamCache; import org.apache.pdfbox.pdmodel.PDDocument; import org.apache.pdfbox.pdmodel.PDDocumentInformation; @@ -66,6 +65,7 @@ import org.apache.tika.exception.EncryptedDocumentException; import org.apache.tika.exception.TikaException; import org.apache.tika.extractor.EmbeddedDocumentExtractor; import org.apache.tika.extractor.EmbeddedDocumentUtil; +import org.apache.tika.io.CacheMemoryBudget; import org.apache.tika.io.TikaInputStream; import org.apache.tika.metadata.AccessPermissions; import org.apache.tika.metadata.HttpHeaders; @@ -181,6 +181,8 @@ public class PDFParser implements Parser, RenderingParser { context.set(OCRPageCounter.class, new OCRPageCounter()); try { if (shouldSpool(localConfig)) { + // later stages re-open the document (xref scan, renderer, per-page renders) + tis.enableRewind(context.get(CacheMemoryBudget.class)); context.set(PDFRenderingState.class, new PDFRenderingState(tis)); } @@ -313,10 +315,7 @@ public class PDFParser implements Parser, RenderingParser { return; } List<StartXRefOffset> xRefOffsets = new ArrayList<>(); - //TODO -- can we use the PDFBox parser's RandomAccessRead - //so that we don't have to reopen from file? - try (RandomAccessRead ra = - new RandomAccessReadBufferedFile(tikaInputStream.getFile())) { + try (RandomAccessRead ra = PDFRandomAccess.open(tikaInputStream, parseContext)) { StartXRefScanner xRefScanner = new StartXRefScanner(ra); xRefOffsets.addAll(xRefScanner.scan()); } catch (IOException e) { @@ -495,21 +494,18 @@ public class PDFParser implements Parser, RenderingParser { ParseContext context) throws IOException, EncryptedDocumentException { try { - PDDocument pdDocument = null; if (tis.hasFile()) { // File based -- send file directly to PDFBox - pdDocument = - getPDDocument(tis.getPath(), password, streamCacheCreateFunction, metadata, context); - } else { - tis.setCloseShield(); - try { - pdDocument = getPDDocumentFromStream(tis, password, - streamCacheCreateFunction, metadata, context); - } finally { - tis.removeCloseShield(); - } + return getPDDocument(tis.getPath(), password, streamCacheCreateFunction, metadata, context); + } + // PDFBox owns the reader and closes it with the document + RandomAccessRead ra = PDFRandomAccess.open(tis, context); + try { + return getPDDocument(ra, password, streamCacheCreateFunction, metadata, context); + } catch (IOException | RuntimeException e) { + ra.close(); + throw e; } - return pdDocument; } catch (IOException e) { if (e.getMessage() != null && e.getMessage().contains("No security handler for filter")) { @@ -519,6 +515,12 @@ public class PDFParser implements Parser, RenderingParser { } } + /** + * @deprecated no longer called by this parser: stream-backed input is loaded through + * {@link #getPDDocument(RandomAccessRead, String, RandomAccessStreamCache.StreamCacheCreateFunction, Metadata, ParseContext)}, + * which does not copy the document into heap the way {@code RandomAccessReadBuffer(InputStream)} does + */ + @Deprecated protected PDDocument getPDDocumentFromStream(InputStream inputStream, String password, RandomAccessStreamCache.StreamCacheCreateFunction streamCacheCreateFunction, Metadata metadata, @@ -526,6 +528,13 @@ public class PDFParser implements Parser, RenderingParser { return Loader.loadPDF(new RandomAccessReadBuffer(inputStream), password, streamCacheCreateFunction); } + protected PDDocument getPDDocument(RandomAccessRead source, String password, + RandomAccessStreamCache.StreamCacheCreateFunction streamCacheCreateFunction, + Metadata metadata, + ParseContext parseContext) throws IOException { + return Loader.loadPDF(source, password, streamCacheCreateFunction); + } + protected PDDocument getPDDocument(Path path, String password, RandomAccessStreamCache.StreamCacheCreateFunction streamCacheCreateFunction, Metadata metadata, diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFRandomAccess.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFRandomAccess.java new file mode 100644 index 0000000000..923f5043d4 --- /dev/null +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFRandomAccess.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.parser.pdf; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.SeekableByteChannel; + +import org.apache.pdfbox.io.RandomAccessRead; +import org.apache.pdfbox.io.RandomAccessReadBuffer; +import org.apache.pdfbox.io.RandomAccessReadBufferedFile; + +import org.apache.tika.io.CacheMemoryBudget; +import org.apache.tika.io.TikaInputStream; +import org.apache.tika.parser.ParseContext; + +/** + * One way to hand PDFBox a document: in-memory content is read in place through a zero-copy + * view, anything else through PDFBox's buffered file reader. Neither copies the document, and + * every call starts from byte 0 regardless of the stream's position -- so a renderer can + * re-open the same document per page. Closing the returned reader releases the view. + */ +public final class PDFRandomAccess { + + private PDFRandomAccess() { + } + + public static RandomAccessRead open(TikaInputStream tis, ParseContext context) + throws IOException { + if (tis.hasFile()) { + return new RandomAccessReadBufferedFile(tis.getFile()); + } + tis.enableRewind(context == null ? null : context.get(CacheMemoryBudget.class)); + SeekableByteChannel channel = tis.getSeekableByteChannel(); + ByteBuffer view = TikaInputStream.inMemoryContent(channel); + if (view == null) { + // the drain spilled: the content is on disk now + channel.close(); + return new RandomAccessReadBufferedFile(tis.getFile()); + } + return new RandomAccessReadBuffer(view) { + @Override + public void close() throws IOException { + try { + super.close(); + } finally { + channel.close(); + } + } + }; + } +} diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/renderer/pdf/pdfbox/PDFBoxRenderer.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/renderer/pdf/pdfbox/PDFBoxRenderer.java index 8c5c94592a..20bc2117ef 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/renderer/pdf/pdfbox/PDFBoxRenderer.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/renderer/pdf/pdfbox/PDFBoxRenderer.java @@ -25,7 +25,6 @@ import java.util.Collections; import java.util.Set; import org.apache.pdfbox.Loader; -import org.apache.pdfbox.io.RandomAccessReadBuffer; import org.apache.pdfbox.pdmodel.PDDocument; import org.apache.pdfbox.rendering.ImageType; import org.apache.pdfbox.rendering.PDFRenderer; @@ -47,6 +46,7 @@ import org.apache.tika.mime.MediaType; import org.apache.tika.parser.ParseContext; import org.apache.tika.parser.pdf.PDFParser; import org.apache.tika.parser.pdf.PDFParserConfig; +import org.apache.tika.parser.pdf.PDFRandomAccess; import org.apache.tika.renderer.PageBasedRenderResults; import org.apache.tika.renderer.PageRangeRequest; import org.apache.tika.renderer.RenderRequest; @@ -100,7 +100,9 @@ public class PDFBoxRenderer implements PDDocumentRenderer { if (tis.getOpenContainer() != null) { pdDocument = (PDDocument) tis.getOpenContainer(); } else { - pdDocument = Loader.loadPDF(new RandomAccessReadBuffer(tis)); + // a fresh reader from byte 0 each time, so per-page renders do not depend on + // where the stream was left; the document closes it + pdDocument = Loader.loadPDF(PDFRandomAccess.open(tis, parseContext)); mustClose = true; } PageBasedRenderResults results = new PageBasedRenderResults(new TemporaryResources()); diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/test/java/org/apache/tika/parser/pdf/PDFParserNoTempFileTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/test/java/org/apache/tika/parser/pdf/PDFParserNoTempFileTest.java new file mode 100644 index 0000000000..9529fee4d4 --- /dev/null +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/test/java/org/apache/tika/parser/pdf/PDFParserNoTempFileTest.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.parser.pdf; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.stream.Stream; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.xml.sax.helpers.DefaultHandler; + +import org.apache.tika.TikaTest; +import org.apache.tika.io.CacheMemoryBudget; +import org.apache.tika.io.TemporaryResources; +import org.apache.tika.io.TikaInputStream; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.metadata.PDF; +import org.apache.tika.parser.ParseContext; + +/** + * The incremental-update xref scan used to spool in-memory PDFs to a file; it must scan + * from memory like the main parse does. + */ +public class PDFParserNoTempFileTest extends TikaTest { + + @TempDir + Path tempDir; + + @Test + public void testXRefScanNoTempFileForInMemoryInput() throws Exception { + byte[] bytes; + try (InputStream is = getResourceAsStream("/test-documents/testPDF_incrementalUpdates.pdf")) { + bytes = is.readAllBytes(); + } + PDFParserConfig config = new PDFParserConfig(); + config.setExtractIncrementalUpdateInfo(true); + ParseContext context = new ParseContext(); + context.set(PDFParserConfig.class, config); + context.set(CacheMemoryBudget.class, new CacheMemoryBudget(64L * 1024 * 1024)); + Metadata metadata = new Metadata(); + try (TemporaryResources tmp = new TemporaryResources()) { + tmp.setTemporaryFileDirectory(tempDir); + TikaInputStream tis = TikaInputStream.get(new ByteArrayInputStream(bytes), tmp, metadata); + new PDFParser().parse(tis, new DefaultHandler(), metadata, context); + try (Stream<Path> files = Files.list(tempDir)) { + assertEquals(0, files.count(), "xref scan spooled an in-memory PDF to disk"); + } + } + assertNotNull(metadata.get(PDF.PDF_INCREMENTAL_UPDATE_COUNT), "incremental update info was extracted"); + } +} diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/test/java/org/apache/tika/parser/pdf/PDFPerPageRenderTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/test/java/org/apache/tika/parser/pdf/PDFPerPageRenderTest.java new file mode 100644 index 0000000000..df88be575c --- /dev/null +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/test/java/org/apache/tika/parser/pdf/PDFPerPageRenderTest.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.parser.pdf; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import org.apache.tika.TikaTest; +import org.apache.tika.io.TikaInputStream; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.metadata.PagedText; +import org.apache.tika.metadata.TikaCoreProperties; +import org.apache.tika.parser.ParseContext; + +/** + * Per-page rendering re-opens the document once per page. Each open must start from byte 0 + * regardless of where the stream was left; before PDFRandomAccess the renderer read the + * TikaInputStream from its current position, so page 1 drained it and every later page + * rendered nothing, silently. + */ +public class PDFPerPageRenderTest extends TikaTest { + + private static final String TWO_PAGES = "/test-documents/testPDF_bookmarks.pdf"; + + @TempDir + Path tempDir; + + private static ParseContext renderAtPageEnd() { + PDFParserConfig config = new PDFParserConfig(); + config.setImageStrategy(PDFParserConfig.IMAGE_STRATEGY.RENDER_PAGES_AT_PAGE_END); + ParseContext context = new ParseContext(); + context.set(PDFParserConfig.class, config); + return context; + } + + private static void assertEveryPageRendered(List<Metadata> metadataList) { + Metadata container = metadataList.get(0); + assertNull(container.get(TikaCoreProperties.EMBEDDED_EXCEPTION), + "a page render failure is recorded on the container, not thrown"); + int pages = container.getInt(PagedText.N_PAGES); + assertEquals(2, pages, "fixture must be multi-page for this test to mean anything"); + long rendered = metadataList.stream() + .filter(m -> TikaCoreProperties.EmbeddedResourceType.RENDERING.name() + .equals(m.get(TikaCoreProperties.EMBEDDED_RESOURCE_TYPE))) + .count(); + assertEquals(pages, rendered, "one rendering per page"); + } + + @Test + public void testStreamBackedRendersEveryPage() throws Exception { + byte[] bytes; + try (InputStream is = getResourceAsStream(TWO_PAGES)) { + bytes = is.readAllBytes(); + } + try (TikaInputStream tis = TikaInputStream.get(new ByteArrayInputStream(bytes), new Metadata())) { + assertEveryPageRendered(getRecursiveMetadata(tis, AUTO_DETECT_PARSER, new Metadata(), + renderAtPageEnd(), true)); + } + } + + @Test + public void testFileBackedRendersEveryPage() throws Exception { + Path pdf = tempDir.resolve("two-pages.pdf"); + try (InputStream is = getResourceAsStream(TWO_PAGES)) { + Files.copy(is, pdf); + } + try (TikaInputStream tis = TikaInputStream.get(pdf)) { + assertEveryPageRendered(getRecursiveMetadata(tis, AUTO_DETECT_PARSER, new Metadata(), + renderAtPageEnd(), true)); + } + } +}
