This is an automated email from the ASF dual-hosted git repository. tballison pushed a commit to branch TIKA-4868-performance-improvements in repository https://gitbox.apache.org/repos/asf/tika.git
commit a864bc897e203f2c94dd814db7ab2c058e9dfc19 Author: tallison <[email protected]> AuthorDate: Wed Sep 2 09:38:01 2026 -0400 TIKA-4868 fixes/reviewer feedback --- CHANGES.txt | 17 +- tika-app/src/main/resources/log4j2.xml | 2 + .../java/org/apache/tika/config/ParseTimeout.java | 2 +- .../apache/tika/config/TransientParseState.java | 26 +++ .../java/org/apache/tika/detect/MagicDetector.java | 18 +- .../java/org/apache/tika/io/ByteArraySource.java | 5 + .../java/org/apache/tika/io/CachingSource.java | 9 + .../main/java/org/apache/tika/io/FileSource.java | 5 + .../java/org/apache/tika/io/ReopenableSource.java | 9 + .../java/org/apache/tika/io/TikaInputSource.java | 6 + .../java/org/apache/tika/io/TikaInputStream.java | 46 +++-- .../main/java/org/apache/tika/mime/MimeTypes.java | 5 +- .../main/java/org/apache/tika/mime/Patterns.java | 22 ++- .../org/apache/tika/parser/CompositeParser.java | 20 +- .../java/org/apache/tika/parser/ParseRecord.java | 3 +- .../apache/tika/sax/FastMarkdownTextRenderer.java | 9 +- .../apache/tika/sax/ToMarkdownContentHandler.java | 3 +- .../org/apache/tika/io/DeclaredLengthTest.java | 111 ++++++++++++ .../org/apache/tika/mime/MimeDetectionTest.java | 14 ++ .../java/org/apache/tika/mime/PatternsTest.java | 16 ++ .../tika/sax/FastMarkdownTextRendererTest.java | 201 +++++++++++++++++++++ .../tika/sax/ToMarkdownContentHandlerTest.java | 18 ++ .../tika/parser/microsoft/WordExtractor.java | 4 - .../org/apache/tika/parser/csv/CSVSniffer.java | 12 +- .../apache/tika/pipes/core/ContentBytesConfig.java | 5 +- .../tika/pipes/core/async/AsyncProcessor.java | 6 + .../tika/pipes/core/emitter/EmitDataImpl.java | 20 ++ .../apache/tika/pipes/core/server/EmitHandler.java | 7 +- .../serdes/ParseContextSerializer.java | 8 + .../TestParseContextSerialization.java | 15 ++ .../tika/server/core/resource/TikaResource.java | 6 +- .../tika-server-core/src/main/resources/log4j2.xml | 2 + .../src/main/resources/log4j2.xml | 2 + 33 files changed, 591 insertions(+), 63 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index f125119319..f0a5cf99d3 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -33,8 +33,9 @@ Release 4.1.0 - unreleased (4.1.0-dev ran it twice) to 52us per document; eml requests ~-45%, ppt ~-35% in tika-server. MagicDetector also precomputes a 256-entry first-byte table per pattern (mask and case fold become one array - load per scanned position). Adds an opt-in RESOURCE_TIMING log on - org.apache.tika.pipes.timing.resource (TIKA-4868). + load per scanned position). Adds a RESOURCE_TIMING log on + org.apache.tika.pipes.timing.resource, silenced by default in the + shipped log4j2 configs (TIKA-4868). * WordExtractor (.doc) cleans each character run in one pass instead of four chained replace/replaceAll copies, and tests paragraph blankness @@ -77,15 +78,19 @@ Release 4.1.0 - unreleased * Pipes workers no longer stall between pre-parse and parse waiting for the client to acknowledge the intermediate-result frame; the - ACK round trip now overlaps the parse. Adds opt-in per-request - timing logs on org.apache.tika.pipes.timing.* (TIKA-4868). + ACK round trip now overlaps the parse. Adds per-request timing logs + on org.apache.tika.pipes.timing.*, silenced by default in the shipped + log4j2 configs; raise that logger to info to enable (TIKA-4868). * DefaultDetector honors CONTENT_TYPE_USER_OVERRIDE and CONTENT_TYPE_PARSER_OVERRIDE before running magic detection, matching CompositeDetector's contract. Removes the second full magic scan every - pipes parse paid per document. Compat note: with a user override set, + pipes parse paid per document. Compat note: with either override set, DefaultDetector no longer lets a more specific magic result overrule - the override (TIKA-4868). + it; parts whose parser declares a type from container headers (e.g. + inline text/* mail parts) now report the declared type, and + CONTENT_TYPE_MAGIC_DETECTED is not recorded when an override short + circuits detection (TIKA-4868). * Markdown output is ~4x faster on large documents: ToMarkdownContentHandler now buffers the commonmark renderer's diff --git a/tika-app/src/main/resources/log4j2.xml b/tika-app/src/main/resources/log4j2.xml index c88e66e99e..d22d891023 100644 --- a/tika-app/src/main/resources/log4j2.xml +++ b/tika-app/src/main/resources/log4j2.xml @@ -25,6 +25,8 @@ </Console> </Appenders> <Loggers> + <!-- opt-in per-request timing lines (TIKA-4868): raise to info to enable --> + <Logger name="org.apache.tika.pipes.timing" level="warn"/> <Root level="info"> <AppenderRef ref="Console"/> </Root> diff --git a/tika-core/src/main/java/org/apache/tika/config/ParseTimeout.java b/tika-core/src/main/java/org/apache/tika/config/ParseTimeout.java index 4f9d385907..df95726f46 100644 --- a/tika-core/src/main/java/org/apache/tika/config/ParseTimeout.java +++ b/tika-core/src/main/java/org/apache/tika/config/ParseTimeout.java @@ -49,7 +49,7 @@ import org.apache.tika.parser.ParseContext; * * @since Apache Tika 4.0 */ -public class ParseTimeout { +public class ParseTimeout implements TransientParseState { private static final Logger LOG = LoggerFactory.getLogger(ParseTimeout.class); diff --git a/tika-core/src/main/java/org/apache/tika/config/TransientParseState.java b/tika-core/src/main/java/org/apache/tika/config/TransientParseState.java new file mode 100644 index 0000000000..cf3d0ef3f2 --- /dev/null +++ b/tika-core/src/main/java/org/apache/tika/config/TransientParseState.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.config; + +/** + * Marker for per-parse runtime state that parsers store in a + * {@link org.apache.tika.parser.ParseContext} (e.g. {@code ParseRecord}, + * {@code ParseTimeout}). Such entries are never configuration: serializers + * skip them instead of failing on an unregistered component. + */ +public interface TransientParseState { +} diff --git a/tika-core/src/main/java/org/apache/tika/detect/MagicDetector.java b/tika-core/src/main/java/org/apache/tika/detect/MagicDetector.java index d9d6b2ad2d..2af36ee54f 100644 --- a/tika-core/src/main/java/org/apache/tika/detect/MagicDetector.java +++ b/tika-core/src/main/java/org/apache/tika/detect/MagicDetector.java @@ -459,15 +459,6 @@ public class MagicDetector implements Detector { } } - /** - * Core matching logic that checks if the pattern matches anywhere in the buffer - * within the specified offset range. - * - * @param buffer the byte array to search in - * @param startOffset the first position in the buffer to start matching (inclusive) - * @param endOffset the last position in the buffer to start matching (inclusive) - * @return true if a match is found, false otherwise - */ // Which raw first bytes can begin a match; encodes mask[0] and the case fold so the // scan loop is a single table load. Built lazily, idempotent under racing builds. private transient volatile boolean[] firstByteMatches; @@ -485,6 +476,15 @@ public class MagicDetector implements Detector { return table; } + /** + * Core matching logic that checks if the pattern matches anywhere in the buffer + * within the specified offset range. + * + * @param buffer the byte array to search in + * @param startOffset the first position in the buffer to start matching (inclusive) + * @param endOffset the last position in the buffer to start matching (inclusive) + * @return true if a match is found, false otherwise + */ private boolean matchesBuffer(byte[] buffer, int startOffset, int endOffset) { if (this.isRegex) { int bufferLen = Math.min(buffer.length - startOffset, length + (endOffset - startOffset)); diff --git a/tika-core/src/main/java/org/apache/tika/io/ByteArraySource.java b/tika-core/src/main/java/org/apache/tika/io/ByteArraySource.java index d5eee0b07d..76a3842ce7 100644 --- a/tika-core/src/main/java/org/apache/tika/io/ByteArraySource.java +++ b/tika-core/src/main/java/org/apache/tika/io/ByteArraySource.java @@ -123,6 +123,11 @@ class ByteArraySource extends InputStream implements TikaInputSource { return spilledPath; } + @Override + public boolean hasReliableLength() { + return true; + } + @Override public long getLength() { return length; diff --git a/tika-core/src/main/java/org/apache/tika/io/CachingSource.java b/tika-core/src/main/java/org/apache/tika/io/CachingSource.java index 8b424696c7..47255ccfa4 100644 --- a/tika-core/src/main/java/org/apache/tika/io/CachingSource.java +++ b/tika-core/src/main/java/org/apache/tika/io/CachingSource.java @@ -49,6 +49,8 @@ class CachingSource extends InputStream implements TikaInputSource { // temp-file suffix for threshold spills, which precede any getPath(suffix) call private final String suffix; private long length; + // getLength() is a declared hint until a drain/spill measures it + private boolean lengthMeasured; // Passthrough mode: just a BufferedInputStream private BufferedInputStream passthroughStream; @@ -224,6 +226,7 @@ class CachingSource extends InputStream implements TikaInputSource { // Record the drained length like getPath() does (feeds SecureContentHandler's // zip-bomb ratio) length = channel.size(); + lengthMeasured = true; if (metadata != null && StringUtils.isBlank(metadata.get(HttpHeaders.CONTENT_LENGTH))) { metadata.set(HttpHeaders.CONTENT_LENGTH, Long.toString(length)); @@ -309,6 +312,7 @@ class CachingSource extends InputStream implements TikaInputSource { // The spooled size is ground truth, even when it is 0 and a // Content-Length claimed otherwise length = Files.size(spilledPath); + lengthMeasured = true; // Update metadata if not already set if (metadata != null && @@ -326,6 +330,11 @@ class CachingSource extends InputStream implements TikaInputSource { return length; } + @Override + public boolean hasReliableLength() { + return lengthMeasured; + } + // seekTo() reopens fileStream, so the registered resource must close whichever // handle is current rather than the one open at registration time. private void closeFileStream() throws IOException { diff --git a/tika-core/src/main/java/org/apache/tika/io/FileSource.java b/tika-core/src/main/java/org/apache/tika/io/FileSource.java index 257cbacbc3..a57abf7ba9 100644 --- a/tika-core/src/main/java/org/apache/tika/io/FileSource.java +++ b/tika-core/src/main/java/org/apache/tika/io/FileSource.java @@ -115,6 +115,11 @@ class FileSource extends InputStream implements TikaInputSource { return path; } + @Override + public boolean hasReliableLength() { + return true; + } + @Override public long getLength() { return length; diff --git a/tika-core/src/main/java/org/apache/tika/io/ReopenableSource.java b/tika-core/src/main/java/org/apache/tika/io/ReopenableSource.java index 332e56eba3..8cbaa66881 100644 --- a/tika-core/src/main/java/org/apache/tika/io/ReopenableSource.java +++ b/tika-core/src/main/java/org/apache/tika/io/ReopenableSource.java @@ -53,6 +53,8 @@ class ReopenableSource extends InputStream implements TikaInputSource { private final TemporaryResources tmp; private final String suffix; private long length; + // getLength() is a declared hint until a drain/spill measures it + private boolean lengthMeasured; private InputStream currentStream; // lazily opened private long position; @@ -160,6 +162,7 @@ class ReopenableSource extends InputStream implements TikaInputSource { spilledPath = p; // The spooled size is ground truth, even over a lying declared length length = Files.size(p); + lengthMeasured = true; } return spilledPath; } @@ -169,6 +172,11 @@ class ReopenableSource extends InputStream implements TikaInputSource { return length; } + @Override + public boolean hasReliableLength() { + return lengthMeasured; + } + @Override public void enableRewind(CacheMemoryBudget budget) throws IOException { if (position != 0) { @@ -291,6 +299,7 @@ class ReopenableSource extends InputStream implements TikaInputSource { retainedReservation = reservedHere; // The full read is ground truth, even over a lying declared length length = total; + lengthMeasured = true; return true; } diff --git a/tika-core/src/main/java/org/apache/tika/io/TikaInputSource.java b/tika-core/src/main/java/org/apache/tika/io/TikaInputSource.java index 9e7b4a8d30..9f3dc96146 100644 --- a/tika-core/src/main/java/org/apache/tika/io/TikaInputSource.java +++ b/tika-core/src/main/java/org/apache/tika/io/TikaInputSource.java @@ -63,6 +63,12 @@ interface TikaInputSource extends Closeable { */ long getLength(); + /** + * True when {@link #getLength()} is ground truth (file size, byte array, + * fully-drained cache) rather than a caller-declared hint, which may lie. + */ + boolean hasReliableLength(); + /** * Enables full rewind capability. * <p> diff --git a/tika-core/src/main/java/org/apache/tika/io/TikaInputStream.java b/tika-core/src/main/java/org/apache/tika/io/TikaInputStream.java index b58cd548db..ae62397b9e 100644 --- a/tika-core/src/main/java/org/apache/tika/io/TikaInputStream.java +++ b/tika-core/src/main/java/org/apache/tika/io/TikaInputStream.java @@ -100,7 +100,8 @@ public class TikaInputStream extends TaggedInputStream { return (TikaInputStream) stream; } String ext = getExtension(metadata); - TikaInputSource inputSource = new CachingSource(stream, tmp, -1, metadata, ext); + TikaInputSource inputSource = + new CachingSource(stream, tmp, declaredLength(metadata), metadata, ext); return new TikaInputStream(inputSource, tmp, ext); } @@ -123,18 +124,8 @@ public class TikaInputStream extends TaggedInputStream { throw new NullPointerException("The opener must not be null"); } String ext = getExtension(metadata); - long length = -1; - if (metadata != null) { - String cl = metadata.get(HttpHeaders.CONTENT_LENGTH); - if (cl != null) { - try { - length = Long.parseLong(cl); - } catch (NumberFormatException e) { - length = -1; - } - } - } - TikaInputSource inputSource = new ReopenableSource(opener, tmp, length, ext); + TikaInputSource inputSource = + new ReopenableSource(opener, tmp, declaredLength(metadata), ext); return new TikaInputStream(inputSource, tmp, ext); } @@ -284,6 +275,21 @@ public class TikaInputStream extends TaggedInputStream { return tis; } + private static long declaredLength(Metadata metadata) { + if (metadata == null) { + return -1; + } + String cl = metadata.get(HttpHeaders.CONTENT_LENGTH); + if (cl == null) { + return -1; + } + try { + return Long.parseLong(cl); + } catch (NumberFormatException e) { + return -1; + } + } + private static String getExtension(Metadata metadata) { if (metadata == null) { return StringUtils.EMPTY; @@ -440,6 +446,20 @@ public class TikaInputStream extends TaggedInputStream { return source != null && source.getLength() != -1; } + /** + * True when {@link #getLength()} would return a measured, ground-truth length + * (file, byte array, fully-drained cache, explicit override) without forcing a + * spool. False when the only length available is a caller-declared hint + * (Content-Length metadata, HTTP header, archive central directory), which may lie. + */ + public boolean hasReliableLength() { + if (overrideLength >= 0) { + return true; + } + TikaInputSource source = inputSource(); + return source != null && source.hasReliableLength() && source.getLength() != -1; + } + /** * The stream length. For a stream-backed instance with no declared length this * spools the entire remaining stream to a temporary file to measure it. diff --git a/tika-core/src/main/java/org/apache/tika/mime/MimeTypes.java b/tika-core/src/main/java/org/apache/tika/mime/MimeTypes.java index eb0657c3a3..d2e9f082ff 100644 --- a/tika-core/src/main/java/org/apache/tika/mime/MimeTypes.java +++ b/tika-core/src/main/java/org/apache/tika/mime/MimeTypes.java @@ -542,8 +542,9 @@ public final class MimeTypes implements Detector, Serializable { // Get type based on magic prefix if (tis != null) { int toRead = getMinLength(); - // hasLength() is non-forcing; getLength() would only spool when unknown - if (tis.hasLength()) { + // Only a measured length may shrink the read: a lying declared + // Content-Length would silently truncate the magic prefix. + if (tis.hasReliableLength()) { long known = tis.getLength() - tis.getPosition(); if (known >= 0 && known < toRead) { toRead = (int) known; diff --git a/tika-core/src/main/java/org/apache/tika/mime/Patterns.java b/tika-core/src/main/java/org/apache/tika/mime/Patterns.java index 8cc8ce9728..f5f8c234e0 100644 --- a/tika-core/src/main/java/org/apache/tika/mime/Patterns.java +++ b/tika-core/src/main/java/org/apache/tika/mime/Patterns.java @@ -54,9 +54,10 @@ class Patterns implements Serializable { /** * Compiled forms of {@link #globs}' keys. Matching recompiled every glob regex * per lookup before; for names that miss the name/extension indexes that was - * a Pattern.compile per glob per call. + * a Pattern.compile per glob per call. Transient with a lazy rebuild so a + * serialized form from a build without this field still deserializes. */ - private final Map<String, Pattern> compiledGlobs = new HashMap<>(); + private transient Map<String, Pattern> compiledGlobs = new HashMap<>(); private int minExtensionLength = Integer.MAX_VALUE; private int maxExtensionLength = 0; @@ -120,11 +121,23 @@ class Patterns implements Serializable { } } + private Map<String, Pattern> compiledGlobs() { + Map<String, Pattern> compiled = compiledGlobs; + if (compiled == null) { + compiled = new HashMap<>(); + for (String glob : globs.keySet()) { + compiled.put(glob, Pattern.compile(glob)); + } + compiledGlobs = compiled; + } + return compiled; + } + private void addGlob(String glob, MimeType type) throws MimeTypeException { MimeType previous = globs.get(glob); if (previous == null || registry.isSpecializationOf(previous.getType(), type.getType())) { globs.put(glob, type); - compiledGlobs.put(glob, Pattern.compile(glob)); + compiledGlobs().put(glob, Pattern.compile(glob)); } else if (previous == type || registry.isSpecializationOf(type.getType(), previous.getType())) { // do nothing @@ -167,8 +180,9 @@ class Patterns implements Serializable { } // And finally, try complex regexp matching + Map<String, Pattern> compiled = compiledGlobs(); for (Map.Entry<String, MimeType> entry : globs.entrySet()) { - Pattern glob = compiledGlobs.get(entry.getKey()); + Pattern glob = compiled.get(entry.getKey()); boolean matched = glob != null ? glob.matcher(name).matches() : name.matches(entry.getKey()); if (matched) { diff --git a/tika-core/src/main/java/org/apache/tika/parser/CompositeParser.java b/tika-core/src/main/java/org/apache/tika/parser/CompositeParser.java index 4e97d96fc0..7d6a46e0b3 100644 --- a/tika-core/src/main/java/org/apache/tika/parser/CompositeParser.java +++ b/tika-core/src/main/java/org/apache/tika/parser/CompositeParser.java @@ -31,6 +31,7 @@ import org.xml.sax.ContentHandler; import org.xml.sax.SAXException; import org.apache.tika.config.ParseTimeout; +import org.apache.tika.config.TransientParseState; import org.apache.tika.exception.EmbeddedLimitReachedException; import org.apache.tika.exception.TikaException; import org.apache.tika.exception.WriteLimitReachedException; @@ -247,16 +248,19 @@ public class CompositeParser implements Parser { } /** - * Per-parse cache of built parser maps, stored in the ParseContext so every - * embedded document in a parse reuses the container's map instead of - * rebuilding it (a full walk of every parser's supported types). Keyed by - * parser instance because nested composites share one context. The map is - * built once per (parser, context); a context entry that would change a - * parser's supported types mid-parse is not picked up until the next parse. + * Cache of built parser maps, stored in the ParseContext so every embedded + * document in a parse reuses the container's map instead of rebuilding it + * (a full walk of every parser's supported types). Keyed by parser instance + * because nested composites share one context. The map is built once per + * (parser, context) and lives as long as the context: a context entry that + * would change a parser's supported types is not picked up mid-parse, nor on + * later parses that reuse the same ParseContext instance. */ - private static final class ParserMapCache { + private static final class ParserMapCache implements TransientParseState { + // synchronized: contexts are occasionally shared across threads, and a + // concurrent put into a bare IdentityHashMap can corrupt it private final Map<CompositeParser, Map<MediaType, Parser>> maps = - new IdentityHashMap<>(); + Collections.synchronizedMap(new IdentityHashMap<>()); } private Map<MediaType, Parser> getParsersCached(ParseContext context) { diff --git a/tika-core/src/main/java/org/apache/tika/parser/ParseRecord.java b/tika-core/src/main/java/org/apache/tika/parser/ParseRecord.java index 88100d532e..84ce139b2d 100644 --- a/tika-core/src/main/java/org/apache/tika/parser/ParseRecord.java +++ b/tika-core/src/main/java/org/apache/tika/parser/ParseRecord.java @@ -23,6 +23,7 @@ import java.util.Set; import org.apache.tika.config.EmbeddedLimits; import org.apache.tika.config.TimeoutLimits; +import org.apache.tika.config.TransientParseState; import org.apache.tika.metadata.Metadata; /** @@ -34,7 +35,7 @@ import org.apache.tika.metadata.Metadata; * which can be configured via {@link #setMaxEmbeddedDepth(int)} and * {@link #setMaxEmbeddedCount(int)}. */ -public class ParseRecord { +public class ParseRecord implements TransientParseState { //hard limits so that specially crafted files //don't cause an OOM diff --git a/tika-core/src/main/java/org/apache/tika/sax/FastMarkdownTextRenderer.java b/tika-core/src/main/java/org/apache/tika/sax/FastMarkdownTextRenderer.java index ad68db7261..e821ff35c9 100644 --- a/tika-core/src/main/java/org/apache/tika/sax/FastMarkdownTextRenderer.java +++ b/tika-core/src/main/java/org/apache/tika/sax/FastMarkdownTextRenderer.java @@ -40,10 +40,11 @@ import org.commonmark.text.AsciiMatcher; * {@code raw()} call. * <p> * The escaping semantics replicate {@code CoreMarkdownNodeRenderer#visit(Text)} for - * commonmark 0.27.x exactly: the line-start disambiguation cases, the heading escape - * variant, the {@code !}-before-link case, and the {@code \n} numeric reference. The - * differential test in {@code ToMarkdownContentHandlerTest} and the corpus differential - * guard the equivalence. + * the commonmark version pinned in tika-parent exactly: the line-start disambiguation + * cases, the heading escape variant, the {@code !}-before-link case, and the + * {@code \n} numeric reference. {@code FastMarkdownTextRendererTest} renders ASTs + * through this renderer and the stock one and requires byte-identical output; a + * commonmark upgrade that changes escaping fails there and must be re-synced here. * <p> * Text inside a table cell renders through the stock per-char path: the tables extension * pushes a raw-escape for {@code |} onto the writer there, and hand-emitted escapes would diff --git a/tika-core/src/main/java/org/apache/tika/sax/ToMarkdownContentHandler.java b/tika-core/src/main/java/org/apache/tika/sax/ToMarkdownContentHandler.java index a6614bffae..6a958b1a7d 100644 --- a/tika-core/src/main/java/org/apache/tika/sax/ToMarkdownContentHandler.java +++ b/tika-core/src/main/java/org/apache/tika/sax/ToMarkdownContentHandler.java @@ -90,7 +90,8 @@ public class ToMarkdownContentHandler extends DefaultHandler { private final Writer writer; private final MarkdownRenderer renderer = MarkdownRenderer.builder().extensions(EXTENSIONS) - // registered first, so it wins Text rendering from the core renderer + // registered before the core fallback, which is the only other + // renderer claiming Text, so first-wins registration picks this one .nodeRendererFactory(FastMarkdownTextRenderer.FACTORY).build(); private final Document document = new Document(); diff --git a/tika-core/src/test/java/org/apache/tika/io/DeclaredLengthTest.java b/tika-core/src/test/java/org/apache/tika/io/DeclaredLengthTest.java new file mode 100644 index 0000000000..1a9a6f5c01 --- /dev/null +++ b/tika-core/src/test/java/org/apache/tika/io/DeclaredLengthTest.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.io; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.ByteArrayInputStream; +import java.nio.file.Path; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import org.apache.tika.metadata.HttpHeaders; +import org.apache.tika.metadata.Metadata; + +/** + * Declared-vs-measured length semantics of {@link TikaInputStream}: a + * Content-Length hint is served without a spool but is never "reliable"; + * a measured length (byte array, file, spool) is. + */ +public class DeclaredLengthTest { + + @TempDir + Path tmpDir; + + @Test + public void testStreamHonorsDeclaredLength() throws Exception { + byte[] data = "0123456789".getBytes(UTF_8); + Metadata metadata = new Metadata(); + metadata.set(HttpHeaders.CONTENT_LENGTH, "7"); + try (TikaInputStream tis = TikaInputStream.get( + new ByteArrayInputStream(data), new TemporaryResources(), metadata)) { + assertTrue(tis.hasLength()); + assertFalse(tis.hasReliableLength(), "declared length is a hint, not ground truth"); + // declared value served without forcing a spool + assertEquals(7, tis.getLength()); + assertFalse(tis.hasFile()); + } + } + + @Test + public void testSpoolMeasuresOverDeclaredLength() throws Exception { + byte[] data = "0123456789".getBytes(UTF_8); + Metadata metadata = new Metadata(); + metadata.set(HttpHeaders.CONTENT_LENGTH, "7"); + try (TikaInputStream tis = TikaInputStream.get( + new ByteArrayInputStream(data), new TemporaryResources(), metadata)) { + tis.getPath(); + assertTrue(tis.hasReliableLength()); + assertEquals(data.length, tis.getLength(), "spooled size wins over the lying hint"); + } + } + + @Test + public void testStreamWithNoDeclaredLengthStillSpoolsToMeasure() throws Exception { + byte[] data = "0123456789".getBytes(UTF_8); + try (TikaInputStream tis = TikaInputStream.get( + new ByteArrayInputStream(data), new TemporaryResources(), new Metadata())) { + assertFalse(tis.hasLength()); + assertEquals(data.length, tis.getLength()); + assertTrue(tis.hasReliableLength()); + } + } + + @Test + public void testByteArrayAndFileAreReliable() throws Exception { + try (TikaInputStream tis = TikaInputStream.get("abc".getBytes(UTF_8))) { + assertTrue(tis.hasReliableLength()); + assertEquals(3, tis.getLength()); + } + Path f = tmpDir.resolve("len.bin"); + java.nio.file.Files.write(f, "abcd".getBytes(UTF_8)); + try (TikaInputStream tis = TikaInputStream.get(f)) { + assertTrue(tis.hasReliableLength()); + assertEquals(4, tis.getLength()); + } + } + + @Test + public void testReopenableDeclaredLengthNotReliableUntilSpool() throws Exception { + byte[] data = "0123456789".getBytes(UTF_8); + Metadata metadata = new Metadata(); + metadata.set(HttpHeaders.CONTENT_LENGTH, "3"); + try (TikaInputStream tis = TikaInputStream.get( + () -> new ByteArrayInputStream(data), new TemporaryResources(), metadata)) { + assertTrue(tis.hasLength()); + assertFalse(tis.hasReliableLength()); + assertEquals(3, tis.getLength()); + tis.getPath(); + assertTrue(tis.hasReliableLength()); + assertEquals(data.length, tis.getLength()); + } + } +} diff --git a/tika-core/src/test/java/org/apache/tika/mime/MimeDetectionTest.java b/tika-core/src/test/java/org/apache/tika/mime/MimeDetectionTest.java index 5748f185c1..a3794f3184 100644 --- a/tika-core/src/test/java/org/apache/tika/mime/MimeDetectionTest.java +++ b/tika-core/src/test/java/org/apache/tika/mime/MimeDetectionTest.java @@ -306,4 +306,18 @@ public class MimeDetectionTest { public void testPNGWithSomeEmlHeaders() throws IOException { testFile("image/png", "test-pngNotEml.bin"); } + + @Test + public void testLyingDeclaredLengthDoesNotTruncateMagic() throws Exception { + byte[] pdf = "%PDF-1.4\nsome pdf content".getBytes(UTF_8); + Metadata metadata = new Metadata(); + // an under-declared Content-Length must not shrink the magic read + metadata.set(HttpHeaders.CONTENT_LENGTH, "2"); + try (TikaInputStream tis = TikaInputStream.get( + new java.io.ByteArrayInputStream(pdf), + new org.apache.tika.io.TemporaryResources(), metadata)) { + assertEquals(MediaType.application("pdf"), + MIME_TYPES.detect(tis, metadata, new ParseContext())); + } + } } diff --git a/tika-core/src/test/java/org/apache/tika/mime/PatternsTest.java b/tika-core/src/test/java/org/apache/tika/mime/PatternsTest.java index 25721b15fd..9d3c7f3aa9 100644 --- a/tika-core/src/test/java/org/apache/tika/mime/PatternsTest.java +++ b/tika-core/src/test/java/org/apache/tika/mime/PatternsTest.java @@ -97,4 +97,20 @@ public class PatternsTest { assertTrue(extensions.contains(".jpeg")); } + + @Test + public void testSerializationRoundTrip() throws Exception { + patterns.add("*ile*.txt", text); + java.io.ByteArrayOutputStream bos = new java.io.ByteArrayOutputStream(); + try (java.io.ObjectOutputStream oos = new java.io.ObjectOutputStream(bos)) { + oos.writeObject(patterns); + } + Patterns copy; + try (java.io.ObjectInputStream ois = new java.io.ObjectInputStream( + new java.io.ByteArrayInputStream(bos.toByteArray()))) { + copy = (Patterns) ois.readObject(); + } + // compiledGlobs is transient; the glob path must rebuild it lazily + assertEquals(text, copy.matches("file7.txt")); + } } diff --git a/tika-core/src/test/java/org/apache/tika/sax/FastMarkdownTextRendererTest.java b/tika-core/src/test/java/org/apache/tika/sax/FastMarkdownTextRendererTest.java new file mode 100644 index 0000000000..9122e84269 --- /dev/null +++ b/tika-core/src/test/java/org/apache/tika/sax/FastMarkdownTextRendererTest.java @@ -0,0 +1,201 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.sax; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.Arrays; +import java.util.List; +import java.util.Random; + +import org.commonmark.Extension; +import org.commonmark.ext.gfm.strikethrough.StrikethroughExtension; +import org.commonmark.ext.gfm.tables.TablesExtension; +import org.commonmark.node.Document; +import org.commonmark.node.Heading; +import org.commonmark.node.Link; +import org.commonmark.node.Node; +import org.commonmark.node.Paragraph; +import org.commonmark.node.Text; +import org.commonmark.parser.Parser; +import org.commonmark.renderer.markdown.MarkdownRenderer; +import org.junit.jupiter.api.Test; + +/** + * Differential guard for {@link FastMarkdownTextRenderer}: every AST must render + * byte-identically through the fast renderer and through commonmark's stock + * renderer. If a commonmark upgrade changes {@code CoreMarkdownNodeRenderer}'s + * escaping, these tests fail and the fast renderer must be re-synced. + */ +public class FastMarkdownTextRendererTest { + + private static final List<Extension> EXTENSIONS = + Arrays.asList(TablesExtension.create(), StrikethroughExtension.create()); + + private static final MarkdownRenderer FAST = MarkdownRenderer.builder() + .extensions(EXTENSIONS) + .nodeRendererFactory(FastMarkdownTextRenderer.FACTORY).build(); + + private static final MarkdownRenderer STOCK = + MarkdownRenderer.builder().extensions(EXTENSIONS).build(); + + private static final Parser PARSER = Parser.builder().extensions(EXTENSIONS).build(); + + private static void assertSameRendering(Node document, String label) { + assertEquals(STOCK.render(document), FAST.render(document), label); + } + + private static void assertSameRendering(String markdown) { + assertSameRendering(PARSER.parse(markdown), "diverged on: " + markdown); + } + + private static void assertSameRendering(Node document) { + assertSameRendering(document, "diverged on hand-built AST"); + } + + private static Node paragraphs(String... literals) { + Document doc = new Document(); + Paragraph p = new Paragraph(); + for (String literal : literals) { + p.appendChild(new Text(literal)); + } + doc.appendChild(p); + return doc; + } + + @Test + public void testLineStartEscapes() { + // Literals whose first char would parse as block structure at line start + for (String literal : new String[]{ + "- not a list", "-not a list", "-", "+ plus", "* star", + "# not a heading", "#hash", "#", + "= not setext", "=", + "1. not ordered", "12. also not", "123456789. limit", "1) paren", + "1234567890. ten digits is not a marker", "12x. no", + "> not a quote", + " leading space", "\tleading tab", " two spaces"}) { + assertSameRendering(paragraphs(literal), "diverged on literal: " + literal); + } + } + + @Test + public void testSecondTextNodeNotAtLineStart() { + // The line-start branches must not fire mid-line + assertSameRendering(paragraphs("before ", "- mid", "# mid", "= mid", "12. mid")); + } + + @Test + public void testEscapableCharacters() { + for (String literal : new String[]{ + "a[b]c", "a<b>c", "a`b`c", "a*b*c", "a_b_c", "a&b;c", "a\\b", + "pipe | in text", "tilde ~~x~~", "", "plain text, no specials.", + "text\nwith\nnewlines", "\n", "trailing newline\n", "*", "\\"}) { + assertSameRendering(paragraphs(literal), "diverged on literal: " + literal); + } + } + + @Test + public void testBangBeforeLink() { + Document doc = new Document(); + Paragraph p = new Paragraph(); + p.appendChild(new Text("see!")); + Link link = new Link("http://example.com";, null); + link.appendChild(new Text("here")); + p.appendChild(link); + doc.appendChild(p); + assertSameRendering(doc, "bang before link"); + + // bang NOT followed by a link needs no escape + assertSameRendering(paragraphs("no link here!")); + } + + @Test + public void testHeadingEscapeSet() { + for (String literal : new String[]{"plain", "with # hash", "with\nnewline", "a`b"}) { + Document doc = new Document(); + Heading h = new Heading(); + h.setLevel(2); + h.appendChild(new Text(literal)); + doc.appendChild(h); + assertSameRendering(doc, "heading literal: " + literal); + } + } + + @Test + public void testParsedDocuments() { + // Round-trips through the parser: escaped specials come back as raw Text + for (String markdown : new String[]{ + "\\- not a list\n", + "\\# not a heading\n", + "12\\. not ordered\n", + "para one\n\npara two with \\*stars\\* and \\[brackets\\]\n", + "**bold** and _em_ and ~~strike~~\n", + "a paragraph\nwith a soft break\n", + "| a | b\\|c |\n|---|---|\n| d | *e* |\n", + "# heading `code` and *em*\n", + "> quoted \\> text\n", + "[link](http://example.com \"ti\\\"tle\") and \n"}) { + assertSameRendering(markdown); + } + } + + @Test + public void testRandomizedDifferential() { + char[] alphabet = ("abc XYZ \t\n-#=!*_[]<>&`\\|~.)0129" + + "é中😀").toCharArray(); + for (long seed = 0; seed < 50; seed++) { + Random random = new Random(seed); + StringBuilder sb = new StringBuilder(); + int len = 1 + random.nextInt(80); + for (int i = 0; i < len; i++) { + sb.append(alphabet[random.nextInt(alphabet.length)]); + } + String literal = sb.toString(); + // the alphabet splits astral chars into chars; skip broken pairs + if (!isWellFormed(literal)) { + continue; + } + assertSameRendering(paragraphs(literal), "seed " + seed + " literal: " + literal); + } + } + + private static boolean isWellFormed(String s) { + for (int i = 0; i < s.length(); i++) { + char c = s.charAt(i); + if (Character.isHighSurrogate(c)) { + if (i + 1 >= s.length() || !Character.isLowSurrogate(s.charAt(i + 1))) { + return false; + } + i++; + } else if (Character.isLowSurrogate(c)) { + return false; + } + } + return true; + } + + @Test + public void testTableCellKeepsStockPath() { + // '|' inside a cell must come out escaped once, not twice + Node doc = PARSER.parse("| a\\|b | *c* |\n|---|---|\n| \\|start | end\\| |\n"); + String stock = STOCK.render(doc); + String fast = FAST.render(doc); + assertEquals(stock, fast, "table cell rendering"); + assertTrue(fast.contains("a\\|b"), "single-escaped pipe expected: " + fast); + } +} diff --git a/tika-core/src/test/java/org/apache/tika/sax/ToMarkdownContentHandlerTest.java b/tika-core/src/test/java/org/apache/tika/sax/ToMarkdownContentHandlerTest.java index 26c8a3d6e5..2b060e504d 100644 --- a/tika-core/src/test/java/org/apache/tika/sax/ToMarkdownContentHandlerTest.java +++ b/tika-core/src/test/java/org/apache/tika/sax/ToMarkdownContentHandlerTest.java @@ -1520,4 +1520,22 @@ public class ToMarkdownContentHandlerTest { handler.writePartialContentIfUnfinished(); assertEquals(afterEnd, writer.toString(), "writePartial after endDocument must be a no-op"); } + + @Test + public void testRenderBufferBoundaries() throws Exception { + // Spans sized around the internal 8 KB render buffer: exact fit, one-over, + // buffer-sized single write (direct path), and a multi-drain run. + for (int size : new int[]{8191, 8192, 8193, 16384, 40000}) { + StringWriter writer = new StringWriter(); + ToMarkdownContentHandler handler = new ToMarkdownContentHandler(writer); + String run = "a".repeat(size); + handler.startDocument(); + startElement(handler, "p"); + chars(handler, run); + endElement(handler, "p"); + handler.endDocument(); + assertTrue(writer.toString().contains(run), + "clean " + size + "-char run must survive buffering intact"); + } + } } diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/WordExtractor.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/WordExtractor.java index 25ad90d49f..18efa9bcd2 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/WordExtractor.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/WordExtractor.java @@ -106,10 +106,6 @@ public class WordExtractor extends AbstractPOIFSExtractor { return count; } - /** - * Given a style name, return what tag should be used, and - * what style should be applied to it. - */ // matches the old regex [\r\n\s]+ ( \s == [ \t\n\x0B\f\r] ), without the // full replaceAll copy it used to make per paragraph private static boolean isBlankParagraph(String text) { diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/main/java/org/apache/tika/parser/csv/CSVSniffer.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/main/java/org/apache/tika/parser/csv/CSVSniffer.java index 164eacfd15..0ef0cd53cf 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/main/java/org/apache/tika/parser/csv/CSVSniffer.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/main/java/org/apache/tika/parser/csv/CSVSniffer.java @@ -21,6 +21,7 @@ import java.io.EOFException; import java.io.IOException; import java.io.Reader; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -58,13 +59,18 @@ class CSVSniffer { } // Every snifflet examines the same window, so read it once into a buffer // instead of once per delimiter through a mark/reset + pushback stack. - char[] buf = new char[markLimit]; + // Grown on demand: markLimit is user-configurable and most inputs are + // far smaller, so an eager char[markLimit] would be waste per parse. + char[] buf = new char[Math.min(markLimit, 8192)]; reader.mark(markLimit); int len = 0; try { while (len < markLimit) { - int n = reader.read(buf, len, markLimit - len); - if (n == -1) { + if (len == buf.length) { + buf = Arrays.copyOf(buf, (int) Math.min((long) buf.length * 2, markLimit)); + } + int n = reader.read(buf, len, buf.length - len); + if (n <= 0) { break; } len += n; diff --git a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/ContentBytesConfig.java b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/ContentBytesConfig.java index 5d00bbd6ff..a8f8d2c5e3 100644 --- a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/ContentBytesConfig.java +++ b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/ContentBytesConfig.java @@ -25,7 +25,10 @@ import org.apache.tika.annotation.TikaComponent; * {@code TIKA_CONTENT} into raw UTF-8 bytes on the {@code EmitData}, which travel * as binary over the IPC instead of a Smile-encoded string. A caller who sets this * must read {@code EmitData.getContentBytes()}; the metadata no longer carries the - * content. tika-server sets it for the raw-output endpoints. + * content. tika-server sets it for the raw-output endpoints. Results routed to a + * regular Emitter get the content restored into the metadata first + * ({@code EmitDataImpl#restoreContentFromBytes()}), so the move only sticks for + * consumers on the passback path. */ @TikaComponent(name = "content-bytes-config") public class ContentBytesConfig implements Serializable { diff --git a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/async/AsyncProcessor.java b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/async/AsyncProcessor.java index 70849fa8c7..1a6b472bbc 100644 --- a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/async/AsyncProcessor.java +++ b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/async/AsyncProcessor.java @@ -54,6 +54,7 @@ import org.apache.tika.pipes.core.PipesResults; import org.apache.tika.pipes.core.RestartReason; import org.apache.tika.pipes.core.ServerManager; import org.apache.tika.pipes.core.SharedServerManager; +import org.apache.tika.pipes.core.emitter.EmitDataImpl; import org.apache.tika.pipes.core.emitter.EmitterManager; import org.apache.tika.pipes.core.reporter.ReporterManager; import org.apache.tika.plugins.TikaPluginManager; @@ -484,6 +485,11 @@ public class AsyncProcessor implements Closeable { long offerStart = System.currentTimeMillis(); if (shouldEmit(result)) { + if (result.emitData() instanceof EmitDataImpl emitDataImpl) { + // a client-side Emitter reads only the metadata list; + // undo any content-bytes move so content is not lost + emitDataImpl.restoreContentFromBytes(); + } LOG.trace("adding result to emitter queue: " + result.emitData()); boolean offered = emitDataTupleQueue.offer( new EmitDataPair(t.getEmitKey().getEmitterId(), result.emitData()), MAX_OFFER_WAIT_MS, diff --git a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/emitter/EmitDataImpl.java b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/emitter/EmitDataImpl.java index ebe53de666..d7683b5e76 100644 --- a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/emitter/EmitDataImpl.java +++ b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/emitter/EmitDataImpl.java @@ -16,9 +16,11 @@ */ package org.apache.tika.pipes.core.emitter; +import java.nio.charset.StandardCharsets; import java.util.List; import org.apache.tika.metadata.Metadata; +import org.apache.tika.metadata.TikaCoreProperties; import org.apache.tika.parser.ParseContext; import org.apache.tika.pipes.api.emitter.EmitData; import org.apache.tika.utils.StringUtils; @@ -66,6 +68,24 @@ public class EmitDataImpl implements EmitData { this.contentBytes = contentBytes; } + /** + * Inverse of the content-bytes move: puts the content back in + * {@code TIKA_CONTENT} for consumers that only read the metadata list + * (e.g. a regular Emitter). No-op when there are no content bytes or + * the metadata already carries content. + */ + public void restoreContentFromBytes() { + if (contentBytes == null || metadataList == null || metadataList.isEmpty()) { + return; + } + Metadata m = metadataList.get(0); + if (m.get(TikaCoreProperties.TIKA_CONTENT) == null) { + m.set(TikaCoreProperties.TIKA_CONTENT, + new String(contentBytes, StandardCharsets.UTF_8)); + } + contentBytes = null; + } + public long getEstimatedSizeBytes() { long sz = estimateSizeInBytes(getEmitKey(), getMetadataList(), containerStackTrace); if (contentBytes != null) { diff --git a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/EmitHandler.java b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/EmitHandler.java index 57bac19e86..7a73feb4a7 100644 --- a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/EmitHandler.java +++ b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/EmitHandler.java @@ -172,11 +172,12 @@ class EmitHandler { return; } Metadata m = metadataList.get(0); - String content = m.get(TikaCoreProperties.TIKA_CONTENT); - if (content == null) { + String[] content = m.getValues(TikaCoreProperties.TIKA_CONTENT); + // remove() drops every value; only move when that loses nothing + if (content.length != 1) { return; } - emitData.setContentBytes(content.getBytes(StandardCharsets.UTF_8)); + emitData.setContentBytes(content[0].getBytes(StandardCharsets.UTF_8)); m.remove(TikaCoreProperties.TIKA_CONTENT); } diff --git a/tika-serialization/src/main/java/org/apache/tika/serialization/serdes/ParseContextSerializer.java b/tika-serialization/src/main/java/org/apache/tika/serialization/serdes/ParseContextSerializer.java index 592aaed460..658dcaaf51 100644 --- a/tika-serialization/src/main/java/org/apache/tika/serialization/serdes/ParseContextSerializer.java +++ b/tika-serialization/src/main/java/org/apache/tika/serialization/serdes/ParseContextSerializer.java @@ -29,6 +29,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializerProvider; import org.apache.tika.config.JsonConfig; +import org.apache.tika.config.TransientParseState; import org.apache.tika.config.loader.TikaObjectMapperFactory; import org.apache.tika.parser.ParseContext; import org.apache.tika.serialization.ComponentNameResolver; @@ -79,6 +80,13 @@ public class ParseContextSerializer extends JsonSerializer<ParseContext> { continue; } + // Per-parse runtime state (ParseRecord, ParseTimeout, parser-map cache) + // is never configuration; a context that has been through a parse must + // still serialize. + if (value instanceof TransientParseState) { + continue; + } + // Find the friendly component name — all serializable components must be registered String keyName = ComponentNameResolver.getFriendlyName(value.getClass()); if (keyName == null) { diff --git a/tika-serialization/src/test/java/org/apache/tika/serialization/TestParseContextSerialization.java b/tika-serialization/src/test/java/org/apache/tika/serialization/TestParseContextSerialization.java index d93e5924b1..5a5526b941 100644 --- a/tika-serialization/src/test/java/org/apache/tika/serialization/TestParseContextSerialization.java +++ b/tika-serialization/src/test/java/org/apache/tika/serialization/TestParseContextSerialization.java @@ -591,4 +591,19 @@ public class TestParseContextSerialization { "the message must name the offending key: " + e.getMessage()); } + + @Test + public void testTransientParseStateSkipped() throws Exception { + // A context that has been through a parse carries per-parse runtime state + // (ParseRecord, ParseTimeout); serialization must skip it, not throw. + ParseContext pc = new ParseContext(); + pc.set(org.apache.tika.parser.ParseRecord.class, + org.apache.tika.parser.ParseRecord.newInstance(pc)); + org.apache.tika.config.ParseTimeout.getOrCreate(pc); + + String json = serializeParseContext(pc); + ObjectMapper mapper = createMapper(); + JsonNode root = mapper.readTree(json); + assertEquals(0, root.size(), "transient parse state must not serialize: " + json); + } } diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaResource.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaResource.java index ab48d46e85..b955bde571 100644 --- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaResource.java +++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaResource.java @@ -697,13 +697,13 @@ public class TikaResource { // ==================== Internal methods ==================== - /** - * Produces raw streaming output (text, html, xml, md) using pipes-based parsing. - */ /** Per-request resource-layer latency; joins the pipes lines by adjacency at c=1. */ private static final org.slf4j.Logger TIMING_LOG = org.slf4j.LoggerFactory.getLogger("org.apache.tika.pipes.timing.resource"); + /** + * Produces raw streaming output (text, html, xml, md) using pipes-based parsing. + */ private Response produceRawOutput(TikaInputStream tis, Metadata metadata, MultivaluedMap<String, String> httpHeaders, String handlerTypeName) throws IOException { diff --git a/tika-server/tika-server-core/src/main/resources/log4j2.xml b/tika-server/tika-server-core/src/main/resources/log4j2.xml index c88e66e99e..d22d891023 100644 --- a/tika-server/tika-server-core/src/main/resources/log4j2.xml +++ b/tika-server/tika-server-core/src/main/resources/log4j2.xml @@ -25,6 +25,8 @@ </Console> </Appenders> <Loggers> + <!-- opt-in per-request timing lines (TIKA-4868): raise to info to enable --> + <Logger name="org.apache.tika.pipes.timing" level="warn"/> <Root level="info"> <AppenderRef ref="Console"/> </Root> diff --git a/tika-server/tika-server-standard/src/main/resources/log4j2.xml b/tika-server/tika-server-standard/src/main/resources/log4j2.xml index c88e66e99e..d22d891023 100644 --- a/tika-server/tika-server-standard/src/main/resources/log4j2.xml +++ b/tika-server/tika-server-standard/src/main/resources/log4j2.xml @@ -25,6 +25,8 @@ </Console> </Appenders> <Loggers> + <!-- opt-in per-request timing lines (TIKA-4868): raise to info to enable --> + <Logger name="org.apache.tika.pipes.timing" level="warn"/> <Root level="info"> <AppenderRef ref="Console"/> </Root>
