This is an automated email from the ASF dual-hosted git repository. tballison pushed a commit to branch TIKA-4868-performance-improvements in repository https://gitbox.apache.org/repos/asf/tika.git
commit be1b9cd877dcde4dc4fb9d41cb64661fbb2556d4 Author: tallison <[email protected]> AuthorDate: Wed Sep 2 08:07:41 2026 -0400 TIKA-4868: gate rfc822 magic, cache the parser map per parse --- CHANGES.txt | 12 +++++++++ .../java/org/apache/tika/detect/MagicDetector.java | 31 +++++++++++++++++----- .../org/apache/tika/parser/CompositeParser.java | 30 ++++++++++++++++++++- .../org/apache/tika/mime/tika-mimetypes.xml | 5 ++++ .../tika/server/core/resource/TikaResource.java | 31 +++++++++++++++++++++- 5 files changed, 100 insertions(+), 9 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index dee363d60b..0e6e1ad587 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,5 +1,17 @@ Release 4.1.0 - unreleased + * More detection/dispatch savings: the message/rfc822 priority-45 magic + is gated behind a ':' scan of the first 30 bytes (every match provably + has one, so results are unchanged and non-mail text skips its ~40 + clauses); CompositeParser caches the built type->parser map in the + ParseContext, so embedded documents reuse the container's map instead + of rebuilding it per document. Plain-text detection drops from 334us + (4.1.0-dev ran it twice) to 52us per document; eml requests ~-45%, + ppt ~-35% in tika-server. MagicDetector also precomputes a 256-entry + first-byte table per pattern (mask and case fold become one array + load per scanned position). Adds an opt-in RESOURCE_TIMING log on + org.apache.tika.pipes.timing.resource (TIKA-4868). + * WordExtractor (.doc) cleans each character run in one pass instead of four chained replace/replaceAll copies, and tests paragraph blankness without a regex replaceAll; ToMarkdownContentHandler collapses line diff --git a/tika-core/src/main/java/org/apache/tika/detect/MagicDetector.java b/tika-core/src/main/java/org/apache/tika/detect/MagicDetector.java index 839b722c3a..d9d6b2ad2d 100644 --- a/tika-core/src/main/java/org/apache/tika/detect/MagicDetector.java +++ b/tika-core/src/main/java/org/apache/tika/detect/MagicDetector.java @@ -468,6 +468,23 @@ public class MagicDetector implements Detector { * @param endOffset the last position in the buffer to start matching (inclusive) * @return true if a match is found, false otherwise */ + // Which raw first bytes can begin a match; encodes mask[0] and the case fold so the + // scan loop is a single table load. Built lazily, idempotent under racing builds. + private transient volatile boolean[] firstByteMatches; + + private boolean[] buildFirstByteTable() { + boolean[] table = new boolean[256]; + int first = pattern[0]; + for (int b = 0; b < 256; b++) { + int masked = ((byte) b) & mask[0]; + if (this.isStringIgnoreCase) { + masked = Character.toLowerCase(masked); + } + table[b] = (masked == first); + } + return table; + } + private boolean matchesBuffer(byte[] buffer, int startOffset, int endOffset) { if (this.isRegex) { int bufferLen = Math.min(buffer.length - startOffset, length + (endOffset - startOffset)); @@ -497,17 +514,17 @@ public class MagicDetector implements Detector { // degenerate empty pattern: preserves the old loop's outcome return startOffset <= endOffset && startOffset <= buffer.length; } - int first = pattern[0]; - byte firstMask = mask[0]; + // one array load per scanned position instead of mask + case-fold + compare + boolean[] firstMatch = firstByteMatches; + if (firstMatch == null) { + firstMatch = buildFirstByteTable(); + firstByteMatches = firstMatch; + } for (int i = startOffset; i <= endOffset; i++) { if (i + length > buffer.length) { break; } - int masked0 = buffer[i] & firstMask; - if (this.isStringIgnoreCase) { - masked0 = Character.toLowerCase(masked0); - } - if (masked0 != first) { + if (!firstMatch[buffer[i] & 0xFF]) { continue; } boolean match = true; diff --git a/tika-core/src/main/java/org/apache/tika/parser/CompositeParser.java b/tika-core/src/main/java/org/apache/tika/parser/CompositeParser.java index fe198a75b5..4e97d96fc0 100644 --- a/tika-core/src/main/java/org/apache/tika/parser/CompositeParser.java +++ b/tika-core/src/main/java/org/apache/tika/parser/CompositeParser.java @@ -22,6 +22,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; +import java.util.IdentityHashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -245,8 +246,35 @@ public class CompositeParser implements Parser { return getParser(metadata, new ParseContext()); } + /** + * Per-parse cache of built parser maps, stored in the ParseContext so every + * embedded document in a parse reuses the container's map instead of + * rebuilding it (a full walk of every parser's supported types). Keyed by + * parser instance because nested composites share one context. The map is + * built once per (parser, context); a context entry that would change a + * parser's supported types mid-parse is not picked up until the next parse. + */ + private static final class ParserMapCache { + private final Map<CompositeParser, Map<MediaType, Parser>> maps = + new IdentityHashMap<>(); + } + + private Map<MediaType, Parser> getParsersCached(ParseContext context) { + ParserMapCache cache = context.get(ParserMapCache.class); + if (cache == null) { + cache = new ParserMapCache(); + context.set(ParserMapCache.class, cache); + } + Map<MediaType, Parser> map = cache.maps.get(this); + if (map == null) { + map = getParsers(context); + cache.maps.put(this, map); + } + return map; + } + protected Parser getParser(Metadata metadata, ParseContext context) { - Map<MediaType, Parser> map = getParsers(context); + Map<MediaType, Parser> map = getParsersCached(context); //check for parser override first String contentTypeString = metadata.get(TikaCoreProperties.CONTENT_TYPE_PARSER_OVERRIDE); if (contentTypeString == null) { diff --git a/tika-core/src/main/resources/org/apache/tika/mime/tika-mimetypes.xml b/tika-core/src/main/resources/org/apache/tika/mime/tika-mimetypes.xml index 9cb4a899bc..bd98b2ae40 100644 --- a/tika-core/src/main/resources/org/apache/tika/mime/tika-mimetypes.xml +++ b/tika-core/src/main/resources/org/apache/tika/mime/tika-mimetypes.xml @@ -7447,6 +7447,10 @@ </match> </magic> <magic priority="45"> + <!-- gate: the first branch below requires a known header at offset 0 (or after a + BOM), and every such header puts a ':' within the first 30 bytes, so files + without an early colon skip the ~40-clause evaluation entirely --> + <match value=":" type="string" offset="0:29"> <!-- be a bit more flexible, but require one from each of these --> <match minShouldMatch="2"> @@ -7515,6 +7519,7 @@ <match value="\nARC-" type="string" offset="0:1024"/> </match> </match> + </match> </magic> <magic priority="40"> <!-- lower priority than message/news --> diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaResource.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaResource.java index eca7b938c4..ab48d46e85 100644 --- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaResource.java +++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaResource.java @@ -700,13 +700,18 @@ public class TikaResource { /** * Produces raw streaming output (text, html, xml, md) using pipes-based parsing. */ + /** Per-request resource-layer latency; joins the pipes lines by adjacency at c=1. */ + private static final org.slf4j.Logger TIMING_LOG = + org.slf4j.LoggerFactory.getLogger("org.apache.tika.pipes.timing.resource"); + private Response produceRawOutput(TikaInputStream tis, Metadata metadata, MultivaluedMap<String, String> httpHeaders, String handlerTypeName) throws IOException { + long entryNanos = System.nanoTime(); fillMetadata(null, metadata, httpHeaders); ParseContext context = createRequestContext(); setupContentHandlerFactory(context, handlerTypeName); - return produceRawOutputWithContext(tis, metadata, context, handlerTypeName); + return produceRawOutputWithContext(tis, metadata, context, handlerTypeName, entryNanos); } /** @@ -718,6 +723,14 @@ public class TikaResource { private Response produceRawOutputWithContext(TikaInputStream tis, Metadata metadata, ParseContext context, String handlerTypeName) throws IOException { + return produceRawOutputWithContext(tis, metadata, context, handlerTypeName, + System.nanoTime()); + } + + private Response produceRawOutputWithContext(TikaInputStream tis, Metadata metadata, + ParseContext context, + String handlerTypeName, long entryNanos) + throws IOException { logRequest(LOG, "/tika", metadata); // Ensure content handler factory is set (config may have set it) @@ -729,8 +742,10 @@ public class TikaResource { // Parse with pipes using CONTENT_ONLY mode - the metadata filter in // EmitHandler will strip everything except tk:content, and the content comes // back as raw UTF-8 bytes rather than a Smile-encoded string + long parseStartNanos = System.nanoTime(); PipesParsingHelper.ParseOutput parsed = parseWithPipesRaw(tis, metadata, context); + long parseEndNanos = System.nanoTime(); List<Metadata> metadataList = parsed.metadataList(); LOG.debug("produceRawOutput: parseWithPipes returned {} metadata objects", metadataList.size()); @@ -760,9 +775,23 @@ public class TikaResource { // container exception should use /rmeta. final byte[] finalContent = content == null ? new byte[0] : content; + final long buildEndNanos = System.nanoTime(); StreamingOutput streamingOutput = outputStream -> { + long writeStart = System.nanoTime(); outputStream.write(finalContent); outputStream.flush(); + if (TIMING_LOG.isInfoEnabled()) { + // pre = header/context setup before the pipes call; build = result + // unpacking after it; write = streaming the body (invoked later by the + // JAX-RS runtime, so anything left over vs the client-observed total is + // the HTTP stack itself) + TIMING_LOG.info("RESOURCE_TIMING pre_us={} pipes_us={} build_us={} write_us={} bytes={}", + (parseStartNanos - entryNanos) / 1000, + (parseEndNanos - parseStartNanos) / 1000, + (buildEndNanos - parseEndNanos) / 1000, + (System.nanoTime() - writeStart) / 1000, + finalContent.length); + } }; return Response.status(hasException ? 422 : Response.Status.OK.getStatusCode()) .entity(streamingOutput)
