This is an automated email from the ASF dual-hosted git repository. tballison pushed a commit to branch TIKA-4809-stage-2 in repository https://gitbox.apache.org/repos/asf/tika.git
commit f2a8c08ab5af4201d0c3ebcc73a43caccdc23ecc Author: tallison <[email protected]> AuthorDate: Fri Aug 7 15:20:25 2026 -0400 TIKA-4809: Migrate /meta onto the shared pipes-backed PipesParser, take 2 --- .../apache/tika/server/core/TikaServerProcess.java | 3 +- .../server/core/resource/PipesParsingHelper.java | 63 +++++++++++++++++----- .../tika/server/core/resource/PipesResource.java | 9 +++- .../tika/server/core/resource/TikaResource.java | 62 +++++++++++---------- .../org/apache/tika/server/core/CXFTestBase.java | 10 +++- .../apache/tika/server/core/StackTraceTest.java | 6 +++ .../org/apache/tika/server/core/TikaPipesTest.java | 2 +- .../apache/tika/server/core/TikaResourceTest.java | 6 ++- .../apache/tika/server/standard/TikaPipesTest.java | 2 +- 9 files changed, 115 insertions(+), 48 deletions(-) diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerProcess.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerProcess.java index 351ae9a0bc..e5ec5e30a9 100644 --- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerProcess.java +++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerProcess.java @@ -430,7 +430,8 @@ public class TikaServerProcess { // Lifecycle (shutdown/close) is owned by whoever built the shared parser, // not by PipesResource. PipesParsingHelper helper = tikaResource.getPipesParsingHelper(); - resourceProviders.add(new SingletonResourceProvider(new PipesResource(helper.getPipesParser()))); + resourceProviders.add(new SingletonResourceProvider( + new PipesResource(helper.getPipesParser(), helper.isReturnStackTrace()))); } resourceProviders.addAll(loadResourceServices(serverStatus)); return resourceProviders; diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/PipesParsingHelper.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/PipesParsingHelper.java index 78913c6875..2d87de0605 100644 --- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/PipesParsingHelper.java +++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/PipesParsingHelper.java @@ -34,6 +34,7 @@ import org.slf4j.LoggerFactory; import org.apache.tika.io.TikaInputStream; import org.apache.tika.metadata.Metadata; +import org.apache.tika.metadata.Property; import org.apache.tika.metadata.TikaCoreProperties; import org.apache.tika.parser.ParseContext; import org.apache.tika.pipes.api.FetchEmitTuple; @@ -168,7 +169,9 @@ public class PipesParsingHelper { PipesResult result = pipesParser.parse(tuple); // Process result - return processResult(result); + List<Metadata> metadataList = processResult(result); + redactExceptionDetail(metadataList); + return metadataList; } catch (InterruptedException e) { Thread.currentThread().interrupt(); @@ -281,6 +284,40 @@ public class PipesParsingHelper { return Collections.emptyList(); } + /** + * Trims CONTAINER_EXCEPTION/EMBEDDED_EXCEPTION to one line unless returnStackTrace is + * on -- unlike buildProcessFailureResponse's family, a 200 response has no other way + * to signal a per-document exception, so we can't omit these fields entirely. + */ + private void redactExceptionDetail(List<Metadata> metadataList) { + if (returnStackTrace || metadataList == null) { + return; + } + for (Metadata m : metadataList) { + summarizeInPlace(m, TikaCoreProperties.CONTAINER_EXCEPTION); + summarizeInPlace(m, TikaCoreProperties.EMBEDDED_EXCEPTION); + } + } + + private static void summarizeInPlace(Metadata m, Property property) { + String full = m.get(property); + if (full != null) { + m.set(property, summarizeStackTrace(full, false)); + } + } + + /** + * First line of a stack trace (the caught exception's own class + message); no-op if + * returnStackTrace. + */ + public static String summarizeStackTrace(String fullTrace, boolean returnStackTrace) { + if (returnStackTrace || fullTrace == null || fullTrace.isBlank()) { + return fullTrace; + } + int newline = fullTrace.indexOf('\n'); + return newline < 0 ? fullTrace : fullTrace.substring(0, newline); + } + /** * Maps PipesResult status to HTTP response status. */ @@ -314,6 +351,14 @@ public class PipesParsingHelper { return pipesParser; } + /** + * Whether failure responses may include the (potentially stack-trace-bearing) + * {@code PipesResult} message. Mirrors {@code TikaServerConfig.isReturnStackTrace()}. + */ + public boolean isReturnStackTrace() { + return returnStackTrace; + } + /** * Gets the PipesConfig instance. */ @@ -442,18 +487,10 @@ public class PipesParsingHelper { Metadata containerMetadata = metadataList.get(0); String containerException = containerMetadata.get(TikaCoreProperties.CONTAINER_EXCEPTION); if (containerException != null) { - // Map exception type to HTTP status - // 422 (Unprocessable Entity) for parse-related exceptions - int status = 422; // Default for parse exceptions - if (containerException.contains("EncryptedDocumentException") || - containerException.contains("TikaException") || - containerException.contains("NullPointerException") || - containerException.contains("IllegalStateException")) { - status = 422; - } - // Build response with exception string as body for stack trace support - Response response = Response.status(status) - .entity(containerException) + // 422 already signals failure, so (unlike redactExceptionDetail's + // 200 family) the body can be omitted entirely when off. + Response response = Response.status(422) + .entity(returnStackTrace ? containerException : "") .type("text/plain") .build(); throw new WebApplicationException(response); diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/PipesResource.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/PipesResource.java index e09b479be1..6b7eb9fccd 100644 --- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/PipesResource.java +++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/PipesResource.java @@ -52,14 +52,18 @@ public class PipesResource { private static final Logger LOG = LoggerFactory.getLogger(PipesResource.class); private final PipesParser pipesParser; + private final boolean returnStackTrace; /** * @param pipesParser shared parser, also used by /tika, /rmeta, and /unpack. * Lifecycle (construction, shutdown) is owned by whoever * built it, not by this class. + * @param returnStackTrace whether parse_exception may include the full stack trace + * vs. just the first line. */ - public PipesResource(PipesParser pipesParser) { + public PipesResource(PipesParser pipesParser, boolean returnStackTrace) { this.pipesParser = pipesParser; + this.returnStackTrace = returnStackTrace; } @@ -130,7 +134,8 @@ public class PipesResource { private Map<String, String> parseException(String msg, boolean emitted) { Map<String, String> statusMap = new HashMap<>(); statusMap.put("status", "ok"); - statusMap.put("parse_exception", msg); + // 200 response, so trim rather than omit -- same reasoning as redactExceptionDetail. + statusMap.put("parse_exception", PipesParsingHelper.summarizeStackTrace(msg, returnStackTrace)); statusMap.put("emitted", Boolean.toString(emitted)); return statusMap; } diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaResource.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaResource.java index fd8cd651b4..5ac7fe1b4b 100644 --- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaResource.java +++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaResource.java @@ -515,7 +515,7 @@ public class TikaResource { @PUT @Consumes("*/*") @Produces("text/xml") - public StreamingOutput getXhtml(final InputStream is, @Context HttpHeaders httpHeaders) + public Response getXhtml(final InputStream is, @Context HttpHeaders httpHeaders) throws IOException { TikaInputStream tis = TikaInputStream.get(is); tis.getPath(); // Spool to temp file for pipes-based parsing @@ -530,7 +530,7 @@ public class TikaResource { @Consumes("*/*") @Produces("text/plain") @Path("text") - public StreamingOutput getText(final InputStream is, @Context HttpHeaders httpHeaders) + public Response getText(final InputStream is, @Context HttpHeaders httpHeaders) throws IOException { TikaInputStream tis = TikaInputStream.get(is); tis.getPath(); // Spool to temp file for pipes-based parsing @@ -545,7 +545,7 @@ public class TikaResource { @Consumes("*/*") @Produces("text/html") @Path("html") - public StreamingOutput getHtml(final InputStream is, @Context HttpHeaders httpHeaders) + public Response getHtml(final InputStream is, @Context HttpHeaders httpHeaders) throws IOException { TikaInputStream tis = TikaInputStream.get(is); tis.getPath(); // Spool to temp file for pipes-based parsing @@ -560,7 +560,7 @@ public class TikaResource { @Consumes("*/*") @Produces("text/xml") @Path("xml") - public StreamingOutput getXml(final InputStream is, @Context HttpHeaders httpHeaders) + public Response getXml(final InputStream is, @Context HttpHeaders httpHeaders) throws IOException { TikaInputStream tis = TikaInputStream.get(is); tis.getPath(); // Spool to temp file for pipes-based parsing @@ -575,7 +575,7 @@ public class TikaResource { @Consumes("*/*") @Produces("text/plain") @Path("md") - public StreamingOutput getMarkdown(final InputStream is, @Context HttpHeaders httpHeaders) + public Response getMarkdown(final InputStream is, @Context HttpHeaders httpHeaders) throws IOException { TikaInputStream tis = TikaInputStream.get(is); tis.getPath(); // Spool to temp file for pipes-based parsing @@ -634,7 +634,7 @@ public class TikaResource { @Consumes("multipart/form-data") @Produces("text/xml") @Path("config") - public StreamingOutput postRaw(List<Attachment> attachments, @Context HttpHeaders httpHeaders) + public Response postRaw(List<Attachment> attachments, @Context HttpHeaders httpHeaders) throws IOException, TikaConfigException { ParseContext context = createParseContext(); Metadata metadata = Metadata.newInstance(context); @@ -657,7 +657,7 @@ public class TikaResource { @Consumes("multipart/form-data") @Produces("text/plain") @Path("config/text") - public StreamingOutput postText(List<Attachment> attachments, @Context HttpHeaders httpHeaders) + public Response postText(List<Attachment> attachments, @Context HttpHeaders httpHeaders) throws IOException, TikaConfigException { ParseContext context = createParseContext(); Metadata metadata = Metadata.newInstance(context); @@ -679,7 +679,7 @@ public class TikaResource { @Consumes("multipart/form-data") @Produces("text/html") @Path("config/html") - public StreamingOutput postHtml(List<Attachment> attachments, @Context HttpHeaders httpHeaders) + public Response postHtml(List<Attachment> attachments, @Context HttpHeaders httpHeaders) throws IOException, TikaConfigException { ParseContext context = createParseContext(); Metadata metadata = Metadata.newInstance(context); @@ -701,7 +701,7 @@ public class TikaResource { @Consumes("multipart/form-data") @Produces("text/xml") @Path("config/xml") - public StreamingOutput postXml(List<Attachment> attachments, @Context HttpHeaders httpHeaders) + public Response postXml(List<Attachment> attachments, @Context HttpHeaders httpHeaders) throws IOException, TikaConfigException { ParseContext context = createParseContext(); Metadata metadata = Metadata.newInstance(context); @@ -723,7 +723,7 @@ public class TikaResource { @Consumes("multipart/form-data") @Produces("text/plain") @Path("config/md") - public StreamingOutput postMarkdown(List<Attachment> attachments, @Context HttpHeaders httpHeaders) + public Response postMarkdown(List<Attachment> attachments, @Context HttpHeaders httpHeaders) throws IOException, TikaConfigException { ParseContext context = createParseContext(); Metadata metadata = Metadata.newInstance(context); @@ -760,7 +760,7 @@ public class TikaResource { /** * Produces raw streaming output (text, html, xml, md) using pipes-based parsing. */ - private StreamingOutput produceRawOutput(TikaInputStream tis, Metadata metadata, + private Response produceRawOutput(TikaInputStream tis, Metadata metadata, MultivaluedMap<String, String> httpHeaders, String handlerTypeName) throws IOException { fillMetadata(null, metadata, httpHeaders); @@ -771,8 +771,11 @@ public class TikaResource { /** * Produces raw streaming output with a pre-configured ParseContext (for PUT endpoints). + * A container-level parse exception doesn't discard content already captured -- status + * is 422 (no field to embed the exception in, unlike the JSON endpoints), but the body + * still carries whatever content was actually extracted. */ - private StreamingOutput produceRawOutputWithContext(TikaInputStream tis, Metadata metadata, + private Response produceRawOutputWithContext(TikaInputStream tis, Metadata metadata, ParseContext context, String handlerTypeName) throws IOException { logRequest(LOG, "/tika", metadata); @@ -794,42 +797,45 @@ public class TikaResource { LOG.debug("produceRawOutput: parseWithPipes returned {} metadata objects", metadataList.size()); - // For raw streaming endpoints, throw exception if there was a parse error - // (JSON endpoints return exceptions in metadata) - // Note: CONTAINER_EXCEPTION is extracted before the metadata filter runs, - // so it's available in the passback even though the filter strips it - if (!metadataList.isEmpty()) { - String exception = metadataList.get(0).get(TikaCoreProperties.CONTAINER_EXCEPTION); - if (exception != null && !exception.isEmpty()) { - LOG.debug("produceRawOutput: parse exception: {}", exception); - // Wrap in TikaException so TikaServerParseExceptionMapper returns 422 - throw new TikaServerParseException(new TikaException(exception)); - } - } - - // Extract content from result + // Extract content before checking for an exception -- content must not be + // discarded just because a container-level exception also occurred. String content = ""; + boolean hasException = false; + String exceptionMessage = null; if (!metadataList.isEmpty()) { String extracted = metadataList.get(0).get(TikaCoreProperties.TIKA_CONTENT); LOG.debug("produceRawOutput: TIKA_CONTENT length={}", extracted != null ? extracted.length() : 0); if (extracted != null) { content = extracted; } + exceptionMessage = metadataList.get(0).get(TikaCoreProperties.CONTAINER_EXCEPTION); + hasException = exceptionMessage != null && !exceptionMessage.isEmpty(); + if (hasException) { + LOG.debug("produceRawOutput: parse exception: {}", exceptionMessage); + } + } + // No separate field for the exception here, unlike JSON bodies -- append it, + // gated by returnStackTrace like TikaServerParseExceptionMapper. + if (hasException && pipesParsingHelper != null && pipesParsingHelper.isReturnStackTrace()) { + content = content.isEmpty() ? exceptionMessage : content + "\n" + exceptionMessage; } final String finalContent = content; - return outputStream -> { + StreamingOutput streamingOutput = outputStream -> { try (Writer writer = new OutputStreamWriter(outputStream, UTF_8)) { writer.write(finalContent); writer.flush(); } }; + return Response.status(hasException ? 422 : Response.Status.OK.getStatusCode()) + .entity(streamingOutput) + .build(); } /** * Produces raw streaming output with a pre-configured ParseContext (for POST endpoints). */ - private StreamingOutput produceRawOutput(TikaInputStream tis, Metadata metadata, + private Response produceRawOutput(TikaInputStream tis, Metadata metadata, ParseContext context, String handlerTypeName) throws IOException { return produceRawOutputWithContext(tis, metadata, context, handlerTypeName); diff --git a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/CXFTestBase.java b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/CXFTestBase.java index da625df6dd..ed557ad8cb 100644 --- a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/CXFTestBase.java +++ b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/CXFTestBase.java @@ -215,7 +215,7 @@ public abstract class CXFTestBase { pipesConfig.setEmitStrategy(new EmitStrategyConfig(EmitStrategy.PASSBACK_ALL)); this.pipesParser = PipesParser.load(tikaJsonConfig, pipesConfig, this.pipesConfigPath); PipesParsingHelper pipesParsingHelper = new PipesParsingHelper(this.pipesParser, pipesConfig, - inputTempDirectory, getUnpackEmitterBasePath(), false); + inputTempDirectory, getUnpackEmitterBasePath(), isReturnStackTrace()); tikaResource = new TikaResource(tika, new ServerStatus(), pipesParsingHelper, isAllowPerRequestConfig()); } finally { @@ -377,6 +377,14 @@ public abstract class CXFTestBase { return false; } + /** + * Mirrors TikaServerConfig.isReturnStackTrace(); defaults to false (production + * default). Override in tests that exercise exception-detail visibility. + */ + protected boolean isReturnStackTrace() { + return false; + } + protected InputStream getPipesConfigInputStream() throws IOException { if (getPipesInputPath() == null) { return null; diff --git a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/StackTraceTest.java b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/StackTraceTest.java index 618c019734..752a8af690 100644 --- a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/StackTraceTest.java +++ b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/StackTraceTest.java @@ -67,6 +67,12 @@ public class StackTraceTest extends CXFTestBase { @TempDir private static Path unpackTempDir; + @Override + protected boolean isReturnStackTrace() { + // Matches this class's own TikaServerParseExceptionMapper(true) below. + return true; + } + @Override protected void setUpResources(JAXRSServerFactoryBean sf) { List<ResourceProvider> rCoreProviders = new ArrayList<>(); diff --git a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaPipesTest.java b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaPipesTest.java index 1f0ef15a71..d43abc4c43 100644 --- a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaPipesTest.java +++ b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaPipesTest.java @@ -153,7 +153,7 @@ public class TikaPipesTest extends CXFTestBase { PipesConfig pipesConfig = PipesConfig.load(tikaJsonConfig); pipesConfig.setEmitStrategy(new EmitStrategyConfig(EmitStrategy.EMIT_ALL)); pipesParser = PipesParser.load(tikaJsonConfig, pipesConfig, tikaConfigPath); - pipesResource = new PipesResource(pipesParser); + pipesResource = new PipesResource(pipesParser, false); rCoreProviders.add(new SingletonResourceProvider(pipesResource)); } catch (IOException | TikaConfigException e) { throw new RuntimeException(e); diff --git a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaResourceTest.java b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaResourceTest.java index c5d5ce6da7..c0f01e0863 100644 --- a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaResourceTest.java +++ b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaResourceTest.java @@ -109,7 +109,11 @@ public class TikaResourceTest extends CXFTestBase { assertEquals("Nikolai Lobachevsky", metadata.get("author")); assertEquals("application/mock+xml", metadata.get(Metadata.CONTENT_TYPE)); assertContains("some content", metadata.get(TikaCoreProperties.TIKA_CONTENT)); - assertContains("null pointer message", metadata.get(TikaCoreProperties.CONTAINER_EXCEPTION)); + // returnStackTrace defaults to false here, so CONTAINER_EXCEPTION is trimmed to + // the caught exception's own class + message -- the NPE detail underneath it is + // intentionally not exposed by default. + assertContains("TikaException", metadata.get(TikaCoreProperties.CONTAINER_EXCEPTION)); + assertNotFound("null pointer message", metadata.get(TikaCoreProperties.CONTAINER_EXCEPTION)); } @Test diff --git a/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/TikaPipesTest.java b/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/TikaPipesTest.java index d27b2e1715..08844f84df 100644 --- a/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/TikaPipesTest.java +++ b/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/TikaPipesTest.java @@ -145,7 +145,7 @@ public class TikaPipesTest extends CXFTestBase { PipesConfig pipesConfig = PipesConfig.load(tikaJsonConfig); pipesConfig.setEmitStrategy(new EmitStrategyConfig(EmitStrategy.EMIT_ALL)); pipesParser = PipesParser.load(tikaJsonConfig, pipesConfig, tikaConfigPath); - pipesResource = new PipesResource(pipesParser); + pipesResource = new PipesResource(pipesParser, false); rCoreProviders.add(new SingletonResourceProvider(pipesResource)); } catch (IOException | TikaConfigException e) { throw new RuntimeException(e);
