This is an automated email from the ASF dual-hosted git repository. tballison pushed a commit to branch TIKA-4809-stage-9 in repository https://gitbox.apache.org/repos/asf/tika.git
commit 33c59beef2e3f10de14954a241a94666db8b641f Author: tallison <[email protected]> AuthorDate: Mon Aug 10 16:37:32 2026 -0400 TIKA-4809: Cap /language detection input, and document what the cap does not fix --- .../ROOT/pages/using-tika/server/index.adoc | 13 ++++ .../server/core/resource/LanguageResource.java | 40 +++++++++-- .../tika/server/core/LanguageResourceTest.java | 77 ++++++++++++++++++++++ 3 files changed, 126 insertions(+), 4 deletions(-) diff --git a/docs/modules/ROOT/pages/using-tika/server/index.adoc b/docs/modules/ROOT/pages/using-tika/server/index.adoc index 40594bed85..e979e6d926 100644 --- a/docs/modules/ROOT/pages/using-tika/server/index.adoc +++ b/docs/modules/ROOT/pages/using-tika/server/index.adoc @@ -195,6 +195,19 @@ and parsing); selecting either without it causes the server to refuse to start. is a plain opt-in endpoint — enable it simply by listing it under `endpoints`. See <<_security_configuration,Security Configuration>>. +WARNING: `/language` and `/detect/stream` do their work *in the server's own JVM*, not in a +forked pipes worker. They are therefore outside the process isolation that protects `/tika`, +`/rmeta`, `/meta`, and `/unpack` — a crash or memory exhaustion takes the server with it +rather than one worker. + +`/language` caps detection at the first 100,000 characters, since accuracy saturates well +before that. That bounds the CPU per request, but *not* the memory: the request body is read +into the server's heap before the cap applies, and the server has no maximum request size. A +caller can still exhaust the heap with a large enough body, or with enough concurrent ones. +Treat these endpoints as available only to trusted callers, the same as the rest of the +server — see xref:security.adoc[the security model]. If you do not need them, omit them from +`endpoints`. + == Error Responses tika-server distinguishes two different kinds of failure: the forked worker itself diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/LanguageResource.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/LanguageResource.java index 4f14edbeba..852f11dd2d 100644 --- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/LanguageResource.java +++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/LanguageResource.java @@ -20,6 +20,8 @@ import static java.nio.charset.StandardCharsets.UTF_8; import java.io.IOException; import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.Reader; import java.util.Locale; import jakarta.ws.rs.Consumes; @@ -27,7 +29,6 @@ import jakarta.ws.rs.POST; import jakarta.ws.rs.PUT; import jakarta.ws.rs.Path; import jakarta.ws.rs.Produces; -import org.apache.commons.io.IOUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -71,15 +72,46 @@ public class LanguageResource { return detectString(string); } + /** + * Detection accuracy saturates within the first few thousand characters, so anything + * past this only buys work whose size the caller chooses. Input beyond it is ignored + * rather than rejected: the answer is the same either way, and rejecting would break + * callers who legitimately post whole documents. + * <p> + * This bounds the detection, not the request. These endpoints hold the text in the + * server's own heap instead of a pipes child, so a large enough body still costs + * memory before this class sees it -- for the /string variants the body is already a + * String by then. Bounding the body itself needs a request-size limit, which the + * server does not currently have. See the DoS note in the server docs. + */ + public static final int MAX_DETECT_CHARS = 100_000; + private String detectStream(InputStream is) throws IOException { - String fileTxt = IOUtils.toString(is, UTF_8); - return detectString(fileTxt); + return detectString(readAtMost(is, MAX_DETECT_CHARS)); + } + + /** Reads up to maxChars without materializing the rest of the stream. */ + private static String readAtMost(InputStream is, int maxChars) throws IOException { + Reader reader = new InputStreamReader(is, UTF_8); + char[] buffer = new char[Math.min(maxChars, 8192)]; + StringBuilder sb = new StringBuilder(); + int read; + while (sb.length() < maxChars + && (read = reader.read(buffer, 0, Math.min(buffer.length, maxChars - sb.length()))) != -1) { + sb.append(buffer, 0, read); + } + return sb.toString(); } private String detectString(String string) throws IOException { + String text = string; + if (text != null && text.length() > MAX_DETECT_CHARS) { + LOG.debug("truncating {} chars to {} for language detection", text.length(), MAX_DETECT_CHARS); + text = text.substring(0, MAX_DETECT_CHARS); + } LanguageResult language = LanguageDetector.getDefaultLanguageDetector() .loadModels() - .detect(string); + .detect(text); String detectedLang = toIso1(language.getLanguage()); LOG.debug("Detecting language for incoming resource: [{}]", detectedLang); return detectedLang; diff --git a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/LanguageResourceTest.java b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/LanguageResourceTest.java index d3f1e30dca..bf7bd83c3f 100644 --- a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/LanguageResourceTest.java +++ b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/LanguageResourceTest.java @@ -18,8 +18,11 @@ package org.apache.tika.server.core; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import java.io.ByteArrayInputStream; import java.io.InputStream; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; @@ -110,4 +113,78 @@ public class LanguageResourceTest extends CXFTestBase { assertEquals("fr", readLang); } + /** + * Truncation must not change the answer -- detection saturates far below the cap -- + * and must apply on both the string and the stream path. + */ + @Test + public void testOversizeInputIsTruncatedNotRejected() throws Exception { + StringBuilder sb = new StringBuilder(); + while (sb.length() < LanguageResource.MAX_DETECT_CHARS * 2) { + sb.append(ENGLISH_STRING).append(' '); + } + String oversize = sb.toString(); + assertTrue(oversize.length() > LanguageResource.MAX_DETECT_CHARS); + + Response stringResponse = WebClient + .create(endPoint + LANG_STRING_PATH) + .type("text/plain") + .accept("text/plain") + .put(oversize); + assertEquals(200, stringResponse.getStatus()); + assertEquals("en", getStringFromInputStream((InputStream) stringResponse.getEntity())); + + Response streamResponse = WebClient + .create(endPoint + LANG_STREAM_PATH) + .type("text/plain") + .accept("text/plain") + .put(new ByteArrayInputStream(oversize.getBytes(StandardCharsets.UTF_8))); + assertEquals(200, streamResponse.getStatus()); + assertEquals("en", getStringFromInputStream((InputStream) streamResponse.getEntity())); + } + + /** A stream longer than the cap must not be read past it. */ + @Test + public void testStreamIsNotDrainedPastTheCap() throws Exception { + byte[] head = ENGLISH_STRING.repeat( + (LanguageResource.MAX_DETECT_CHARS / ENGLISH_STRING.length()) + 1) + .getBytes(StandardCharsets.UTF_8); + CountingInputStream counting = new CountingInputStream( + new ByteArrayInputStream(head)); + assertEquals("en", detectViaResource(counting)); + assertTrue(counting.count <= LanguageResource.MAX_DETECT_CHARS + 8192, + "read " + counting.count + " bytes for a " + LanguageResource.MAX_DETECT_CHARS + + "-char cap"); + } + + private String detectViaResource(InputStream is) throws Exception { + return new LanguageResource().detectPutStream(is); + } + + private static class CountingInputStream extends java.io.FilterInputStream { + private int count; + + CountingInputStream(InputStream in) { + super(in); + } + + @Override + public int read(byte[] b, int off, int len) throws java.io.IOException { + int read = super.read(b, off, len); + if (read > 0) { + count += read; + } + return read; + } + + @Override + public int read() throws java.io.IOException { + int read = super.read(); + if (read >= 0) { + count++; + } + return read; + } + } + }
