This is an automated email from the ASF dual-hosted git repository.
tballison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git
The following commit(s) were added to refs/heads/main by this push:
new a6f2502858 Trivial fixes from the post-4.1.0d review (#3211)
a6f2502858 is described below
commit a6f2502858d7520bd9a9ecf262e005686f120fa6
Author: Tim Allison <[email protected]>
AuthorDate: Sat Sep 19 18:10:13 2026 -0400
Trivial fixes from the post-4.1.0d review (#3211)
---
.github/workflows/main-jdk17-build.yml | 2 +-
.skills/devs/development/SKILL.md | 24 ++
docs/modules/ROOT/pages/configuration/index.adoc | 9 +-
.../parsers/tesseract-ocr-parser.adoc | 2 +-
.../pages/migration-to-4x/migrating-to-4x.adoc | 4 +-
docs/modules/ROOT/pages/pipes/cpu-sizing.adoc | 20 +-
docs/modules/ROOT/pages/pipes/performance.adoc | 13 +-
release-tools/uat/run-uat.sh | 16 +-
.../java/org/apache/tika/config/JsonConfig.java | 5 +
.../org/apache/tika/io/BoundedInputStream.java | 29 +-
.../org/apache/tika/io/CachingInputStream.java | 20 +-
.../java/org/apache/tika/io/CachingSource.java | 14 +
.../main/java/org/apache/tika/io/FileSource.java | 18 +-
.../java/org/apache/tika/io/ReopenableSource.java | 29 +-
.../main/java/org/apache/tika/io/StreamCache.java | 56 ++-
.../java/org/apache/tika/io/TikaInputStream.java | 15 +-
.../java/org/apache/tika/parser/ParseContext.java | 9 +-
.../tika/parser/inference/InferenceDispatcher.java | 12 +-
.../org/apache/tika/io/BoundedInputStreamTest.java | 68 ++++
.../apache/tika/io/InMemoryContentViewTest.java | 8 +-
.../org/apache/tika/io/ReopenableSourceTest.java | 20 +-
.../tika/io/TikaInputStreamOwnershipTest.java | 15 +-
.../apache/tika/io/TikaInputStreamPathsTest.java | 400 +++++++++++++++++++++
.../org/apache/tika/parser/ParseContextTest.java | 16 +
.../parser/inference/InferenceDispatcherTest.java | 36 ++
.../java/org/apache/tika/http/TikaHttpClient.java | 4 +
.../tika/parser/ocr/tess4j/Tess4JConfig.java | 2 +-
.../tika/parser/ocr/tess4j/Tess4JConfigTest.java | 11 +
.../apache/tika/parser/vlm/AbstractVLMParser.java | 22 +-
.../tika/parser/vlm/MarkdownToXHTMLEmitter.java | 7 +
.../parser/vlm/MarkdownToXHTMLEmitterTest.java | 10 +
.../tika/parser/vlm/OpenAIVLMParserTest.java | 9 +
.../org/apache/tika/detect/ogg/OggDetector.java | 6 +-
.../org/apache/tika/parser/image/ImageParser.java | 12 +-
.../apache/tika/parser/image/RawTiffParser.java | 3 +-
.../tika/parser/image/RawTiffParserTest.java | 12 +
.../microsoft/MSEmbeddedStreamTranslator.java | 8 +-
.../microsoft/ooxml/AbstractOOXMLExtractor.java | 14 +-
.../apache/tika/parser/ocr/TesseractOCRParser.java | 27 +-
.../parser/ocr/TesseractOCRConfigTrustTest.java | 49 +++
.../apache/tika/parser/pdf/AbstractPDF2XHTML.java | 6 +-
.../apache/tika/parser/pdf/PDFPagesEmitTest.java | 41 +++
.../org/apache/tika/parser/pkg/SevenZParser.java | 52 ++-
.../java/org/apache/tika/parser/pkg/ZipParser.java | 5 +-
.../tika/parser/pkg/PackageEmbeddedBudgetTest.java | 66 +++-
.../serdes/ParseContextDeserializer.java | 2 +-
.../serdes/ParseContextTrustTest.java | 36 ++
tika-server/docker-build/docker-tool.sh | 3 +-
.../sample-configs/vision/vlm-claude.json | 2 +-
.../sample-configs/vision/vlm-gemini.json | 2 +-
.../tika/server/standard/TikaResourceTest.java | 3 +-
51 files changed, 1072 insertions(+), 202 deletions(-)
diff --git a/.github/workflows/main-jdk17-build.yml
b/.github/workflows/main-jdk17-build.yml
index b084a79456..f0d9b8d0e6 100644
--- a/.github/workflows/main-jdk17-build.yml
+++ b/.github/workflows/main-jdk17-build.yml
@@ -72,7 +72,7 @@ jobs:
key: maven-${{ runner.os }}-${{ hashFiles('**/pom.xml') }}
restore-keys: maven-${{ runner.os }}-
- name: Install external tools
- run: sudo apt-get update && sudo apt-get install -y ffmpeg
libimage-exiftool-perl
+ run: sudo apt-get update && sudo apt-get install -y ffmpeg
libimage-exiftool-perl tesseract-ocr
# Everything except $IT_MODULES, which the integration-tests job owns.
# sed turns each ':artifactId' into the '!:artifactId' exclusion form.
# rat runs in its own SERIAL invocation: apache-rat-plugin 0.18 is not
diff --git a/.skills/devs/development/SKILL.md
b/.skills/devs/development/SKILL.md
index 176b12f492..43ee5377a5 100644
--- a/.skills/devs/development/SKILL.md
+++ b/.skills/devs/development/SKILL.md
@@ -166,6 +166,30 @@ personal configuration does not override it.
legitimate exception: a path that is the data under test (e.g. an expected
metadata value extracted from a test document) — leave those untouched.
+## NEVER: two mistakes that ship as vulnerabilities
+
+**Never trust a length that came from user input or from inside a file.**
`Content-Length`
+on a request, a zip or 7z entry's declared size, an OLE/OOXML part size, a PDF
embedded
+file's `/Size`, an ISO box length, a count line in a mail file: every one of
them is an
+untrusted number. Never allocate, reserve, `mark()`, `skip()` or bound a loop
by it as if it
+were true. A lie small truncates content silently; a lie large lets one
document allocate the
+heap or a budget, or spin a reader waiting for bytes that never come. Use a
declared length
+only to *decline* work (a size the budget cannot cover is not attempted) or as
a hint that the
+actual read corrects; grow from a floor and reserve what was actually read;
treat the measured
+end of the stream as the only ground truth (`hasReliableLength()`), and
overwrite the claim
+with it. See TIKA-4908 and the 2026-09-19 stream audit for what this cost.
+
+**Never call `InputStream.skip()` on a stream that could be a
`FileInputStream` (or a
+`BufferedInputStream` over one) and trust the answer.**
`FileInputStream.skip(n)` seeks past
+EOF and returns `n` even when no bytes exist, so a reader that skips a payload
and then reads
+the next header believes it is somewhere it is not, and loops or mis-parses
forever;
+`BufferedInputStream.skip` can return 0 before EOF, which spins any `while
(remaining > 0)
+remaining -= skip(remaining)` loop. Skip by reading (`IOUtils.skip` /
`IOUtils.skipFully`) or
+seek on a channel or an in-memory buffer whose size you own;
`TikaInputStream.skip`,
+`BoundedInputStream.skip` and every source's `seekTo` read for this reason. Do
not "optimize"
+them back. Anything hot enough to need a real seek gets a channel from
+`getSeekableByteChannel()`, never a raw `skip`.
+
## Test Discipline
- A behavioral change gets a regression test that fails without it. Where
diff --git a/docs/modules/ROOT/pages/configuration/index.adoc
b/docs/modules/ROOT/pages/configuration/index.adoc
index 3d6aa1d0eb..2867cbf523 100644
--- a/docs/modules/ROOT/pages/configuration/index.adoc
+++ b/docs/modules/ROOT/pages/configuration/index.adoc
@@ -282,8 +282,13 @@ stops the server at startup rather than reaching a service
as an empty key. A li
before.
The forked parse workers of tika-server and Pipes inherit the parent's
environment and read the same
-config file, so a reference resolves there too. Per-request configuration
(`/config` endpoints,
-presets, pipes tuples) is never interpolated: a request cannot read the
server's environment.
+config file, so a reference resolves there too. Presets written inline in the
config file resolve
+at startup with the rest of it; catalog presets and per-request configuration
(`/config`
+endpoints, pipes tuples) never do: a request cannot read the server's
environment.
+
+A reference keeps a secret out of tika-config.json, not out of every file:
with a persistent
+config store (`configStoreType` `file` or Ignite) the resolved fetcher and
emitter configurations
+are written to that store. The default in-memory store writes nothing.
== Windows file paths
diff --git
a/docs/modules/ROOT/pages/configuration/parsers/tesseract-ocr-parser.adoc
b/docs/modules/ROOT/pages/configuration/parsers/tesseract-ocr-parser.adoc
index e1eb49aaab..b61d6b6452 100644
--- a/docs/modules/ROOT/pages/configuration/parsers/tesseract-ocr-parser.adoc
+++ b/docs/modules/ROOT/pages/configuration/parsers/tesseract-ocr-parser.adoc
@@ -73,7 +73,7 @@ include::example$tesseract-full.json[]
icon:github[]
https://github.com/apache/tika/blob/main/tika-parsers/tika-parsers-standard/tika-parsers-standard-integration-tests/src/test/resources/config-examples/tesseract-full.json[View
source on GitHub]
Arbitrary Tesseract variables go in `otherTesseractConfig` as a name-to-value
map, as in the
-example above. The map is server-side configuration only: a per-request
`tesseract-ocr-parser`
+example above. The map is operator configuration only (tika-config.json,
including its presets): a per-request `tesseract-ocr-parser`
block that carries it is rejected, because variables such as `debug_file` name
files the
tesseract binary opens. Converting a 3.x config? See
xref:migration-to-4x/migrating-to-4x.adoc[Migrating to 4.x] — the automatic
converter rewrites the
diff --git a/docs/modules/ROOT/pages/migration-to-4x/migrating-to-4x.adoc
b/docs/modules/ROOT/pages/migration-to-4x/migrating-to-4x.adoc
index 7f67aa39e3..28b7fd7d44 100644
--- a/docs/modules/ROOT/pages/migration-to-4x/migrating-to-4x.adoc
+++ b/docs/modules/ROOT/pages/migration-to-4x/migrating-to-4x.adoc
@@ -300,8 +300,8 @@ change with the aliases: page renders are emitted at the
end of each page whiche
and a page the parser does not read (`maxPages`) is not rendered. Two floors
are new: a page
narrower or shorter than 2 pixels at the target dpi is not rendered, and an
image under 2 x 2
pixels reaches no text recognizer or inference binding
(`pages.render.minWidth`/`minHeight`,
-`_min-width`/`_min-height` on a recognizer entry, `minWidth`/`minHeight` on a
binding; 0 restores
-4.0). A per-request `ocr` block now changes only the fields it names (4.0
replaced the whole
+`_min-width`/`_min-height` on a recognizer entry, `minWidth`/`minHeight` on a
binding). A binding's
+`minWidth: 0` restores 4.0; the 2-pixel floor before the recognizers stays
whatever the entry says. A per-request `ocr` block now changes only the fields
it names (4.0 replaced the whole
block). In Java, `PDFParserConfig.getOcr()` and `getImageStrategy()` are gone:
`getPages()` is
the overlay, the aliases write to it, the deprecated setters stay.
diff --git a/docs/modules/ROOT/pages/pipes/cpu-sizing.adoc
b/docs/modules/ROOT/pages/pipes/cpu-sizing.adoc
index 5dbedc88b7..8e2208eac7 100644
--- a/docs/modules/ROOT/pages/pipes/cpu-sizing.adoc
+++ b/docs/modules/ROOT/pages/pipes/cpu-sizing.adoc
@@ -59,12 +59,12 @@ PARENT_RESERVED_CORES = 2
MIN_AUTO_CAP_SLICE = 2
----
-If `slice ≥ 2`, Tika injects `-XX:ActiveProcessorCount=<slice>` into each
-forked JVM. If `slice < 2`, the auto-cap is *skipped* and a `WARN` is
-logged advising the operator to lower `numClients`. Skipping is intentional:
-at `slice=1` the fork's only CPU is fully consumed by parsing, so its
-socket-reader thread cannot run and the parent's writes block on
-receiver-side back-pressure — measurably worse than no cap at all.
+Tika injects `-XX:ActiveProcessorCount=<slice>` into each forked JVM. A fair
+share below 2 is clamped to 2 rather than left uncapped: at `slice=1` the
+fork's only CPU is fully consumed by parsing, so its socket-reader thread
+cannot run, and with no cap at all every fork sizes its GC and JIT pools for
+the whole host. The clamp is logged as `slice=2 (clamped; fair share would be
+N)`, and with more than one client a `WARN` advises lowering `numClients`.
== Recommended sizing
@@ -74,14 +74,14 @@ For typical cloud-VM core counts:
|===
|hostCores |numClients |slice |Notes
-|2 |1 |1 → skipped |Tight; auto-cap declines. Acceptable for low throughput.
+|2 |1 |1 → 2 (clamped) |Tight. Acceptable for low throughput.
|4 |1 |2 |Comfortable single-fork deployment.
-|4 |2 |1 → skipped |Auto-cap declines; consider `numClients=1`.
+|4 |2 |1 → 2 (clamped) |Over-provisioned; consider `numClients=1`.
|8 |1 |6 |Lots of headroom; single-fork lifecycle isolation is fine.
|8 |3 |2 |Sweet spot for medium pods.
|16 |4 |3 |Sweet spot for 16-core hosts. Measured winner in benchmarks.
|16 |6 |2 |Higher concurrency; tighter per-fork breathing room.
-|16 |8 |1 → skipped |Doesn't fit 16 cores. Keep at 4 or 6.
+|16 |8 |1 → 2 (clamped) |Doesn't fit 16 cores. Keep at 4 or 6.
|32 |8 |3 |Same shape as 16/4.
|===
@@ -122,7 +122,7 @@ INFO pipes-cpu-sizing: hostCores=16, numClients=4,
parentReserved=2, autoCap=sl
The `autoCap` field is one of:
* `slice=N` — the auto-cap fired; each fork sees N CPUs.
-* `skipped (slice<2)` — over-provisioned; operator should reduce `numClients`.
+* `slice=2 (clamped; fair share would be N)` — over-provisioned; reduce
`numClients`.
* `user-set in forkedJvmArgs` — operator set `-XX:ActiveProcessorCount`
themselves.
Two `WARN`-level messages call out clearly-bad provisioning:
diff --git a/docs/modules/ROOT/pages/pipes/performance.adoc
b/docs/modules/ROOT/pages/pipes/performance.adoc
index 2ad85cb059..525c592a43 100644
--- a/docs/modules/ROOT/pages/pipes/performance.adoc
+++ b/docs/modules/ROOT/pages/pipes/performance.adoc
@@ -241,11 +241,10 @@ outputs on the same pair of spindles. The corpus is far
larger than RAM, so
every run is effectively cold-cache. The configuration we settled on:
* **Per-client mode, `numClients = 7`.** Tika auto-injects
- `-XX:ActiveProcessorCount` per fork as `(cores − 2) / numClients` but only
- when that slice is at least 2. On 16 logical cores, 8 workers give 1.75 and
- the cap is *skipped* — eight JVMs each sizing GC and JIT for 16 cores, which
- is what the 7.5 h run did. Seven workers get 2 cores each and the cap
- applies. Per-client cost about 7% versus shared-server here (29.0 min vs
+ `-XX:ActiveProcessorCount` per fork as `(cores − 2) / numClients`, clamped
+ at 2. Before 4.1.0 a share below 2 switched the cap off: on 16 logical
+ cores, 8 workers gave 1.75 and eight JVMs each sized GC and JIT for 16
+ cores, which is what the 7.5 h run did. Seven workers get 2 cores each.
Per-client cost about 7% versus shared-server here (29.0 min vs
27.2 min in the controlled study above)
and dropped no files, where the shared worker loses the in-flight documents
of every other client when one document crashes it.
@@ -271,8 +270,8 @@ box:
cache (or use a subset you have not touched) and write outputs to the real
destination. Warm-cache runs with outputs on tmpfs hid this entire problem
from us for weeks.
-* **Does your `numClients` trip the cap skip?** Check the startup log for the
- `ActiveProcessorCount` decision; if it reports `skipped`, lower
+* **Is your `numClients` over-provisioned?** Check the startup log for the
+ `ActiveProcessorCount` decision; if it reports `clamped`, lower
`numClients` or set the cap yourself in `forkedJvmArgs`.
Beyond those: fix concurrency equal to the worker count when comparing, exclude
diff --git a/release-tools/uat/run-uat.sh b/release-tools/uat/run-uat.sh
index 0e924c7ab5..2ad0923b6f 100755
--- a/release-tools/uat/run-uat.sh
+++ b/release-tools/uat/run-uat.sh
@@ -300,17 +300,23 @@ if [[ -n "$INFERENCE" ]]; then
# T44: per-request switches (allowPerRequestConfig is on in the UAT config):
the
# recognizer off removes the OCR text; a bindings subset runs only that
binding.
- OFF=$(curl -s -X POST -F "file=@$OCRPNG" \
+ OFF=$(curl -s -o /tmp/uat-t44.out -w '%{http_code}' -X POST -F
"file=@$OCRPNG" \
-F 'config={"parse-context":{"text-recognizers":{"enabled":false}}}'
"$BASE/tika/config/text")
- if printf '%s' "$OFF" | grep -qF "The quick brown fox"; then
+ if [[ "$OFF" != "200" ]]; then
+ bad "T44 text-recognizers.enabled=false per request" "HTTP 200" "HTTP $OFF"
+ elif grep -qF "The quick brown fox" /tmp/uat-t44.out; then
bad "T44 text-recognizers.enabled=false per request" "no OCR text" "OCR
text present"
else
ok "T44 text-recognizers.enabled=false per request"
fi
- SUBSET=$(curl -s -X POST -F "file=@$OCRPNG" \
+ SUBSET_CODE=$(curl -s -o /tmp/uat-t45.out -w '%{http_code}' -X POST -F
"file=@$OCRPNG" \
-F 'config={"parse-context":{"inference":{"bindings":["text"]}}}' \
-H 'Accept: application/json' "$BASE/rmeta/config")
- if printf '%s' "$SUBSET" | grep -qF '\"producer\":\"pictures\"'; then
+ if [[ "$SUBSET_CODE" != "200" ]]; then
+ bad "T45 inference.bindings subset per request" "HTTP 200" "HTTP
$SUBSET_CODE"
+ elif ! grep -qF '\"producer\":\"text\"' /tmp/uat-t45.out; then
+ bad "T45 inference.bindings subset per request" "text chunk present" "no
text chunk"
+ elif grep -qF '\"producer\":\"pictures\"' /tmp/uat-t45.out; then
bad "T45 inference.bindings subset per request" "no pictures chunk"
"pictures binding ran"
else
ok "T45 inference.bindings subset per request"
@@ -329,6 +335,8 @@ if [[ -n "$INFERENCE" ]]; then
else
skip "T46 MEDIA bindings -- no segment vectors (ffmpeg not available on
server)"
fi
+ elif [[ "${TIKA_UAT_REQUIRE_MEDIA:-}" == "1" ]]; then
+ bad "T46 MEDIA bindings" "$VIDEO present (TIKA_UAT_REQUIRE_MEDIA=1)"
"missing"
else
skip "T46 MEDIA bindings -- $VIDEO missing"
fi
diff --git a/tika-core/src/main/java/org/apache/tika/config/JsonConfig.java
b/tika-core/src/main/java/org/apache/tika/config/JsonConfig.java
index 7d4d1eadaa..ab507c5d86 100644
--- a/tika-core/src/main/java/org/apache/tika/config/JsonConfig.java
+++ b/tika-core/src/main/java/org/apache/tika/config/JsonConfig.java
@@ -35,4 +35,9 @@ public interface JsonConfig {
* @return the JSON configuration
*/
String json();
+
+ /** Operator-authored (config file, presets), as opposed to per-request
input. */
+ default boolean trusted() {
+ return false;
+ }
}
diff --git a/tika-core/src/main/java/org/apache/tika/io/BoundedInputStream.java
b/tika-core/src/main/java/org/apache/tika/io/BoundedInputStream.java
index 0155bd40c7..456963cee7 100644
--- a/tika-core/src/main/java/org/apache/tika/io/BoundedInputStream.java
+++ b/tika-core/src/main/java/org/apache/tika/io/BoundedInputStream.java
@@ -36,6 +36,7 @@ public class BoundedInputStream extends InputStream {
private final long max;
private final InputStream in;
private long pos;
+ private long markPos;
public BoundedInputStream(long max, InputStream in) {
this.max = max;
@@ -93,19 +94,14 @@ public class BoundedInputStream extends InputStream {
return bytesRead;
}
- /**
- * Invokes the delegate's <code>skip(long)</code> method.
- * As with InputStream generally, this does not guarantee reading n bytes.
- * Use IOUtils' skipFully for that functionality.
- *
- * @param n the number of bytes to skip
- * @return the actual number of bytes skipped
- * @throws IOException if an I/O error occurs
- */
+ /** Reads, never {@code in.skip()}: FileInputStream seeks past EOF and
reports the full count. */
@Override
public long skip(final long n) throws IOException {
final long toSkip = max >= 0 ? Math.min(n, max - pos) : n;
- final long skippedBytes = in.skip(toSkip);
+ if (toSkip <= 0) {
+ return 0;
+ }
+ final long skippedBytes = IOUtils.skip(in, toSkip);
pos += skippedBytes;
return skippedBytes;
}
@@ -113,28 +109,19 @@ public class BoundedInputStream extends InputStream {
@Override
public void reset() throws IOException {
in.reset();
- pos = 0;
+ pos = markPos;
}
@Override
public void mark(int readLimit) {
in.mark(readLimit);
+ markPos = pos;
}
public boolean hasHitBound() {
return pos >= max;
}
- @Override
- public byte[] readNBytes(int len) throws IOException {
- return in.readNBytes(len);
- }
-
- @Override
- public int readNBytes(byte[] b, int off, int len) throws IOException {
- return in.readNBytes(b, off, len);
- }
-
@Override
public int available() throws IOException {
return in.available();
diff --git a/tika-core/src/main/java/org/apache/tika/io/CachingInputStream.java
b/tika-core/src/main/java/org/apache/tika/io/CachingInputStream.java
index 4d743f00e1..c919b3620d 100644
--- a/tika-core/src/main/java/org/apache/tika/io/CachingInputStream.java
+++ b/tika-core/src/main/java/org/apache/tika/io/CachingInputStream.java
@@ -74,6 +74,7 @@ class CachingInputStream extends InputStream {
cache.append(buffer, 0, n);
}
sourceExhausted = true;
+ cache.trim();
}
@Override
@@ -95,6 +96,7 @@ class CachingInputStream extends InputStream {
int b = source.read();
if (b == -1) {
sourceExhausted = true;
+ cache.trim();
return -1;
}
@@ -129,6 +131,7 @@ class CachingInputStream extends InputStream {
int n = source.read(b, off, len);
if (n == -1) {
sourceExhausted = true;
+ cache.trim();
} else if (n > 0) {
cache.append(b, off, n);
position += n;
@@ -144,13 +147,16 @@ class CachingInputStream extends InputStream {
if (n <= 0) {
return 0;
}
-
- // We need to actually read the bytes to cache them
long skipped = 0;
- byte[] buffer = new byte[4096];
+ long cached = cache.size() - position;
+ if (cached > 0) {
+ skipped = Math.min(n, cached);
+ position += skipped;
+ }
+ // beyond the cache the bytes must be read to be cached
+ byte[] buffer = new byte[8192];
while (skipped < n) {
- int toRead = (int) Math.min(buffer.length, n - skipped);
- int read = read(buffer, 0, toRead);
+ int read = read(buffer, 0, (int) Math.min(buffer.length, n -
skipped));
if (read == -1) {
break;
}
@@ -187,6 +193,10 @@ class CachingInputStream extends InputStream {
return cache.size();
}
+ boolean isSourceExhausted() {
+ return sourceExhausted;
+ }
+
/**
* Force all remaining content to be read and cached, then return the file
path.
*/
diff --git a/tika-core/src/main/java/org/apache/tika/io/CachingSource.java
b/tika-core/src/main/java/org/apache/tika/io/CachingSource.java
index d77b86779b..fc43d1bf4b 100644
--- a/tika-core/src/main/java/org/apache/tika/io/CachingSource.java
+++ b/tika-core/src/main/java/org/apache/tika/io/CachingSource.java
@@ -95,6 +95,8 @@ class CachingSource extends InputStream implements
TikaInputSource {
int b = passthroughStream.read();
if (b != -1) {
passthroughPosition++;
+ } else {
+ measured(passthroughPosition);
}
return b;
}
@@ -114,6 +116,8 @@ class CachingSource extends InputStream implements
TikaInputSource {
int n = passthroughStream.read(b, off, len);
if (n > 0) {
passthroughPosition += n;
+ } else if (n == -1) {
+ measured(passthroughPosition);
}
return n;
}
@@ -330,13 +334,23 @@ class CachingSource extends InputStream implements
TikaInputSource {
return spilledPath;
}
+ /** The end of the source is ground truth, over a declared length either
way. */
+ private void measured(long total) {
+ length = total;
+ lengthMeasured = true;
+ }
+
@Override
public long getLength() {
+ if (!lengthMeasured && cachingStream != null &&
cachingStream.isSourceExhausted()) {
+ measured(cachingStream.getCachedSize());
+ }
return length;
}
@Override
public boolean hasReliableLength() {
+ getLength();
return lengthMeasured;
}
diff --git a/tika-core/src/main/java/org/apache/tika/io/FileSource.java
b/tika-core/src/main/java/org/apache/tika/io/FileSource.java
index a57abf7ba9..babb114604 100644
--- a/tika-core/src/main/java/org/apache/tika/io/FileSource.java
+++ b/tika-core/src/main/java/org/apache/tika/io/FileSource.java
@@ -87,12 +87,9 @@ class FileSource extends InputStream implements
TikaInputSource {
throw new IOException("Cannot seek past end of file. Position: " +
newPosition + ", length: " + length);
}
-
- // Close current stream and reopen at the beginning
currentStream.close();
currentStream = new BufferedInputStream(Files.newInputStream(path));
-
- // Skip to the new position
+ markInStream = false;
if (newPosition > 0) {
IOUtils.skipFully(currentStream, newPosition);
}
@@ -143,11 +140,15 @@ class FileSource extends InputStream implements
TikaInputSource {
}
// Mark/reset support using seekTo
+ private static final int MAX_BUFFERED_MARK = 1024 * 1024;
private long markPosition = -1;
+ private boolean markInStream;
@Override
public synchronized void mark(int readlimit) {
markPosition = position;
+ currentStream.mark(Math.min(readlimit, MAX_BUFFERED_MARK));
+ markInStream = true;
}
@Override
@@ -155,6 +156,15 @@ class FileSource extends InputStream implements
TikaInputSource {
if (markPosition < 0) {
throw new IOException("Mark not set");
}
+ if (markInStream) {
+ try {
+ currentStream.reset(); // within the limit the buffer holds
the mark
+ position = markPosition;
+ return;
+ } catch (IOException e) {
+ // past it: reopen
+ }
+ }
seekTo(markPosition);
}
diff --git a/tika-core/src/main/java/org/apache/tika/io/ReopenableSource.java
b/tika-core/src/main/java/org/apache/tika/io/ReopenableSource.java
index bf55ca44c5..ccd49cd4f8 100644
--- a/tika-core/src/main/java/org/apache/tika/io/ReopenableSource.java
+++ b/tika-core/src/main/java/org/apache/tika/io/ReopenableSource.java
@@ -109,7 +109,12 @@ class ReopenableSource extends InputStream implements
TikaInputSource {
InputStream in = new BufferedInputStream(
spilledPath != null ? Files.newInputStream(spilledPath) :
opener.get());
if (at > 0) {
- IOUtils.skipFully(in, at);
+ try {
+ IOUtils.skipFully(in, at);
+ } catch (IOException e) {
+ in.close();
+ throw e;
+ }
}
return in;
}
@@ -177,6 +182,9 @@ class ReopenableSource extends InputStream implements
TikaInputSource {
try (OutputStream out = Files.newOutputStream(p)) {
if (retainedBuffer != null) {
out.write(retainedBuffer, 0, retainedLength);
+ if (channelPins == 0) {
+ dropRetained(); // the file serves every read from
here on
+ }
} else {
try (InputStream in = opener.get()) {
IOUtils.copy(in, out);
@@ -203,7 +211,7 @@ class ReopenableSource extends InputStream implements
TikaInputSource {
@Override
public void enableRewind(CacheMemoryBudget budget) throws IOException {
- if (position != 0) {
+ if (position != 0 && retainedBuffer == null && spilledPath == null) {
throw new IOException("Cannot enable rewind: position is " +
position +
", must be 0. Call enableRewind() before reading.");
}
@@ -225,16 +233,10 @@ class ReopenableSource extends InputStream implements
TikaInputSource {
if (retainedBuffer != null) {
return true;
}
- if (spilledPath != null || closed || length < 0 || length >
MAX_ARRAY_SIZE) {
+ if (spilledPath != null || closed || length > MAX_ARRAY_SIZE) {
return false;
}
- if (length > IN_MEMORY_FLOOR) {
- if (budget == null ||
- budget.getMaxBytes() - budget.getReservedBytes() < length
- IN_MEMORY_FLOOR) {
- return false;
- }
- }
- if (!tryBufferInMemory()) {
+ if (!mayFitInMemory() || !tryBufferInMemory()) {
return false;
}
if (currentStream != null) {
@@ -286,12 +288,19 @@ class ReopenableSource extends InputStream implements
TikaInputSource {
* grows, reserving, on what is actually read. Does not disturb this
source's read
* position.
*/
+ /** A declared length the budget cannot cover is not attempted; unknown is
attempted. */
+ private boolean mayFitInMemory() {
+ return length <= IN_MEMORY_FLOOR || (budget != null
+ && budget.getMaxBytes() - budget.getReservedBytes() >= length
- IN_MEMORY_FLOOR);
+ }
+
private boolean tryBufferInMemory() throws IOException {
if (length > MAX_ARRAY_SIZE || (length > IN_MEMORY_FLOOR && budget ==
null)) {
return false;
}
long reservedHere = 0;
// Reservation invariant: reservedHere == max(0, data.length -
IN_MEMORY_FLOOR)
+ // never sized from the declared length past the floor: it is the
file's claim
byte[] data = new byte[(int) Math.max(8192, Math.min(length,
IN_MEMORY_FLOOR))];
int total = 0;
boolean fits = false;
diff --git a/tika-core/src/main/java/org/apache/tika/io/StreamCache.java
b/tika-core/src/main/java/org/apache/tika/io/StreamCache.java
index 606910abcd..f5f78a8091 100644
--- a/tika-core/src/main/java/org/apache/tika/io/StreamCache.java
+++ b/tika-core/src/main/java/org/apache/tika/io/StreamCache.java
@@ -25,6 +25,7 @@ import java.io.RandomAccessFile;
import java.nio.channels.SeekableByteChannel;
import java.nio.file.Files;
import java.nio.file.Path;
+import java.util.Arrays;
/**
* Package-private cache that stores bytes in memory up to a threshold,
@@ -57,6 +58,10 @@ class StreamCache implements Closeable {
private Path spillFile;
// one long-lived read handle; opening per read cost ~37x on
byte-at-a-time readers
private RandomAccessFile reader;
+ // a window over the spill file: byte-at-a-time readers otherwise pay a
seek and a read each
+ private final byte[] window = new byte[8192];
+ private long windowStart = -1;
+ private int windowLength;
private OutputStream spillOutputStream;
private long totalSize;
@@ -217,12 +222,18 @@ class StreamCache implements Closeable {
if (memoryBuffer != null) {
return memoryBuffer[(int) position] & 0xFF;
- } else {
+ }
+ if (position < windowStart || position >= windowStart + windowLength) {
flushSpillStream();
RandomAccessFile raf = reader();
raf.seek(position);
- return raf.read();
+ windowLength = Math.max(0, raf.read(window, 0, window.length));
+ windowStart = position;
+ if (windowLength == 0) {
+ return -1;
+ }
}
+ return window[(int) (position - windowStart)] & 0xFF;
}
/**
@@ -241,11 +252,27 @@ class StreamCache implements Closeable {
if (memoryBuffer != null) {
System.arraycopy(memoryBuffer, (int) position, b, off, available);
return available;
- } else {
- flushSpillStream();
- RandomAccessFile raf = reader();
- raf.seek(position);
- return raf.read(b, off, available);
+ }
+ if (position >= windowStart && position + available <= windowStart +
windowLength) {
+ System.arraycopy(window, (int) (position - windowStart), b, off,
available);
+ return available;
+ }
+ flushSpillStream();
+ RandomAccessFile raf = reader();
+ raf.seek(position);
+ return raf.read(b, off, available);
+ }
+
+ /** The drain is complete: give the doubling slack back to the budget. */
+ void trim() {
+ if (memoryBuffer == null || memoryBuffer.length - memorySize <= 8192) {
+ return;
+ }
+ memoryBuffer = Arrays.copyOf(memoryBuffer, Math.max(memorySize, 1));
+ long target = Math.max(0, (long) memoryBuffer.length -
memoryThreshold);
+ if (budget != null && reserved > target) {
+ budget.release(reserved - target);
+ reserved = target;
}
}
@@ -343,16 +370,11 @@ class StreamCache implements Closeable {
closed = true;
memoryBuffer = null;
maybeReleaseReserved();
-
- if (spillOutputStream != null) {
- spillOutputStream.close();
- spillOutputStream = null;
- }
- if (reader != null) {
- reader.close();
- reader = null;
- }
- // spillFile cleanup is handled by TemporaryResources
+ OutputStream out = spillOutputStream;
+ RandomAccessFile raf = reader;
+ spillOutputStream = null;
+ reader = null;
+ TemporaryResources.closeAll(out, raf); // both, even if one throws;
file deletion is tmp's
}
/**
diff --git a/tika-core/src/main/java/org/apache/tika/io/TikaInputStream.java
b/tika-core/src/main/java/org/apache/tika/io/TikaInputStream.java
index 09e6148f60..fcce527435 100644
--- a/tika-core/src/main/java/org/apache/tika/io/TikaInputStream.java
+++ b/tika-core/src/main/java/org/apache/tika/io/TikaInputStream.java
@@ -105,6 +105,9 @@ public class TikaInputStream extends TaggedInputStream {
if (stream instanceof TikaInputStream) {
return (TikaInputStream) stream;
}
+ if (tmp == null) {
+ tmp = new TemporaryResources();
+ }
String ext = getExtension(metadata);
TikaInputSource inputSource =
new CachingSource(stream, tmp, declaredLength(metadata),
metadata, ext);
@@ -331,6 +334,10 @@ public class TikaInputStream extends TaggedInputStream {
*/
@Override
public long skip(long n) throws IOException {
+ if (n <= 0) {
+ return 0;
+ }
+ // reads, never in.skip(): FileInputStream.skip seeks past EOF and
reports the full count
long skipped = IOUtils.skip(in, n);
position += skipped;
return skipped;
@@ -586,7 +593,7 @@ public class TikaInputStream extends TaggedInputStream {
* spilled content from a file channel, and unread stream content is
drained through the
* cache which decides memory-vs-disk as it goes. Use this when random
access is needed
* (e.g. reading a zip central directory); reserve {@code getFile()} for
callers that truly
- * need a {@link java.io.File}. The caller owns closing the returned
channel. Does not
+ * need a {@link java.io.File}. Closed with the stream if the caller does
not close it first. Does not
* disturb this stream's read position.
*
* @throws IOException if this stream has been partially read without
rewind enabled
@@ -596,7 +603,11 @@ public class TikaInputStream extends TaggedInputStream {
if (source == null) {
throw new IOException("No TikaInputSource available");
}
- return source.getSeekableByteChannel();
+ SeekableByteChannel channel = source.getSeekableByteChannel();
+ if (tmp != null) {
+ tmp.addResource(channel); // a caller that throws before closing
it must not leak it
+ }
+ return channel;
}
/**
diff --git a/tika-core/src/main/java/org/apache/tika/parser/ParseContext.java
b/tika-core/src/main/java/org/apache/tika/parser/ParseContext.java
index 19c17eb046..f82d1ddd9b 100644
--- a/tika-core/src/main/java/org/apache/tika/parser/ParseContext.java
+++ b/tika-core/src/main/java/org/apache/tika/parser/ParseContext.java
@@ -142,14 +142,19 @@ public class ParseContext implements Serializable {
* @since Apache Tika 4.0
*/
public void setJsonConfig(String name, String json) {
- setJsonConfig(name, json != null ? new StringJsonConfig(json) : null);
+ setJsonConfig(name, json, false);
+ }
+
+ /** {@code trusted}: operator-authored (config file, presets), never
per-request input. */
+ public void setJsonConfig(String name, String json, boolean trusted) {
+ setJsonConfig(name, json != null ? new StringJsonConfig(json, trusted)
: null);
}
/**
* A simple Serializable implementation of JsonConfig that holds a JSON
string.
* This is used internally to ensure JSON configs can be serialized via
Java serialization.
*/
- private record StringJsonConfig(String json) implements JsonConfig,
Serializable {
+ private record StringJsonConfig(String json, boolean trusted) implements
JsonConfig, Serializable {
private static final long serialVersionUID = 1L;
}
diff --git
a/tika-core/src/main/java/org/apache/tika/parser/inference/InferenceDispatcher.java
b/tika-core/src/main/java/org/apache/tika/parser/inference/InferenceDispatcher.java
index fc87d0c45c..4187d25e36 100644
---
a/tika-core/src/main/java/org/apache/tika/parser/inference/InferenceDispatcher.java
+++
b/tika-core/src/main/java/org/apache/tika/parser/inference/InferenceDispatcher.java
@@ -63,6 +63,7 @@ import org.apache.tika.parser.hook.ParseHooks;
*/
public final class InferenceDispatcher implements ParseHook,
TransientParseState {
+ static final int MAX_MEDIA_UNITS = 1000;
private static final Logger LOG =
LoggerFactory.getLogger(InferenceDispatcher.class);
private static final InferenceSelection ALL = new InferenceSelection();
@@ -170,17 +171,24 @@ public final class InferenceDispatcher implements
ParseHook, TransientParseState
if (!runs(binding, context) || !binding.accepts(kind, type,
target)) {
continue;
}
- // a media unit is the whole file; its maxBytes bounds one
segment, in the task
+ // a media unit is the whole file; maxBytes and maxChunks bound
its segments, in the task
if (kind != InputKind.MEDIA && binding.getMaxBytes() >= 0 && size
> binding.getMaxBytes()) {
state.dropped.merge(binding.getId() + " over maxBytes", 1,
Integer::sum);
continue;
}
List<InferenceUnit> units =
state.byBinding.computeIfAbsent(binding.getId(),
k -> new ArrayList<>());
- if (binding.getMaxChunks() >= 0 && units.size() >=
binding.getMaxChunks()) {
+ if (kind != InputKind.MEDIA && binding.getMaxChunks() >= 0
+ && units.size() >= binding.getMaxChunks()) {
state.dropped.merge(binding.getId() + " over maxChunks", 1,
Integer::sum);
continue;
}
+ // a media unit is a held copy of a whole file; bound how many one
tree may hold
+ if (kind == InputKind.MEDIA && units.size() >= MAX_MEDIA_UNITS) {
+ state.dropped.merge(binding.getId() + " over " +
MAX_MEDIA_UNITS + " media files",
+ 1, Integer::sum);
+ continue;
+ }
if (unit == null) {
ParseHooks.Run run = context.get(ParseHooks.Run.class);
unit = new InferenceUnit(kind, type, target, parent,
idPath(target, run),
diff --git
a/tika-core/src/test/java/org/apache/tika/io/BoundedInputStreamTest.java
b/tika-core/src/test/java/org/apache/tika/io/BoundedInputStreamTest.java
new file mode 100644
index 0000000000..a0226425a9
--- /dev/null
+++ b/tika-core/src/test/java/org/apache/tika/io/BoundedInputStreamTest.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.io;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import java.io.ByteArrayInputStream;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+
+import org.junit.jupiter.api.Test;
+
+public class BoundedInputStreamTest {
+
+ /** A skip that lies (FileInputStream seeks past EOF) must not move the
bound's position. */
+ @Test
+ public void skipReadsAndStaysWithinTheBound() throws Exception {
+ byte[] data = "0123456789".getBytes(StandardCharsets.US_ASCII);
+ InputStream lying = new ByteArrayInputStream(data) {
+ @Override
+ public long skip(long n) {
+ return n; // claims to skip whatever is asked, like a
FileInputStream past EOF
+ }
+ };
+ BoundedInputStream bis = new BoundedInputStream(5, lying);
+ assertEquals(3, bis.skip(3));
+ assertEquals('3', bis.read());
+ assertEquals(1, bis.skip(100), "bounded: only one byte remains under
the limit");
+ assertEquals(-1, bis.read());
+
+ bis = new BoundedInputStream(8, new ByteArrayInputStream(data));
+ bis.read();
+ bis.mark(10);
+ bis.read();
+ bis.read();
+ bis.reset();
+ assertEquals('1', bis.read(), "reset returns to the marked position");
+ assertEquals(6, bis.skip(100), "and the bound counts from the mark,
not from zero");
+ }
+
+ @Test
+ public void readNBytesHonorsTheBound() throws Exception {
+ byte[] data = "0123456789".getBytes(StandardCharsets.US_ASCII);
+ BoundedInputStream bis = new BoundedInputStream(5, new
ByteArrayInputStream(data));
+ assertEquals("01234", new String(bis.readNBytes(100),
StandardCharsets.US_ASCII));
+ assertEquals(-1, bis.read());
+
+ bis = new BoundedInputStream(5, new ByteArrayInputStream(data));
+ byte[] buf = new byte[10];
+ assertEquals(5, bis.readNBytes(buf, 0, 10));
+ assertEquals(0, bis.readNBytes(buf, 5, 5));
+ assertEquals("01234", new String(buf, 0, 5,
StandardCharsets.US_ASCII));
+ }
+}
diff --git
a/tika-core/src/test/java/org/apache/tika/io/InMemoryContentViewTest.java
b/tika-core/src/test/java/org/apache/tika/io/InMemoryContentViewTest.java
index 5f711113ef..274a0c83ec 100644
--- a/tika-core/src/test/java/org/apache/tika/io/InMemoryContentViewTest.java
+++ b/tika-core/src/test/java/org/apache/tika/io/InMemoryContentViewTest.java
@@ -185,10 +185,12 @@ public class InMemoryContentViewTest {
SeekableByteChannel channel = tis.getSeekableByteChannel();
assertNotNull(TikaInputStream.inMemoryContent(channel), "2MB under
a 64MB budget stays in memory");
assertTrue(budget.getReservedBytes() > 0);
- tis.close();
- assertTrue(budget.getReservedBytes() > 0, "open channel still pins
the reservation");
channel.close();
- assertEquals(0, budget.getReservedBytes(), "last close releases");
+ assertTrue(budget.getReservedBytes() > 0, "the live cache keeps
its reservation");
+ SeekableByteChannel again = tis.getSeekableByteChannel();
+ tis.close();
+ assertFalse(again.isOpen(), "a channel is the stream's to close");
+ assertEquals(0, budget.getReservedBytes(), "closing the stream
releases, pin or no pin");
}
}
diff --git
a/tika-core/src/test/java/org/apache/tika/io/ReopenableSourceTest.java
b/tika-core/src/test/java/org/apache/tika/io/ReopenableSourceTest.java
index 49bdc99443..423945595f 100644
--- a/tika-core/src/test/java/org/apache/tika/io/ReopenableSourceTest.java
+++ b/tika-core/src/test/java/org/apache/tika/io/ReopenableSourceTest.java
@@ -107,17 +107,15 @@ public class ReopenableSourceTest {
}
@Test
- public void testRetainInMemoryRefusedWhenLengthUnknown() throws Exception {
- byte[] data = data(8192);
- AtomicInteger opens = new AtomicInteger();
- ReopenableSource source = new ReopenableSource(countingOpener(data,
opens), tmp, -1, ".bin");
- try {
- assertFalse(source.tryRetainInMemory());
- assertEquals(0, opens.get());
- assertArrayEquals(data, readFully(source));
- } finally {
- source.close();
- }
+ public void testRetainInMemoryWithUnknownLength() throws Exception {
+ // under the floor it is held; over it, with no budget, the attempt
stops at the floor
+ byte[] small = new byte[64 * 1024];
+ ReopenableSource fits = new ReopenableSource(() -> new
ByteArrayInputStream(small), null, -1, "");
+ assertTrue(fits.tryRetainInMemory());
+ assertEquals(small.length, fits.getLength());
+ byte[] big = new byte[2 * 1024 * 1024];
+ ReopenableSource over = new ReopenableSource(() -> new
ByteArrayInputStream(big), null, -1, "");
+ assertFalse(over.tryRetainInMemory());
}
@Test
diff --git
a/tika-core/src/test/java/org/apache/tika/io/TikaInputStreamOwnershipTest.java
b/tika-core/src/test/java/org/apache/tika/io/TikaInputStreamOwnershipTest.java
index 4953a370c3..c8a84ecbcd 100644
---
a/tika-core/src/test/java/org/apache/tika/io/TikaInputStreamOwnershipTest.java
+++
b/tika-core/src/test/java/org/apache/tika/io/TikaInputStreamOwnershipTest.java
@@ -17,6 +17,7 @@
package org.apache.tika.io;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertSame;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -140,21 +141,19 @@ public class TikaInputStreamOwnershipTest {
}
@Test
- public void aPinnedChannelOutlivesTmpAndReleasesOnClose() throws Exception
{
+ public void aPinnedChannelClosesWithTmp() throws Exception {
CacheMemoryBudget budget = budget();
TemporaryResources tmp = new TemporaryResources();
TikaInputStream tis = reopenable(data(BIG), tmp);
tis.enableRewind(budget);
SeekableByteChannel channel = tis.getSeekableByteChannel();
- long reserved = budget.getReservedBytes();
- assertTrue(reserved > 0);
+ assertTrue(budget.getReservedBytes() > 0);
- tmp.close();
+ tmp.close(); // the scope ends without the channel being closed
- assertEquals(reserved, budget.getReservedBytes(),
- "a handed-out channel still pins the array; nothing is
released under it");
- assertEquals(BIG, channel.size());
+ assertFalse(channel.isOpen(), "the channel is tmp's to close");
+ assertEquals(0, budget.getReservedBytes(), "and its pin is released");
channel.close();
- assertEquals(0, budget.getReservedBytes(), "the last pin releases the
reservation");
+ assertEquals(0, budget.getReservedBytes(), "never released twice");
}
}
diff --git
a/tika-core/src/test/java/org/apache/tika/io/TikaInputStreamPathsTest.java
b/tika-core/src/test/java/org/apache/tika/io/TikaInputStreamPathsTest.java
new file mode 100644
index 0000000000..5782cc784f
--- /dev/null
+++ b/tika-core/src/test/java/org/apache/tika/io/TikaInputStreamPathsTest.java
@@ -0,0 +1,400 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.io;
+
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+import java.nio.channels.SeekableByteChannel;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Stream;
+
+import org.apache.commons.io.IOUtils;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import org.apache.tika.metadata.HttpHeaders;
+import org.apache.tika.metadata.Metadata;
+
+/** Happy and unhappy paths through every source: what is released, and what
bytes come back. */
+public class TikaInputStreamPathsTest {
+
+ private static final int MB = 1024 * 1024;
+
+ @TempDir
+ Path dir;
+
+ private static byte[] data(int size) {
+ byte[] d = new byte[size];
+ for (int i = 0; i < size; i++) {
+ d[i] = (byte) (i * 13 + 5);
+ }
+ return d;
+ }
+
+ /** Counts closes; throws after {@code failAt} bytes when that is
positive. */
+ private static final class Probe extends InputStream {
+ final ByteArrayInputStream in;
+ final long failAt;
+ long read;
+ int closes;
+
+ Probe(byte[] d, long failAt) {
+ this.in = new ByteArrayInputStream(d);
+ this.failAt = failAt;
+ }
+
+ @Override
+ public int read() throws IOException {
+ if (failAt > 0 && read >= failAt) {
+ throw new IOException("probe failure at " + read);
+ }
+ int b = in.read();
+ if (b != -1) {
+ read++;
+ }
+ return b;
+ }
+
+ @Override
+ public int read(byte[] b, int off, int len) throws IOException {
+ if (failAt > 0 && read >= failAt) {
+ throw new IOException("probe failure at " + read);
+ }
+ int n = in.read(b, off, failAt > 0 ? (int) Math.min(len, failAt -
read) : len);
+ if (n > 0) {
+ read += n;
+ }
+ return n;
+ }
+
+ @Override
+ public void close() {
+ closes++;
+ }
+ }
+
+ private TemporaryResources tmp() {
+ TemporaryResources tmp = new TemporaryResources();
+ tmp.setTemporaryFileDirectory(dir);
+ return tmp;
+ }
+
+ private long tempFiles() throws IOException {
+ try (Stream<Path> s = Files.list(dir)) {
+ return s.count();
+ }
+ }
+
+ private static byte[] readChannel(SeekableByteChannel ch) throws
IOException {
+ ByteBuffer buf = ByteBuffer.allocate((int) ch.size());
+ ch.position(0);
+ while (buf.hasRemaining() && ch.read(buf) != -1) {
+ // fill
+ }
+ return buf.array();
+ }
+
+ @Test
+ public void failureMidReadReleasesBudgetFilesAndStream() throws Exception {
+ CacheMemoryBudget budget = new CacheMemoryBudget(64L * MB);
+ Probe probe = new Probe(data(3 * MB), 2 * MB);
+ TemporaryResources tmp = tmp();
+ TikaInputStream tis = TikaInputStream.get(probe, tmp, new Metadata());
+ tis.enableRewind(budget);
+ assertThrows(IOException.class, () -> IOUtils.toByteArray(tis));
+ assertTrue(budget.getReservedBytes() > 0, "the cache grew past the
floor before the failure");
+ tis.close();
+ assertEquals(0, budget.getReservedBytes());
+ assertEquals(0, tempFiles());
+ assertEquals(1, probe.closes, "the wrapped stream is closed once");
+ }
+
+ @Test
+ public void spillIsCleanedUpAndReplaysTheSameBytes() throws Exception {
+ byte[] d = data(3 * MB);
+ TemporaryResources tmp = tmp();
+ TikaInputStream tis = TikaInputStream.get(new Probe(d, 0), tmp, new
Metadata());
+ tis.enableRewind(null);
+ tis.mark(1000);
+ assertArrayEquals(d, IOUtils.toByteArray(tis), "first pass");
+ assertTrue(tis.hasFile(), "past the threshold the cache spilled");
+ assertEquals(1, tempFiles());
+ tis.reset();
+ assertEquals(d[0] & 0xff, tis.read(), "reset across the spill lands on
the mark");
+ tis.rewind();
+ assertArrayEquals(d, IOUtils.toByteArray(tis), "second pass, from the
spill");
+ assertEquals(d.length, tis.getLength());
+ assertTrue(tis.hasReliableLength());
+ tis.close();
+ tis.close();
+ assertEquals(0, tempFiles(), "spill deleted on close, double close
harmless");
+ }
+
+ @Test
+ public void channelsFromEverySourceCloseWithTheStream() throws Exception {
+ byte[] d = data(3 * MB);
+ Metadata md = new Metadata();
+ md.set(HttpHeaders.CONTENT_LENGTH, Integer.toString(d.length));
+ // spilled cache
+ TikaInputStream caching = TikaInputStream.get(new Probe(d, 0), tmp(),
new Metadata());
+ caching.enableRewind(null);
+ SeekableByteChannel c1 = caching.getSeekableByteChannel();
+ assertArrayEquals(d, readChannel(c1));
+ caching.close();
+ assertFalse(c1.isOpen(), "a spill-file channel is the stream's to
close");
+ // re-openable, spilled (no budget: over the floor it cannot be
retained)
+ TikaInputStream reopen = TikaInputStream.get(() -> new
ByteArrayInputStream(d), tmp(), md);
+ SeekableByteChannel c2 = reopen.getSeekableByteChannel();
+ assertArrayEquals(d, readChannel(c2));
+ reopen.close();
+ assertFalse(c2.isOpen(), "a re-openable source's spill channel is the
stream's to close");
+ // file
+ Path file = dir.resolve("f.bin");
+ Files.write(file, d);
+ TikaInputStream fs = TikaInputStream.get(file);
+ SeekableByteChannel c3 = fs.getSeekableByteChannel();
+ assertArrayEquals(d, readChannel(c3));
+ fs.close();
+ assertFalse(c3.isOpen(), "a file channel is the stream's to close");
+ }
+
+ @Test
+ public void reopenableFailureMidRetentionReleasesTheReservation() throws
Exception {
+ CacheMemoryBudget budget = new CacheMemoryBudget(64L * MB);
+ byte[] d = data(3 * MB);
+ Metadata md = new Metadata();
+ md.set(HttpHeaders.CONTENT_LENGTH, Integer.toString(d.length));
+ TemporaryResources tmp = tmp();
+ TikaInputStream tis = TikaInputStream.get(() -> new Probe(d, 2 * MB),
tmp, md);
+ tis.enableRewind(budget);
+ assertThrows(IOException.class, tis::tryRetainInMemory);
+ assertEquals(0, budget.getReservedBytes(), "a failed drain returns
what it reserved");
+ tis.close();
+ assertEquals(0, budget.getReservedBytes());
+ assertEquals(0, tempFiles());
+ }
+
+ @Test
+ public void unknownLengthOverTheFloorOpensTheSourceTwice() throws
Exception {
+ byte[] d = data(2 * MB);
+ AtomicInteger opens = new AtomicInteger();
+ TikaInputStream tis = TikaInputStream.get(() -> {
+ opens.incrementAndGet();
+ return new ByteArrayInputStream(d);
+ }, tmp(), new Metadata());
+ try (SeekableByteChannel ch = tis.getSeekableByteChannel()) {
+ assertEquals(d.length, ch.size());
+ }
+ // a doomed in-memory attempt up to the floor, then the spool: known
cost, pinned here
+ assertEquals(2, opens.get());
+ tis.close();
+ }
+
+ @Test
+ public void pinnedChannelSurvivesASpill() throws Exception {
+ CacheMemoryBudget budget = new CacheMemoryBudget(2L * MB);
+ byte[] d = data(4 * MB);
+ TemporaryResources tmp = tmp();
+ TikaInputStream tis = TikaInputStream.get(new Probe(d, 0), tmp, new
Metadata());
+ tis.enableRewind(budget);
+ // the channel drains everything: 4 MB does not fit a 2 MB budget, so
it spills
+ try (SeekableByteChannel ch = tis.getSeekableByteChannel()) {
+ assertArrayEquals(d, readChannel(ch));
+ }
+ assertEquals(0, budget.getReservedBytes(), "spilled: nothing stays
reserved");
+ assertArrayEquals(d, IOUtils.toByteArray(tis), "the stream still reads
the whole source");
+ tis.close();
+ assertEquals(0, tempFiles());
+ }
+
+ @Test
+ public void inMemoryChannelIsClosedWithTheStream() throws Exception {
+ CacheMemoryBudget budget = new CacheMemoryBudget(64L * MB);
+ byte[] d = data(3 * MB);
+ TemporaryResources tmp = tmp();
+ TikaInputStream tis = TikaInputStream.get(new Probe(d, 0), tmp, new
Metadata());
+ tis.enableRewind(budget);
+ SeekableByteChannel ch = tis.getSeekableByteChannel();
+ assertTrue(budget.getReservedBytes() > 0);
+ assertArrayEquals(d, readChannel(ch));
+ tis.close(); // a parser that threw before closing its channel
+ assertFalse(ch.isOpen());
+ assertEquals(0, budget.getReservedBytes(), "the pin goes with the
stream");
+ ch.close();
+ assertEquals(0, budget.getReservedBytes(), "closing it again releases
nothing twice");
+ }
+
+ @Test
+ public void failedReopenDoesNotOrphanTheFreshStream() throws Exception {
+ byte[] full = data(64 * 1024);
+ byte[] truncated = data(100);
+ AtomicInteger opens = new AtomicInteger();
+ Probe[] streams = new Probe[2];
+ TikaInputStream tis = TikaInputStream.get(() -> {
+ int n = opens.getAndIncrement();
+ return streams[n] = new Probe(n == 0 ? full : truncated, 0);
+ }, tmp(), new Metadata());
+ tis.readNBytes(new byte[1000], 0, 1000);
+ tis.mark(10);
+ tis.readNBytes(new byte[32 * 1024], 0, 32 * 1024); // past the
limit: reset must reopen
+ assertThrows(IOException.class, tis::reset, "the re-opened source is
too short to skip to the mark");
+ assertEquals(2, opens.get());
+ assertEquals(1, streams[1].closes, "the stream the failed re-open left
behind is closed");
+ tis.close();
+ }
+
+ @Test
+ public void fileSourceMarkResetAndChannelDoNotMovePosition() throws
Exception {
+ byte[] d = data(64 * 1024);
+ Path file = dir.resolve("g.bin");
+ Files.write(file, d);
+ try (TikaInputStream tis = TikaInputStream.get(file)) {
+ tis.readNBytes(new byte[100], 0, 100);
+ tis.mark(1000);
+ tis.readNBytes(new byte[400], 0, 400);
+ try (SeekableByteChannel ch = tis.getSeekableByteChannel()) {
+ assertEquals(d.length, ch.size());
+ }
+ assertEquals(500, tis.getPosition());
+ tis.reset();
+ assertEquals(d[100] & 0xff, tis.read());
+ assertEquals(101, tis.getPosition());
+ tis.mark(10);
+ tis.readNBytes(new byte[32 * 1024], 0, 32 * 1024); // past the
limit: reopen path
+ tis.reset();
+ assertEquals(d[101] & 0xff, tis.read());
+ }
+ }
+
+ @Test
+ public void passthroughRefusesLateRewindButReadsCorrectly() throws
Exception {
+ byte[] d = data(64 * 1024);
+ TemporaryResources tmp = tmp();
+ TikaInputStream tis = TikaInputStream.get(new Probe(d, 0), tmp, new
Metadata());
+ tis.readNBytes(new byte[10], 0, 10);
+ assertThrows(IOException.class, () -> tis.enableRewind(null), "rewind
must be enabled at 0");
+ assertThrows(IOException.class, tis::getPath, "no spool from
mid-stream");
+ tis.mark(100);
+ tis.readNBytes(new byte[50], 0, 50);
+ tis.reset();
+ assertEquals(d[10] & 0xff, tis.read(), "passthrough mark/reset is the
buffer's");
+ tis.close();
+ assertEquals(0, tempFiles());
+ }
+
+ @Test
+ public void lengthIsMeasuredAtEofWithoutASpill() throws Exception {
+ byte[] d = data(5000);
+ // lying declared length, cache mode
+ Metadata lying = new Metadata();
+ lying.set(HttpHeaders.CONTENT_LENGTH, "4000");
+ TikaInputStream tis = TikaInputStream.get(new Probe(d, 0), tmp(),
lying);
+ tis.enableRewind(null);
+ assertFalse(tis.hasReliableLength());
+ IOUtils.toByteArray(tis);
+ assertEquals(5000, tis.getLength());
+ assertTrue(tis.hasReliableLength());
+ assertEquals(0, tempFiles(), "the cache knows the size; nothing is
spooled to measure it");
+ tis.close();
+ // no declared length, passthrough
+ tis = TikaInputStream.get(new Probe(d, 0), tmp(), new Metadata());
+ IOUtils.toByteArray(tis);
+ assertEquals(5000, tis.getLength());
+ assertTrue(tis.hasReliableLength());
+ assertEquals(0, tempFiles());
+ tis.close();
+ }
+
+ @Test
+ public void unknownLengthIsRetainedWhenItFits() throws Exception {
+ byte[] d = data(500 * 1024);
+ AtomicInteger opens = new AtomicInteger();
+ TikaInputStream tis = TikaInputStream.get(() -> {
+ opens.incrementAndGet();
+ return new ByteArrayInputStream(d);
+ }, tmp(), new Metadata());
+ tis.enableRewind(null);
+ assertTrue(tis.tryRetainInMemory(), "under the floor it is held
without a length");
+ assertArrayEquals(d, IOUtils.toByteArray(tis));
+ tis.rewind();
+ assertArrayEquals(d, IOUtils.toByteArray(tis));
+ assertEquals(1, opens.get(), "digest and parse read the retained
bytes");
+ assertEquals(d.length, tis.getLength());
+ tis.close();
+ }
+
+ @Test
+ public void skipAndNullTmpFollowTheContract() throws Exception {
+ byte[] d = data(1000);
+ Probe probe = new Probe(d, 0);
+ TikaInputStream tis = TikaInputStream.get(probe, null, new Metadata());
+ assertEquals(0, tis.skip(-1));
+ assertEquals(0, tis.skip(0));
+ assertEquals(10, tis.skip(10));
+ assertEquals(d[10] & 0xff, tis.read());
+ tis.close();
+ assertEquals(1, probe.closes);
+ }
+
+ @Test
+ public void spilledReplayByteAtATimeAndTrimmedReservation() throws
Exception {
+ byte[] d = data(3 * MB);
+ TemporaryResources tmp = tmp();
+ TikaInputStream tis = TikaInputStream.get(new Probe(d, 0), tmp, new
Metadata());
+ tis.enableRewind(null); // no budget: spills at 1 MB
+ IOUtils.toByteArray(tis);
+ tis.rewind();
+ for (int i = 0; i < 20000; i++) {
+ assertEquals(d[i] & 0xff, tis.read(), "byte " + i + " from the
spill window");
+ }
+ tis.close();
+
+ CacheMemoryBudget budget = new CacheMemoryBudget(64L * MB);
+ tis = TikaInputStream.get(new Probe(d, 0), tmp(), new Metadata());
+ tis.enableRewind(budget);
+ IOUtils.toByteArray(tis);
+ assertEquals(2L * MB, budget.getReservedBytes(),
+ "after the drain the reservation is the content over the
floor, not the doubled capacity");
+ tis.close();
+ assertEquals(0, budget.getReservedBytes());
+ }
+
+ @Test
+ public void tmpClosedFirstThenStream() throws Exception {
+ CacheMemoryBudget budget = new CacheMemoryBudget(64L * MB);
+ byte[] d = data(3 * MB);
+ TemporaryResources tmp = tmp();
+ TikaInputStream tis = TikaInputStream.get(new Probe(d, 0), tmp, new
Metadata());
+ tis.enableRewind(budget);
+ IOUtils.toByteArray(tis);
+ tmp.close();
+ assertEquals(0, budget.getReservedBytes());
+ assertEquals(0, tempFiles());
+ tis.close();
+ assertEquals(0, budget.getReservedBytes(), "never negative");
+ }
+}
diff --git
a/tika-core/src/test/java/org/apache/tika/parser/ParseContextTest.java
b/tika-core/src/test/java/org/apache/tika/parser/ParseContextTest.java
index 4e709a1cf8..ee8a361653 100644
--- a/tika-core/src/test/java/org/apache/tika/parser/ParseContextTest.java
+++ b/tika-core/src/test/java/org/apache/tika/parser/ParseContextTest.java
@@ -17,13 +17,29 @@
package org.apache.tika.parser;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
import org.junit.jupiter.api.Test;
public class ParseContextTest {
+ @Test
+ public void testJsonConfigTrustSurvivesCopy() {
+ ParseContext operator = new ParseContext();
+ operator.setJsonConfig("x", "{}", true);
+ ParseContext request = new ParseContext();
+ request.setJsonConfig("y", "{}");
+ ParseContext merged = new ParseContext();
+ merged.copyFrom(operator);
+ merged.copyFrom(request);
+ assertTrue(merged.getJsonConfig("x").trusted());
+ assertFalse(merged.getJsonConfig("y").trusted());
+ }
+
+
@Test
public void testCopyFromInvalidatesStaleResolvedConfigs() {
// Simulate a "default" context that has already resolved a config
diff --git
a/tika-core/src/test/java/org/apache/tika/parser/inference/InferenceDispatcherTest.java
b/tika-core/src/test/java/org/apache/tika/parser/inference/InferenceDispatcherTest.java
index 139c6ff7e3..894a42163e 100644
---
a/tika-core/src/test/java/org/apache/tika/parser/inference/InferenceDispatcherTest.java
+++
b/tika-core/src/test/java/org/apache/tika/parser/inference/InferenceDispatcherTest.java
@@ -569,6 +569,42 @@ public class InferenceDispatcherTest {
assertEquals(InputKind.MEDIA, InputKind.of(MediaType.audio("mpeg")));
}
+ @Test
+ public void testMediaMaxChunksIsNotAUnitCap() throws Exception {
+ RecordingTask task = new RecordingTask();
+ InferenceDispatcher dispatcher = new InferenceDispatcher(List.of(
+ new InferenceDispatcher.Bound(InferenceBinding.builder("v",
"engine",
+
InputKind.MEDIA).modality(Modality.AUDIO).maxChunks(1).build(),
+ new RecordingEngine(), List.of(task))));
+ ParseContext context = new ParseContext();
+ MediaType mp3 = MediaType.audio("mpeg");
+ for (int i = 0; i < 3; i++) {
+ dispatcher.offer(mp3, new Metadata(), new Metadata(), file("clip "
+ i), context);
+ }
+ dispatcher.flush(new Metadata(), context);
+ assertEquals(3, task.runs.get(0).size(), "maxChunks caps segments per
file, not files");
+ }
+
+ @Test
+ public void testMediaUnitsAreBoundedPerTree() throws Exception {
+ RecordingTask task = new RecordingTask();
+ InferenceDispatcher dispatcher = new InferenceDispatcher(List.of(
+ new InferenceDispatcher.Bound(InferenceBinding.builder("v",
"engine",
+ InputKind.MEDIA).modality(Modality.AUDIO).build(),
+ new RecordingEngine(), List.of(task))));
+ ParseContext context = new ParseContext();
+ MediaType mp3 = MediaType.audio("mpeg");
+ Path clip = file("clip");
+ for (int i = 0; i < InferenceDispatcher.MAX_MEDIA_UNITS + 5; i++) {
+ dispatcher.offer(mp3, new Metadata(), new Metadata(), clip,
context);
+ }
+ Metadata root = new Metadata();
+ dispatcher.flush(root, context);
+ assertEquals(InferenceDispatcher.MAX_MEDIA_UNITS,
task.runs.get(0).size());
+ assertTrue(String.join(" ",
root.getValues(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING))
+ .contains("media files"), "the drop is reported on the root");
+ }
+
@Test
public void testMediaConfigValidation() throws Exception {
MediaConfig config = new MediaConfig();
diff --git
a/tika-parsers/tika-http-jdk/src/main/java/org/apache/tika/http/TikaHttpClient.java
b/tika-parsers/tika-http-jdk/src/main/java/org/apache/tika/http/TikaHttpClient.java
index f15da1652d..8688a72241 100644
---
a/tika-parsers/tika-http-jdk/src/main/java/org/apache/tika/http/TikaHttpClient.java
+++
b/tika-parsers/tika-http-jdk/src/main/java/org/apache/tika/http/TikaHttpClient.java
@@ -127,6 +127,10 @@ public class TikaHttpClient implements Closeable {
return new TikaHttpClient(client, executor, connectTimeoutSeconds,
maxRetries);
}
+ public int getMaxRetries() {
+ return maxRetries;
+ }
+
/**
* POST a JSON body to {@code url} and return the response body as a
string.
* <p>
diff --git
a/tika-parsers/tika-parsers-ml/tika-parser-tess4j-module/src/main/java/org/apache/tika/parser/ocr/tess4j/Tess4JConfig.java
b/tika-parsers/tika-parsers-ml/tika-parser-tess4j-module/src/main/java/org/apache/tika/parser/ocr/tess4j/Tess4JConfig.java
index b8517355a9..fb29d4374e 100644
---
a/tika-parsers/tika-parsers-ml/tika-parser-tess4j-module/src/main/java/org/apache/tika/parser/ocr/tess4j/Tess4JConfig.java
+++
b/tika-parsers/tika-parsers-ml/tika-parser-tess4j-module/src/main/java/org/apache/tika/parser/ocr/tess4j/Tess4JConfig.java
@@ -294,7 +294,7 @@ public class Tess4JConfig implements Serializable {
final String[] langs = language.split("\\+");
for (String lang : langs) {
if (!lang.matches(
-
"([a-zA-Z]{3}(_[a-zA-Z]{3,4}){0,2})|script(/|\\\\)[A-Z][a-zA-Z_]+")) {
+
"([a-zA-Z]{3}(_[a-zA-Z]{3,4}){0,2})|(script(/|\\\\))?[A-Z][a-zA-Z_]+")) {
invalidLangs.add(lang + " (invalid syntax)");
} else {
validLangs.add(lang);
diff --git
a/tika-parsers/tika-parsers-ml/tika-parser-tess4j-module/src/test/java/org/apache/tika/parser/ocr/tess4j/Tess4JConfigTest.java
b/tika-parsers/tika-parsers-ml/tika-parser-tess4j-module/src/test/java/org/apache/tika/parser/ocr/tess4j/Tess4JConfigTest.java
index 44c82e6882..3a4f31d455 100644
---
a/tika-parsers/tika-parsers-ml/tika-parser-tess4j-module/src/test/java/org/apache/tika/parser/ocr/tess4j/Tess4JConfigTest.java
+++
b/tika-parsers/tika-parsers-ml/tika-parser-tess4j-module/src/test/java/org/apache/tika/parser/ocr/tess4j/Tess4JConfigTest.java
@@ -55,6 +55,17 @@ public class Tess4JConfigTest {
assertThrows(IllegalArgumentException.class, () ->
config.setLanguage("xy"));
}
+ @Test
+ public void testScriptModelsByBareName() {
+ Tess4JConfig config = new Tess4JConfig();
+ for (String ok : new String[]{"Latin", "eng+Latin", "script/Latin",
"Japanese_vert"}) {
+ config.setLanguage(ok);
+ assertEquals(ok, config.getLanguage());
+ }
+ assertThrows(IllegalArgumentException.class, () ->
config.setLanguage("latin1"));
+ assertThrows(IllegalArgumentException.class, () ->
config.setLanguage("script/"));
+ }
+
@Test
public void testSetLanguageLeadingPlus() {
Tess4JConfig config = new Tess4JConfig();
diff --git
a/tika-parsers/tika-parsers-ml/tika-vlm/src/main/java/org/apache/tika/parser/vlm/AbstractVLMParser.java
b/tika-parsers/tika-parsers-ml/tika-vlm/src/main/java/org/apache/tika/parser/vlm/AbstractVLMParser.java
index 95690f4e02..c9e4fd4ad2 100644
---
a/tika-parsers/tika-parsers-ml/tika-vlm/src/main/java/org/apache/tika/parser/vlm/AbstractVLMParser.java
+++
b/tika-parsers/tika-parsers-ml/tika-vlm/src/main/java/org/apache/tika/parser/vlm/AbstractVLMParser.java
@@ -95,7 +95,14 @@ public abstract class AbstractVLMParser implements Parser,
Initializable, Closea
protected AbstractVLMParser(VLMOCRConfig config) {
this.defaultConfig = config;
- this.httpClient = buildHttpClient();
+ }
+
+ /** Built on first use, so the setters that feed it have run. */
+ synchronized TikaHttpClient httpClient() {
+ if (httpClient == null) {
+ httpClient = buildHttpClient();
+ }
+ return httpClient;
}
// ---- abstract contract for subclasses ---------------------------------
@@ -220,7 +227,7 @@ public abstract class AbstractVLMParser implements Parser,
Initializable, Closea
String responseText;
try {
- String responseBody = httpClient.postJson(
+ String responseBody = httpClient().postJson(
call.url(), call.json(), call.headers(),
config.getTimeoutMillis(), parseContext);
responseText = extractResponseText(responseBody, metadata);
ParseTimeout.checkpoint(parseContext);
@@ -244,9 +251,6 @@ public abstract class AbstractVLMParser implements Parser,
Initializable, Closea
@Override
public void initialize() throws TikaConfigException {
- if (httpClient == null) {
- httpClient = buildHttpClient();
- }
String healthUrl = getHealthCheckUrl(defaultConfig);
if (healthUrl == null) {
// No health check configured (e.g. Claude) — assume available
@@ -257,7 +261,7 @@ public abstract class AbstractVLMParser implements Parser,
Initializable, Closea
Map<String, String> healthHeaders = defaultConfig.getApiKey() !=
null
? Map.of("Authorization", "Bearer " +
defaultConfig.getApiKey())
: Map.of();
- httpClient.get(healthUrl, healthHeaders,
defaultConfig.getTimeoutMillis());
+ httpClient().get(healthUrl, healthHeaders,
defaultConfig.getTimeoutMillis());
serverAvailable = true;
LOG.info("VLM server is available at {}",
defaultConfig.getBaseUrl());
} catch (TikaException e) {
@@ -430,8 +434,12 @@ public abstract class AbstractVLMParser implements Parser,
Initializable, Closea
}
/** Retries of a 429/502/503/504 answer from the service; 0 fails at once.
*/
- public void setMaxRetries(int maxRetries) {
+ public synchronized void setMaxRetries(int maxRetries) {
defaultConfig.setMaxRetries(maxRetries);
+ if (httpClient != null) {
+ httpClient.close();
+ httpClient = null;
+ }
}
public String getApiKey() {
diff --git
a/tika-parsers/tika-parsers-ml/tika-vlm/src/main/java/org/apache/tika/parser/vlm/MarkdownToXHTMLEmitter.java
b/tika-parsers/tika-parsers-ml/tika-vlm/src/main/java/org/apache/tika/parser/vlm/MarkdownToXHTMLEmitter.java
index fe306231f9..9c1073ce65 100644
---
a/tika-parsers/tika-parsers-ml/tika-vlm/src/main/java/org/apache/tika/parser/vlm/MarkdownToXHTMLEmitter.java
+++
b/tika-parsers/tika-parsers-ml/tika-vlm/src/main/java/org/apache/tika/parser/vlm/MarkdownToXHTMLEmitter.java
@@ -243,8 +243,13 @@ class MarkdownToXHTMLEmitter {
@Override
public void visit(HtmlBlock htmlBlock) {
Element body =
Jsoup.parseBodyFragment(htmlBlock.getLiteral()).body();
+ org.jsoup.nodes.Node previous = null;
for (org.jsoup.nodes.Node child : body.childNodes()) {
+ if (previous instanceof TextNode && child instanceof TextNode)
{
+ characters(" "); // orphaned cells: jsoup dropped the
tags between them
+ }
emitHtml(child, false);
+ previous = child;
}
}
@@ -274,6 +279,8 @@ class MarkdownToXHTMLEmitter {
}
if (emit) {
endElement(tag);
+ } else if (((Element) node).isBlock()) {
+ characters(" "); // a dropped block still separates its text
from the next
}
}
diff --git
a/tika-parsers/tika-parsers-ml/tika-vlm/src/test/java/org/apache/tika/parser/vlm/MarkdownToXHTMLEmitterTest.java
b/tika-parsers/tika-parsers-ml/tika-vlm/src/test/java/org/apache/tika/parser/vlm/MarkdownToXHTMLEmitterTest.java
index afc03b2e71..d5d421238f 100644
---
a/tika-parsers/tika-parsers-ml/tika-vlm/src/test/java/org/apache/tika/parser/vlm/MarkdownToXHTMLEmitterTest.java
+++
b/tika-parsers/tika-parsers-ml/tika-vlm/src/test/java/org/apache/tika/parser/vlm/MarkdownToXHTMLEmitterTest.java
@@ -40,6 +40,16 @@ public class MarkdownToXHTMLEmitterTest {
assertContains("<h3>Heading 3</h3>", xml);
}
+ @Test
+ void testDroppedElementsStillSeparateText() throws Exception {
+ String split =
"<table>\n<tr><td>North</td><td>120</td></tr>\n\n<tr><td>South</td><td>99</td></tr>\n</table>";
+ String xml = emit(split);
+ assertContains("<td>North</td>", xml);
+ assertContains("South 99", xml);
+ assertContains("k v", emit("<dl><dt>k</dt><dd>v</dd></dl>"));
+ assertContains("a b", emit("<div>a</div><div>b</div>"));
+ }
+
@Test
void testParagraph() throws Exception {
String md = "Hello world.\n\nSecond paragraph.";
diff --git
a/tika-parsers/tika-parsers-ml/tika-vlm/src/test/java/org/apache/tika/parser/vlm/OpenAIVLMParserTest.java
b/tika-parsers/tika-parsers-ml/tika-vlm/src/test/java/org/apache/tika/parser/vlm/OpenAIVLMParserTest.java
index b4ee3f6637..6d1f28da99 100644
---
a/tika-parsers/tika-parsers-ml/tika-vlm/src/test/java/org/apache/tika/parser/vlm/OpenAIVLMParserTest.java
+++
b/tika-parsers/tika-parsers-ml/tika-vlm/src/test/java/org/apache/tika/parser/vlm/OpenAIVLMParserTest.java
@@ -42,6 +42,15 @@ import org.apache.tika.sax.BodyContentHandler;
public class OpenAIVLMParserTest {
+ @Test
+ public void testSetMaxRetriesReachesTheClient() throws Exception {
+ OpenAIVLMParser parser = new OpenAIVLMParser();
+ assertEquals(4, parser.httpClient().getMaxRetries());
+ parser.setMaxRetries(0);
+ assertEquals(0, parser.httpClient().getMaxRetries());
+ }
+
+
private static final ObjectMapper MAPPER = new ObjectMapper();
private TikaTestHttpServer server;
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/main/java/org/apache/tika/detect/ogg/OggDetector.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/main/java/org/apache/tika/detect/ogg/OggDetector.java
index ab92888b4c..434cd3d0f7 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/main/java/org/apache/tika/detect/ogg/OggDetector.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/main/java/org/apache/tika/detect/ogg/OggDetector.java
@@ -34,6 +34,7 @@ import org.gagravarr.skeleton.SkeletonPacketFactory;
import org.apache.tika.annotation.TikaComponent;
import org.apache.tika.detect.Detector;
+import org.apache.tika.io.CacheMemoryBudget;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.mime.MediaType;
@@ -76,7 +77,10 @@ public class OggDetector implements Detector {
// We could potentially need to go a long way through the
// file in order to figure out what it is
- tis.mark((int)tis.getLength() + 1);
+ // rewind-enabled, the mark is a position in a cache that spills past
its budget,
+ // not a buffer sized by a limit; the limit itself is never taken from
the length
+ tis.enableRewind(parseContext == null ? null :
parseContext.get(CacheMemoryBudget.class));
+ tis.mark(Integer.MAX_VALUE);
try {
// Open the Ogg file - underlying stream stays open as detecting
only
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/ImageParser.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/ImageParser.java
index 733648ddb7..2fe3cf9402 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/ImageParser.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/ImageParser.java
@@ -39,6 +39,7 @@ import org.xml.sax.SAXException;
import org.apache.tika.annotation.TikaComponent;
import org.apache.tika.exception.TikaException;
+import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.HttpHeaders;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.metadata.Property;
@@ -170,8 +171,8 @@ public class ImageParser extends AbstractImageParser {
if (iterator.hasNext()) {
ImageReader reader = iterator.next();
try {
- try (ImageInputStream imageStream = ImageIO
-
.createImageInputStream(CloseShieldInputStream.wrap(stream))) {
+ // a file we already have beats ImageIO copying the stream
to its own cache file
+ try (ImageInputStream imageStream = imageInput(stream)) {
reader.setInput(imageStream);
try {
int numImages = reader.getNumImages(true);
@@ -209,6 +210,13 @@ public class ImageParser extends AbstractImageParser {
}
}
+ private static ImageInputStream imageInput(InputStream stream) throws
IOException {
+ if (stream instanceof TikaInputStream tis && tis.hasFile()) {
+ return ImageIO.createImageInputStream(tis.getPath().toFile());
+ }
+ return
ImageIO.createImageInputStream(CloseShieldInputStream.wrap(stream));
+ }
+
@Override
MediaType normalizeMediaType(MediaType mediaType) {
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/RawTiffParser.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/RawTiffParser.java
index 825e2cc2f0..8bacdfec00 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/RawTiffParser.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/RawTiffParser.java
@@ -168,7 +168,8 @@ public class RawTiffParser extends TiffParser {
@Override
protected ImageMetadataConfig getImageMetadataConfig(ParseContext context)
throws TikaException, IOException {
- return getConfig(context);
+ ImageMetadataConfig shared = context.get(ImageMetadataConfig.class);
+ return shared != null ? shared : getConfig(context);
}
private void extractPreviews(TikaInputStream tis, XHTMLContentHandler
xhtml, Metadata metadata,
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/test/java/org/apache/tika/parser/image/RawTiffParserTest.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/test/java/org/apache/tika/parser/image/RawTiffParserTest.java
index deb7b0f17c..a658d8622e 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/test/java/org/apache/tika/parser/image/RawTiffParserTest.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/test/java/org/apache/tika/parser/image/RawTiffParserTest.java
@@ -17,6 +17,7 @@
package org.apache.tika.parser.image;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -38,6 +39,17 @@ import org.apache.tika.parser.Parser;
public class RawTiffParserTest extends TikaTest {
+ @Test
+ public void testSharedImageMetadataConfigApplies() throws Exception {
+ ImageMetadataConfig shared = new ImageMetadataConfig();
+ shared.setIncludeIccCurvesAndLuts(true);
+ ParseContext context = new ParseContext();
+ context.set(ImageMetadataConfig.class, shared);
+ assertTrue(new
RawTiffParser().getImageMetadataConfig(context).isIncludeIccCurvesAndLuts());
+ assertFalse(new RawTiffParser().getImageMetadataConfig(new
ParseContext())
+ .isIncludeIccCurvesAndLuts());
+ }
+
private List<Metadata> parseByName(String fileName) throws Exception {
Metadata metadata = new Metadata();
metadata.set(TikaCoreProperties.RESOURCE_NAME_KEY, fileName);
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/extractor/microsoft/MSEmbeddedStreamTranslator.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/extractor/microsoft/MSEmbeddedStreamTranslator.java
index fd4c47a99f..826a6d9bb4 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/extractor/microsoft/MSEmbeddedStreamTranslator.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/extractor/microsoft/MSEmbeddedStreamTranslator.java
@@ -16,13 +16,13 @@
*/
package org.apache.tika.extractor.microsoft;
+import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import org.apache.commons.io.IOUtils;
import org.apache.commons.io.output.CloseShieldOutputStream;
-import org.apache.commons.io.output.UnsynchronizedByteArrayOutputStream;
import org.apache.poi.poifs.filesystem.DirectoryEntry;
import org.apache.poi.poifs.filesystem.DocumentEntry;
import org.apache.poi.poifs.filesystem.DocumentInputStream;
@@ -58,13 +58,11 @@ public class MSEmbeddedStreamTranslator implements
EmbeddedStreamTranslator {
public void translate(TikaInputStream tis, Metadata metadata, OutputStream
os) throws IOException {
String contentType =
metadata.get(org.apache.tika.metadata.HttpHeaders.CONTENT_TYPE);
if
("application/vnd.openxmlformats-officedocument.oleObject".equals(contentType))
{
- UnsynchronizedByteArrayOutputStream bos =
UnsynchronizedByteArrayOutputStream.builder().get();
- IOUtils.copy(tis, bos);
- POIFSFileSystem poifs = new POIFSFileSystem(bos.toInputStream());
+ byte[] data = IOUtils.toByteArray(tis);
+ POIFSFileSystem poifs = new POIFSFileSystem(new
ByteArrayInputStream(data));
OfficeParser.POIFSDocumentType type =
OfficeParser.POIFSDocumentType.detectType(poifs);
String name = metadata.get(TikaCoreProperties.RESOURCE_NAME_KEY);
- byte[] data = bos.toByteArray();
if (type == OfficeParser.POIFSDocumentType.OLE10_NATIVE) {
try {
Ole10Native ole =
Ole10Native.createFromEmbeddedOleObject(poifs);
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/AbstractOOXMLExtractor.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/AbstractOOXMLExtractor.java
index 96acd30d40..11d611a672 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/AbstractOOXMLExtractor.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/AbstractOOXMLExtractor.java
@@ -225,7 +225,7 @@ public abstract class AbstractOOXMLExtractor implements
OOXMLExtractor {
if (embeddedExtractor.shouldParseEmbedded(thumbnailMetadata,
context)) {
try (TikaInputStream tis =
TikaInputStream.get(tPart::getInputStream,
- new TemporaryResources(), null)) {
+ new TemporaryResources(), sized(tPart))) {
embeddedExtractor.parseEmbedded(tis,
new EmbeddedContentHandler(handler),
thumbnailMetadata, context, false);
}
@@ -510,7 +510,7 @@ public abstract class AbstractOOXMLExtractor implements
OOXMLExtractor {
//the part is in the package already: re-open it on rewind instead
of
//caching a copy that a digest of a large part would spill to disk
try (TikaInputStream tis =
TikaInputStream.get(part::getInputStream,
- new TemporaryResources(), null)) {
+ new TemporaryResources(), sized(part))) {
embeddedExtractor
.parseEmbedded(tis, xhtml, metadata, context, true);
}
@@ -730,4 +730,14 @@ public abstract class AbstractOOXMLExtractor implements
OOXMLExtractor {
}
+
+ /** The part's size, so the stream can be retained and its length is known
without a read. */
+ private static Metadata sized(PackagePart part) {
+ Metadata m = new Metadata();
+ long size = part.getSize();
+ if (size >= 0) {
+ m.set(HttpHeaders.CONTENT_LENGTH, Long.toString(size));
+ }
+ return m;
+ }
}
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-ocr-module/src/main/java/org/apache/tika/parser/ocr/TesseractOCRParser.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-ocr-module/src/main/java/org/apache/tika/parser/ocr/TesseractOCRParser.java
index 35cc780bb7..1a8bee22e9 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-ocr-module/src/main/java/org/apache/tika/parser/ocr/TesseractOCRParser.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-ocr-module/src/main/java/org/apache/tika/parser/ocr/TesseractOCRParser.java
@@ -304,23 +304,20 @@ public class TesseractOCRParser extends
AbstractExternalProcessParser
}
}
- private TesseractOCRConfig getConfig(ParseContext parseContext) throws
TikaConfigException, IOException {
- // Check for JSON config with component-specific runtime config
+ TesseractOCRConfig getConfig(ParseContext parseContext) throws
TikaConfigException, IOException {
if (parseContext.hasJsonConfig("tesseract-ocr-parser")) {
- // First validate that no paths are being set by deserializing
into RuntimeConfig
- // The RuntimeConfig setters will throw TikaConfigException if
JSON contains path fields
- TesseractOCRConfig.RuntimeConfig runtimeConfig =
ParseContextConfig.getConfig(
- parseContext,
- "tesseract-ocr-parser",
- TesseractOCRConfig.RuntimeConfig.class,
- new TesseractOCRConfig.RuntimeConfig());
-
- // Short-circuit if skipOcr is set
- if (runtimeConfig.isSkipOcr()) {
- return runtimeConfig;
+ // per-request JSON is screened by RuntimeConfig (no paths, no
otherTesseractConfig);
+ // operator JSON (config file, presets) is not
+ if (!parseContext.getJsonConfig("tesseract-ocr-parser").trusted())
{
+ TesseractOCRConfig.RuntimeConfig runtimeConfig =
ParseContextConfig.getConfig(
+ parseContext,
+ "tesseract-ocr-parser",
+ TesseractOCRConfig.RuntimeConfig.class,
+ new TesseractOCRConfig.RuntimeConfig());
+ if (runtimeConfig.isSkipOcr()) {
+ return runtimeConfig;
+ }
}
-
- // Now merge with defaultConfig using the base class
return ParseContextConfig.getConfig(
parseContext,
"tesseract-ocr-parser",
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-ocr-module/src/test/java/org/apache/tika/parser/ocr/TesseractOCRConfigTrustTest.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-ocr-module/src/test/java/org/apache/tika/parser/ocr/TesseractOCRConfigTrustTest.java
new file mode 100644
index 0000000000..04eab0e81a
--- /dev/null
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-ocr-module/src/test/java/org/apache/tika/parser/ocr/TesseractOCRConfigTrustTest.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.ocr;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import org.junit.jupiter.api.Test;
+
+import org.apache.tika.parser.ParseContext;
+
+/** otherTesseractConfig is accepted from operator JSON (config file,
presets), refused per request. */
+public class TesseractOCRConfigTrustTest {
+
+ private static final String JSON =
+ "{\"otherTesseractConfig\": {\"tessedit_char_whitelist\":
\"0123456789\"}}";
+
+ @Test
+ public void operatorJsonIsAccepted() throws Exception {
+ ParseContext context = new ParseContext();
+ context.setJsonConfig("tesseract-ocr-parser", JSON, true);
+ assertEquals("0123456789", new TesseractOCRParser().getConfig(context)
+ .getOtherTesseractConfig().get("tessedit_char_whitelist"));
+ }
+
+ @Test
+ public void requestJsonIsRefused() {
+ ParseContext context = new ParseContext();
+ context.setJsonConfig("tesseract-ocr-parser", JSON);
+ Exception e = assertThrows(Exception.class, () -> new
TesseractOCRParser().getConfig(context));
+ assertTrue(String.valueOf(e).contains("otherTesseractConfig")
+ ||
String.valueOf(e.getCause()).contains("otherTesseractConfig"),
String.valueOf(e));
+ }
+}
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/AbstractPDF2XHTML.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/AbstractPDF2XHTML.java
index 7f1018c8a2..cb73cdc034 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/AbstractPDF2XHTML.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/AbstractPDF2XHTML.java
@@ -631,8 +631,12 @@ class AbstractPDF2XHTML extends PDFTextStripper {
}
//the file is in the document already: re-open (re-decode) it on rewind
//rather than cache a copy that a digest of a large attachment would
spill
+ Metadata sized = new Metadata();
+ if (pdEmbeddedFile.getSize() >= 0) {
+ sized.set(HttpHeaders.CONTENT_LENGTH,
Long.toString(pdEmbeddedFile.getSize()));
+ }
TikaInputStream tis =
TikaInputStream.get(pdEmbeddedFile::createInputStream,
- new TemporaryResources(), null);
+ new TemporaryResources(), sized);
setOrReplaceAttribute("class", "embedded", attributes);
setOrReplaceAttribute("id", fileName, attributes);
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/test/java/org/apache/tika/parser/pdf/PDFPagesEmitTest.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/test/java/org/apache/tika/parser/pdf/PDFPagesEmitTest.java
index e89686e55b..d02ceb5001 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/test/java/org/apache/tika/parser/pdf/PDFPagesEmitTest.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/test/java/org/apache/tika/parser/pdf/PDFPagesEmitTest.java
@@ -368,6 +368,47 @@ public class PDFPagesEmitTest extends TikaTest {
() -> parser.parse(tis, new ToXMLContentHandler(),
metadata, context));
}
assertEquals(2, calls[0], "once per page: OCR asked, emission did not
ask again");
+ assertEquals(2,
metadata.getValues(Rendering.RENDER_FAILED_PAGE).length,
+ "one failure per page, none re-recorded after the parse
threw");
+ }
+
+ /** A page whose render already failed is not rendered again when the
writer then dies on it. */
+ @Test
+ public void testFailedPageIsNotRetriedWhenTheWriterDies() throws Exception
{
+ PDFParserConfig config = emitting();
+ config.pages().setText(TextPolicy.EXTRACT_AND_OCR);
+ int[] calls = new int[1];
+ PDFParser parser = new PDFParser();
+ parser.setRenderer(new PDFBoxRenderer() {
+ @Override
+ public RenderResults render(TikaInputStream tis, Metadata metadata,
+ ParseContext parseContext,
RenderRequest... requests)
+ throws IOException {
+ calls[0]++;
+ throw new IOException("engine down");
+ }
+ });
+ parser.setContentEnrichers(new CompositeContentEnricher(List.of(new
ImageSink())));
+ ParseContext context = new ParseContext();
+ context.set(PDFParserConfig.class, config);
+ context.set(Parser.class, new AutoDetectParser(new ImageSink()));
+ Metadata metadata = new Metadata();
+ ContentHandler dying = new ToXMLContentHandler() {
+ int pageDivs = 0;
+ @Override
+ public void endElement(String uri, String localName, String qName)
throws SAXException {
+ super.endElement(uri, localName, qName);
+ if ("div".equals(localName) && ++pageDivs == 2) {
+ throw new SAXException("writer died on page 2");
+ }
+ }
+ };
+ try (TikaInputStream tis = TikaInputStream.get(
+ getResourceAsStream("/test-documents/" + TWO_PAGES))) {
+ assertThrows(Exception.class, () -> parser.parse(tis, dying,
metadata, context));
+ }
+ assertEquals(2, calls[0], "page 2's failed render is not attempted
again");
+ assertEquals(2,
metadata.getValues(Rendering.RENDER_FAILED_PAGE).length);
}
/**
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/main/java/org/apache/tika/parser/pkg/SevenZParser.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/main/java/org/apache/tika/parser/pkg/SevenZParser.java
index b0c97c1973..a57c8fbca4 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/main/java/org/apache/tika/parser/pkg/SevenZParser.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/main/java/org/apache/tika/parser/pkg/SevenZParser.java
@@ -113,42 +113,27 @@ public class SevenZParser extends AbstractArchiveParser {
password = provider.getPassword(metadata);
}
- SevenZFile sevenZFile;
- // SevenZFile.close() closes the channel it was built on
- SeekableByteChannel channel = tis.getSeekableByteChannel();
- try {
- // Use setMaxMemoryLimitKiB (direct KiB); setMaxMemoryLimitKb
divides the arg by 1024.
- SevenZFile.Builder builder = new SevenZFile.Builder()
- .setSeekableByteChannel(channel)
- .setMaxMemoryLimitKiB(defaultConfig.getMemoryLimitInKb());
- if (password == null) {
- sevenZFile = builder.get();
- } else {
- sevenZFile = builder.setPassword(password.toCharArray()).get();
- }
- } catch (PasswordRequiredException e) {
- channel.close();
- throw new EncryptedDocumentException(e);
- } catch (MemoryLimitException e) {
- // The limit can be exceeded at open time (assertValidity) as well
as lazily on the
- // first getNextEntry() when the LZMA/LZMA2 dictionary is
allocated.
- channel.close();
- throw new TikaMemoryLimitException(e.getMessage());
- } catch (IOException | RuntimeException e) {
- // commons-compress throws unchecked exceptions on corrupt headers
- channel.close();
- throw e;
- }
-
metadata.set(HttpHeaders.CONTENT_TYPE, SEVENZ.toString());
EmbeddedDocumentExtractor extractor =
EmbeddedDocumentUtil.getEmbeddedDocumentExtractor(context);
XHTMLContentHandler xhtml = new XHTMLContentHandler(handler, metadata,
context);
- xhtml.startDocument();
+ // SevenZFile.close() closes the channel it was built on; until it
exists the channel
+ // is ours to close, and the SAX document is not started until the
archive opened
+ SeekableByteChannel channel = tis.getSeekableByteChannel();
+ SevenZFile sevenZFile = null;
+ boolean started = false;
try {
+ // Use setMaxMemoryLimitKiB (direct KiB); setMaxMemoryLimitKb
divides the arg by 1024.
+ SevenZFile.Builder builder = new SevenZFile.Builder()
+ .setSeekableByteChannel(channel)
+ .setMaxMemoryLimitKiB(defaultConfig.getMemoryLimitInKb());
+ sevenZFile = password == null ? builder.get()
+ : builder.setPassword(password.toCharArray()).get();
+ xhtml.startDocument();
+ started = true;
SevenZArchiveEntry entry = sevenZFile.getNextEntry();
while (entry != null) {
if (!entry.isDirectory()) {
@@ -159,10 +144,17 @@ public class SevenZParser extends AbstractArchiveParser {
} catch (PasswordRequiredException e) {
throw new EncryptedDocumentException(e);
} catch (MemoryLimitException e) {
+ // at open time (assertValidity) or lazily on the first
getNextEntry()
throw new TikaMemoryLimitException(e.getMessage());
} finally {
- sevenZFile.close();
- xhtml.endDocument();
+ if (sevenZFile != null) {
+ sevenZFile.close();
+ } else {
+ channel.close();
+ }
+ if (started) {
+ xhtml.endDocument();
+ }
}
}
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/main/java/org/apache/tika/parser/pkg/ZipParser.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/main/java/org/apache/tika/parser/pkg/ZipParser.java
index eb3ed69419..d0e012ba11 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/main/java/org/apache/tika/parser/pkg/ZipParser.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/main/java/org/apache/tika/parser/pkg/ZipParser.java
@@ -52,6 +52,7 @@ import org.apache.tika.exception.TikaConfigException;
import org.apache.tika.exception.TikaException;
import org.apache.tika.extractor.EmbeddedDocumentExtractor;
import org.apache.tika.extractor.EmbeddedDocumentUtil;
+import org.apache.tika.io.CacheMemoryBudget;
import org.apache.tika.io.TemporaryResources;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.HttpHeaders;
@@ -242,7 +243,7 @@ public class ZipParser extends AbstractArchiveParser {
parseWithZipFile(zipFile, tis, handler, metadata, context, config);
} else {
// Use streaming - enable rewind for DATA_DESCRIPTOR retry
- tis.enableRewind();
+ tis.enableRewind(context.get(CacheMemoryBudget.class));
String dataDescriptorRequired =
metadata.get(Zip.DETECTOR_DATA_DESCRIPTOR_REQUIRED);
parseWithStream(tis, handler, metadata, context, config,
"true".equals(dataDescriptorRequired));
@@ -294,7 +295,7 @@ public class ZipParser extends AbstractArchiveParser {
// Perform integrity check if enabled
if (config.isIntegrityCheck()) {
- tis.enableRewind();
+ tis.enableRewind(context.get(CacheMemoryBudget.class));
tis.rewind();
performIntegrityCheck(tis, metadata, centralDirectoryEntries,
config);
}
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/test/java/org/apache/tika/parser/pkg/PackageEmbeddedBudgetTest.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/test/java/org/apache/tika/parser/pkg/PackageEmbeddedBudgetTest.java
index e73e9b1fd4..2c82171155 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/test/java/org/apache/tika/parser/pkg/PackageEmbeddedBudgetTest.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/test/java/org/apache/tika/parser/pkg/PackageEmbeddedBudgetTest.java
@@ -19,15 +19,22 @@ package org.apache.tika.parser.pkg;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
+import java.io.OutputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Base64;
import java.util.List;
+import org.apache.commons.compress.archivers.ArchiveOutputStream;
+import org.apache.commons.compress.archivers.ar.ArArchiveEntry;
+import org.apache.commons.compress.archivers.ar.ArArchiveOutputStream;
+import org.apache.commons.compress.archivers.cpio.CpioArchiveEntry;
+import org.apache.commons.compress.archivers.cpio.CpioArchiveOutputStream;
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
-import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
import org.apache.tika.digest.DigesterFactory;
import org.apache.tika.digest.InputStreamDigester;
@@ -47,41 +54,64 @@ public class PackageEmbeddedBudgetTest extends
AbstractPkgTest {
@TempDir
Path tmp;
+ private static byte[] entry(int i) {
+ byte[] data = new byte[2 * 1024 * 1024];
+ for (int j = 0; j < data.length; j++) {
+ data[j] = (byte) ('a' + (j + i) % 26);
+ }
+ return data;
+ }
+
/** Three entries over the 1 MB StreamCache threshold, so each one
reserves budget. */
- private Path buildTar() throws Exception {
- Path tar = tmp.resolve("entries.tar");
- try (TarArchiveOutputStream out =
- new TarArchiveOutputStream(Files.newOutputStream(tar))) {
+ private Path build(String ext) throws Exception {
+ Path archive = tmp.resolve("entries." + ext);
+ try (OutputStream os = Files.newOutputStream(archive);
+ ArchiveOutputStream<?> out = switch (ext) {
+ case "tar" -> new TarArchiveOutputStream(os);
+ case "cpio" -> new CpioArchiveOutputStream(os);
+ case "ar" -> new ArArchiveOutputStream(os);
+ default -> throw new IllegalArgumentException(ext);
+ }) {
for (int i = 0; i < 3; i++) {
- byte[] data = new byte[2 * 1024 * 1024];
- for (int j = 0; j < data.length; j++) {
- data[j] = (byte) ('a' + (j + i) % 26);
+ byte[] data = entry(i);
+ String name = "entry-" + i + ".txt";
+ switch (ext) {
+ case "tar" -> {
+ TarArchiveEntry e = new TarArchiveEntry(name);
+ e.setSize(data.length);
+ ((TarArchiveOutputStream) out).putArchiveEntry(e);
+ }
+ case "cpio" -> {
+ CpioArchiveEntry e = new CpioArchiveEntry(name);
+ e.setSize(data.length);
+ ((CpioArchiveOutputStream) out).putArchiveEntry(e);
+ }
+ default -> ((ArArchiveOutputStream) out)
+ .putArchiveEntry(new ArArchiveEntry(name,
data.length));
}
- TarArchiveEntry entry = new TarArchiveEntry("entry-" + i +
".txt");
- entry.setSize(data.length);
- out.putArchiveEntry(entry);
out.write(data);
out.closeArchiveEntry();
}
}
- return tar;
+ return archive;
}
- @Test
- public void budgetIsReleasedAfterEachEntry() throws Exception {
+ @ParameterizedTest
+ @ValueSource(strings = {"tar", "cpio", "ar"})
+ public void budgetIsReleasedAfterEachEntry(String ext) throws Exception {
CacheMemoryBudget budget = new CacheMemoryBudget(64L * 1024 * 1024);
ParseContext context = new ParseContext();
context.set(DigesterFactory.class, () -> new
InputStreamDigester("MD5", "tk:digest:MD5",
bytes -> Base64.getEncoder().encodeToString(bytes)));
context.set(CacheMemoryBudget.class, budget);
- List<Metadata> metadataList = getRecursiveMetadata(buildTar(),
context, false);
+ List<Metadata> metadataList = getRecursiveMetadata(build(ext),
context, false);
- assertEquals(4, metadataList.size(), "container plus three entries");
+ assertEquals(4, metadataList.size(), ext + ": container plus three
entries");
for (int i = 1; i < metadataList.size(); i++) {
- assertNotNull(metadataList.get(i).get("tk:digest:MD5"), "entry " +
i + " digested");
+ assertNotNull(metadataList.get(i).get("tk:digest:MD5"), ext + "
entry " + i + " digested");
}
assertEquals(0, budget.getReservedBytes(),
- "every entry's in-memory cache must return its reservation on
close");
+ ext + ": every entry's in-memory cache must return its
reservation on close");
}
}
diff --git
a/tika-serialization/src/main/java/org/apache/tika/serialization/serdes/ParseContextDeserializer.java
b/tika-serialization/src/main/java/org/apache/tika/serialization/serdes/ParseContextDeserializer.java
index 9b79b412ce..2ac419eb95 100644
---
a/tika-serialization/src/main/java/org/apache/tika/serialization/serdes/ParseContextDeserializer.java
+++
b/tika-serialization/src/main/java/org/apache/tika/serialization/serdes/ParseContextDeserializer.java
@@ -116,7 +116,7 @@ public class ParseContextDeserializer extends
JsonDeserializer<ParseContext> {
JsonNode value = contextNode.get(name);
checkForDuplicateContextKey(name, seenContextKeys);
// Store as a lazy JSON config (plain mapper: the main one may be
binary/Smile).
- parseContext.setJsonConfig(name,
plainMapper().writeValueAsString(value));
+ parseContext.setJsonConfig(name,
plainMapper().writeValueAsString(value), !restricted);
}
return parseContext;
diff --git
a/tika-serialization/src/test/java/org/apache/tika/serialization/serdes/ParseContextTrustTest.java
b/tika-serialization/src/test/java/org/apache/tika/serialization/serdes/ParseContextTrustTest.java
new file mode 100644
index 0000000000..582b167516
--- /dev/null
+++
b/tika-serialization/src/test/java/org/apache/tika/serialization/serdes/ParseContextTrustTest.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.serialization.serdes;
+
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.junit.jupiter.api.Test;
+
+/** Operator JSON (restricted=false) is trusted; per-request JSON
(restricted=true) is not. */
+public class ParseContextTrustTest {
+
+ @Test
+ public void testTrustFollowsRestriction() throws Exception {
+ var node = new ObjectMapper().readTree("{\"tesseract-ocr-parser\":
{\"skipOcr\": true}}");
+ assertTrue(ParseContextDeserializer.readParseContext(node, false)
+ .getJsonConfig("tesseract-ocr-parser").trusted());
+ assertFalse(ParseContextDeserializer.readParseContext(node, true)
+ .getJsonConfig("tesseract-ocr-parser").trusted());
+ }
+}
diff --git a/tika-server/docker-build/docker-tool.sh
b/tika-server/docker-build/docker-tool.sh
index ac5d03fef0..8fd789b68e 100755
--- a/tika-server/docker-build/docker-tool.sh
+++ b/tika-server/docker-build/docker-tool.sh
@@ -148,7 +148,8 @@ test_docker_image_uat() {
# Whether the image's HTTP client retries a 429 (TIKA-4912): the images carry
no unzip, so the
# jar is copied out of a stopped container and inspected on the host.
image_client_retries() {
- local probe="$1-probe" tmp rc=1
+ local probe tmp rc=1
+ probe="$(printf "%s" "$1" | tr "/:" "__")-probe" # a container name cannot
hold / or :
tmp="$(mktemp -d /tmp/tika-uat-probe.XXXXXX)"
docker create --name "$probe" "$1" >/dev/null 2>&1 || { rm -rf "$tmp";
return 1; }
if docker cp "$probe:/opt/tika-server/lib/." "$tmp/" >/dev/null 2>&1; then
diff --git a/tika-server/docker-build/sample-configs/vision/vlm-claude.json
b/tika-server/docker-build/sample-configs/vision/vlm-claude.json
index c56f812982..ad27c5a509 100644
--- a/tika-server/docker-build/sample-configs/vision/vlm-claude.json
+++ b/tika-server/docker-build/sample-configs/vision/vlm-claude.json
@@ -9,7 +9,7 @@
"engines": {
"claude": {
"claude-vlm-parser": {
- "apiKey": "${ANTHROPIC_API_KEY}",
+ "apiKey": "${env:ANTHROPIC_API_KEY}",
"model": "claude-sonnet-4-20250514"
}
}
diff --git a/tika-server/docker-build/sample-configs/vision/vlm-gemini.json
b/tika-server/docker-build/sample-configs/vision/vlm-gemini.json
index e8f80b845a..f6a0e0ea9c 100644
--- a/tika-server/docker-build/sample-configs/vision/vlm-gemini.json
+++ b/tika-server/docker-build/sample-configs/vision/vlm-gemini.json
@@ -8,7 +8,7 @@
"engines": {
"gemini": {
"gemini-vlm-parser": {
- "apiKey": "${GEMINI_API_KEY}",
+ "apiKey": "${env:GEMINI_API_KEY}",
"model": "gemini-2.5-flash"
}
}
diff --git
a/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/TikaResourceTest.java
b/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/TikaResourceTest.java
index 8820c6bccd..64f62c7c47 100644
---
a/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/TikaResourceTest.java
+++
b/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/TikaResourceTest.java
@@ -503,7 +503,8 @@ public class TikaResourceTest extends CXFTestBase {
.create(endPoint + TIKA_PATH + "/config")
.type("multipart/form-data")
.post(new MultipartBody(Arrays.asList(fileAtt,
configAtt)));
- assertEquals(500, response.getStatus());
+ // a raw NUL in the JSON is rejected before any parse: bad
request, not a parse failure
+ assertEquals(400, response.getStatus());
} catch (ProcessingException e) {
//can't tell why this intermittently happens. :(
//started after the upgrade to 3.2.7
