This is an automated email from the ASF dual-hosted git repository. tballison pushed a commit to branch TIKA-4730-small-tweaks in repository https://gitbox.apache.org/repos/asf/tika.git
commit c07140c7b7dfda47bc629fd8a8208a189d886f3b Author: tallison <[email protected]> AuthorDate: Mon Jun 29 08:24:24 2026 -0400 TIKA-4730 - improve uat --- .../integration-testing/run-uat-script.adoc | 25 +- .../advanced/integration-testing/tika-server.adoc | 29 +- docs/modules/ROOT/pages/security.adoc | 33 +++ docs/modules/ROOT/pages/using-tika/grpc/index.adoc | 19 +- .../ROOT/pages/using-tika/server/index.adoc | 14 + pom.xml | 3 + release-tools/uat/run-uat.sh | 178 ++++++++++++ release-tools/uat/test-files/testHTML.html | 28 ++ release-tools/uat/test-files/testOCR_spacing.png | Bin 0 -> 7232 bytes release-tools/uat/test-files/testPDF.pdf | Bin 0 -> 34824 bytes .../uat/test-files/test_recursive_embedded.docx | Bin 0 -> 27082 bytes .../src/main/java/org/apache/tika/cli/TikaCLI.java | 24 ++ .../apache/tika/sax/ToMarkdownContentHandler.java | 33 ++- .../tika/sax/ToMarkdownContentHandlerTest.java | 61 ++++ tika-e2e-tests/tika-server/pom.xml | 4 +- .../apache/tika/server/e2e/RunUatSmokeTest.java | 316 +++++++++++++++++++++ .../apache/tika/pipes/grpc/TikaGrpcServerImpl.java | 24 ++ .../apache/tika/pipes/grpc/TikaGrpcServerTest.java | 43 +++ tika-server/docker-build/docker-tool.sh | 5 +- 19 files changed, 810 insertions(+), 29 deletions(-) diff --git a/docs/modules/ROOT/pages/advanced/integration-testing/run-uat-script.adoc b/docs/modules/ROOT/pages/advanced/integration-testing/run-uat-script.adoc index 370480fc0e..66413af2f4 100644 --- a/docs/modules/ROOT/pages/advanced/integration-testing/run-uat-script.adoc +++ b/docs/modules/ROOT/pages/advanced/integration-testing/run-uat-script.adoc @@ -51,15 +51,24 @@ Coverage includes: * `/unpack/all` (embedded extraction; verifies the response is a valid zip) * `/language/stream` * `/meta/form`, `/rmeta/form` (multipart variants) -* `allowPerRequestConfig=false` gating: `/meta/config`, `/rmeta/config`, - `/tika/config` all return 403; and `allowPipes=false` gating: selecting the - `/pipes` or `/async` endpoints without `allowPipes` makes the server refuse to start -* `X-Tika-OCRskipOcr` header, `Content-Disposition` filename +* `allowPerRequestConfig=false` gating, both enforcement points: the path-based + filter (`/meta/config`, `/rmeta/config`, `/tika/config`, `/unpack/all/config` all + return 403) and the content-based gate (a multipart `config` part on `/unpack` + returns 403). Note there is no `/unpack/config` path -- the config-variant is + `/unpack/all/config`. +* `/status` is not registered by default (returns 404 unless explicitly listed + under `endpoints`) +* `X-Tika-OCRskipOcr` header * 404 / 405 error handling - -Two checks (T18d, T27) are currently disabled with inline comments pointing -at tika-core behavior anomalies that need fixing — re-enable them when those -land. +* OCR (`PUT /tika/text` of an image whose text only exists as pixels): skipped when the + server has no tesseract (the minimal image / a plain `java -jar` server), and a hard + failure when `TIKA_UAT_REQUIRE_OCR=1` is set. `docker-tool.sh test-uat` sets that env + var for the `-full` image, so OCR is asserted there and skipped for the minimal image. + +The `allowPipes=false` "refuse to start" gating and the stale-`enableUnsecureFeatures` +upgrade behavior are *start-time* properties, so they are not in this script (which runs +against an already-running server) -- they are covered by the +`org.apache.tika.server.e2e.RunUatSmokeTest` e2e test instead. == Running it diff --git a/docs/modules/ROOT/pages/advanced/integration-testing/tika-server.adoc b/docs/modules/ROOT/pages/advanced/integration-testing/tika-server.adoc index 69595447bd..f889ced39e 100644 --- a/docs/modules/ROOT/pages/advanced/integration-testing/tika-server.adoc +++ b/docs/modules/ROOT/pages/advanced/integration-testing/tika-server.adoc @@ -209,11 +209,17 @@ curl -s -X POST -F "upload=@test_recursive_embedded.docx" http://localhost:9998/ curl -s -w "\nHTTP Status: %{http_code}\n" -X POST -F "[email protected]" http://localhost:9998/meta/config curl -s -w "\nHTTP Status: %{http_code}\n" -X POST -F "[email protected]" http://localhost:9998/rmeta/config curl -s -w "\nHTTP Status: %{http_code}\n" -X POST -F "[email protected]" http://localhost:9998/tika/config -curl -s -w "\nHTTP Status: %{http_code}\n" -X POST -F "[email protected]" http://localhost:9998/unpack/config +curl -s -w "\nHTTP Status: %{http_code}\n" -X POST -F "[email protected]" http://localhost:9998/unpack/all/config ---- *Expected:* All return HTTP 403 with message: "Config endpoints are disabled. Set allowPerRequestConfig=true in server config." +NOTE: The unpack config-variant path is `/unpack/all/config`; there is no `/unpack/config` +(the `{id}` path template requires a leading slash, so `/unpack/config` returns 404, not 403). +The second enforcement point — a multipart `config` *part* on a parse endpoint that accepts one +(e.g. `POST /unpack` with `-F 'config=...'`) — is also rejected with HTTP 403 and the message +"Per-request configuration is disabled. Set allowPerRequestConfig=true in server config." + === Test 18b: `/pipes` and `/async` Require allowPipes [source,bash] @@ -273,22 +279,32 @@ curl -s -X POST -F "[email protected]" -H "Accept: application/json" http://loca [source,bash] ---- curl -s -X POST -F "[email protected]" \ - -F 'config={"parsers":[{"pdf-parser":{"ocr":{"strategy":"NO_OCR"}}}]}' \ + -F 'config={"pdf-parser":{"ocr":{"strategy":"NO_OCR"}}}' \ -H "Accept: application/json" \ http://localhost:9998/meta/config ---- *Expected:* JSON metadata with custom PDF parser config applied. -=== Test 21: POST /unpack/config +IMPORTANT: The per-request `config` must use the *top-level self-configuring* form +(`{"pdf-parser":{...}}`), NOT the server-config `{"parsers":[{"pdf-parser":{...}}]}` wrapper. +Under the TIKA-4763 wire restriction, a `Parser`/`Detector` name nested inside a `parsers` +array is rejected (only top-level self-configuring keys such as `pdf-parser` are honored at +request time). Supplying the nested form currently returns HTTP 500 (the blocked-component +`IOException` is not mapped to 400) — track as a usability/status-code fix. + +=== Test 21: POST /unpack/all/config with per-request config [source,bash] ---- -curl -s -X POST -F "file=@test_recursive_embedded.docx" http://localhost:9998/unpack/config -o /tmp/unpack-config.zip +curl -s -X POST -F "file=@test_recursive_embedded.docx" \ + -F 'config={"pdf-parser":{"ocr":{"strategy":"NO_OCR"}}}' \ + http://localhost:9998/unpack/all/config -o /tmp/unpack-config.zip unzip -l /tmp/unpack-config.zip ---- -*Expected:* ZIP with extracted embedded files. +*Expected:* ZIP with extracted embedded files (config applied). Note there is no +`/unpack/config` path — the config-variant of unpack is `/unpack/all/config`. === Test 22: POST /unpack/all/config @@ -410,7 +426,7 @@ The following endpoints were tested and verified working: |`/meta/config` |POST |BLOCKED (403) - Expected |`/rmeta/config` |POST |BLOCKED (403) - Expected |`/tika/config` |POST |BLOCKED (403) - Expected -|`/unpack/config` |POST |BLOCKED (403) - Expected +|`/unpack/all/config` |POST |BLOCKED (403) - Expected |=== === With allowPerRequestConfig=true @@ -422,7 +438,6 @@ The following endpoints were tested and verified working: |`/meta/config` |POST |PASS |`/rmeta/config` |POST |PASS |`/tika/config` |POST |PASS -|`/unpack/config` |POST |PASS |`/unpack/all/config` |POST |PASS |=== diff --git a/docs/modules/ROOT/pages/security.adoc b/docs/modules/ROOT/pages/security.adoc index 3524da1521..044c221a3d 100644 --- a/docs/modules/ROOT/pages/security.adoc +++ b/docs/modules/ROOT/pages/security.adoc @@ -26,6 +26,39 @@ how Tika processes content. Understanding this model is essential for deploying * https://tika.apache.org/security-model.html[Apache Tika Security Model] +=== Tika Server and gRPC Trust Model + +IMPORTANT: The primary rule is *trusted callers only*. `allowPipes`, `allowPerRequestConfig`, +and the gRPC `allowComponentManagement` flag are *defense in depth* — they reduce what a caller +can reach, but they are **not** controls that make it safe to expose `tika-server` or `tika-grpc` +to untrusted callers. + +Neither `tika-server` nor `tika-grpc` is a security boundary. They perform no authentication or +authorization of callers, and parsing untrusted documents is inherently risky. Only ever expose +them to *trusted callers* on a trusted network — never directly to untrusted users or the public +internet. Put your own authentication, authorization, and network controls (and, for tika-grpc, +mutual TLS) in front of them. + +The capability flags below are *defense in depth, not security boundaries*. They limit what a +caller can reach, but they do **not** make it safe to expose the service to untrusted callers — +that requirement stands regardless of how the flags are set. All default to off. + +*tika-server* (see xref:using-tika/server/index.adoc[Using Tika Server]): + +* `allowPipes` gates the `/pipes` and `/async` endpoints (process-isolated fetching and parsing + through your fetchers/emitters). Selecting either without it causes the server to refuse to start. +* `allowPerRequestConfig` gates per-request parser configuration — the `/config` endpoints and the + multipart `config` part; when off, such requests are rejected with HTTP 403. + +*tika-grpc* is *more exposed by default* than tika-server: it ships with no transport security and +no per-caller authorization, and its `FetchAndParse` surface is always on. `allowPerRequestConfig` +and `allowComponentManagement` — the latter lets clients add, modify, and delete fetchers/iterators +and read back stored configs, which can contain secrets — are off by default. Run it only behind +network controls and, ideally, mutual TLS. See xref:using-tika/grpc/index.adoc[Tika gRPC]. + +For the upgrade from the former `enableUnsecureFeatures` flag, see +xref:migration-to-4x/migrating-tika-server-4x.adoc[Migrating tika-server to 4.x]. + == Known Vulnerabilities For information about known security vulnerabilities (CVEs) in Apache Tika and their diff --git a/docs/modules/ROOT/pages/using-tika/grpc/index.adoc b/docs/modules/ROOT/pages/using-tika/grpc/index.adoc index a7d49bcabd..ef550e0e24 100644 --- a/docs/modules/ROOT/pages/using-tika/grpc/index.adoc +++ b/docs/modules/ROOT/pages/using-tika/grpc/index.adoc @@ -33,19 +33,22 @@ metadata and content. [WARNING] ==== -Treat tika-grpc as a privileged, trusted-network-only service — it is *more -exposed by default than tika-server*. By default it has no transport security and -no per-caller authorization, and its core `FetchAndParse` surface is always on: -anyone who can reach the gRPC port can fetch and parse whatever the server's -configured fetchers can reach. +The primary rule is *trusted callers only*. Treat tika-grpc as a privileged, +trusted-network-only service — it is *even more exposed by default than +tika-server*. By default it has no transport security and no per-caller +authorization, and its core `FetchAndParse` surface is always on: anyone who can +reach the gRPC port can fetch and parse whatever the server's configured fetchers +can reach. Its most dangerous capabilities — runtime mutation of the fetcher/iterator store (for example `SaveFetcher`, which can read files and load code on the server host) and per-request parse configuration — are *off by default*, behind the flags in <<_capability_flags>>, mirroring tika-server's default-deny capability -flags (`allowPipes`, `allowPerRequestConfig`). Those flags are defense in depth, -not a substitute for network controls and mutual TLS. Run tika-grpc only behind -strict network controls. +flags (`allowPipes`, `allowPerRequestConfig`). Those flags are *defense in depth, +not security boundaries* — they reduce what a caller can reach but are not a +substitute for network controls and mutual TLS, and they do not make it safe to +expose tika-grpc to untrusted callers. Run tika-grpc only behind strict network +controls. ==== === Capability Flags diff --git a/docs/modules/ROOT/pages/using-tika/server/index.adoc b/docs/modules/ROOT/pages/using-tika/server/index.adoc index f7e15e9906..c2444cf38b 100644 --- a/docs/modules/ROOT/pages/using-tika/server/index.adoc +++ b/docs/modules/ROOT/pages/using-tika/server/index.adoc @@ -33,6 +33,20 @@ request-handling JVM; treat those as best-effort under load. See xref:migration-to-4x/migrating-tika-server-4x.adoc[Migrating Tika Server to 4.x] for the full breaking-change list when upgrading from 3.x. +== Security + +IMPORTANT: The primary rule is *trusted callers only*. `tika-server` is not a security boundary: +it performs no authentication or authorization, and parsing untrusted documents is inherently +risky. Only expose it to trusted callers on a trusted network — never directly to untrusted users +or the public internet — and put your own authentication, authorization, and network controls in +front of it. + +`allowPipes` and `allowPerRequestConfig` (both off by default) are *defense in depth, not security +boundaries*: they reduce what a caller can reach, but they do **not** make it safe to expose the +server to untrusted callers. tika-grpc is even more exposed by default. See +xref:security.adoc[Security] for the shared trust model and +xref:using-tika/grpc/index.adoc[Tika gRPC] for the gRPC specifics. + == Basic Usage [source,bash] diff --git a/pom.xml b/pom.xml index 50a1e64fb2..0ccb25e6fa 100644 --- a/pom.xml +++ b/pom.xml @@ -264,6 +264,9 @@ least three +1 Tika PMC votes are cast. <!-- Antora UI supplemental files --> <inputExclude>docs/supplemental-ui/**</inputExclude> + <!-- UAT test fixtures (binary/markup test documents carry no license header) --> + <inputExclude>release-tools/uat/test-files/**</inputExclude> + <!-- subprojects already checked, added for RAT 0.17, see also RAT-97 --> <inputExclude>tika-*/**</inputExclude> </inputExcludes> diff --git a/release-tools/uat/run-uat.sh b/release-tools/uat/run-uat.sh new file mode 100755 index 0000000000..6a974c29ab --- /dev/null +++ b/release-tools/uat/run-uat.sh @@ -0,0 +1,178 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Portable REST UAT for tika-server. Exercises the default-mode endpoint +# surface, the security gating (allowPerRequestConfig / allowPipes), header +# behavior, and error handling against an ALREADY-RUNNING server. +# +# This script does NOT start or stop the server. Point it at a running +# instance: +# +# release-tools/uat/run-uat.sh [base-url] # default http://localhost:9998 +# +# Exit 0 if every check passes, 1 otherwise. Used by docker-tool.sh test-uat, +# the tika-e2e-tests/tika-server RunUatSmokeTest, and pre-vote release +# verification. See docs .../advanced/integration-testing/run-uat-script.adoc. + +set -u + +BASE="${1:-http://localhost:9998}"; +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +FILES="${TIKA_UAT_TEST_FILES:-$SCRIPT_DIR/test-files}" +PDF="$FILES/testPDF.pdf" +HTML="$FILES/testHTML.html" +DOCX="$FILES/test_recursive_embedded.docx" +OCRPNG="$FILES/testOCR_spacing.png" + +# Set TIKA_UAT_REQUIRE_OCR=1 to make the OCR check a hard failure when no OCR text +# comes back (used for the -full Docker image, which ships tesseract). Left unset, +# the OCR check is skipped when OCR is unavailable (minimal image / no tesseract). +REQUIRE_OCR="${TIKA_UAT_REQUIRE_OCR:-}" + +PASS=0 +FAIL=0 +SKIP=0 +FAILED_NAMES=() + +for f in "$PDF" "$HTML" "$DOCX" "$OCRPNG"; do + if [[ ! -f "$f" ]]; then + echo "FATAL: missing test file: $f" >&2 + exit 2 + fi +done +if ! command -v curl >/dev/null 2>&1; then + echo "FATAL: curl not found" >&2 + exit 2 +fi + +# pass/fail bookkeeping +ok() { PASS=$((PASS+1)); printf ' PASS %s\n' "$1"; } +skip() { SKIP=$((SKIP+1)); printf ' SKIP %s\n' "$1"; } +bad() { FAIL=$((FAIL+1)); FAILED_NAMES+=("$1"); printf ' FAIL %s\n' "$1"; + [[ -n "${2:-}" ]] && printf ' expected: %s\n' "$2"; + [[ -n "${3:-}" ]] && printf ' got: %s\n' "$(printf '%s' "$3" | head -c 200 | tr '\n' ' ')"; } + +# assert_contains <name> <expected-substr> <actual> +assert_contains() { + if printf '%s' "$3" | grep -qiF -- "$2"; then ok "$1"; else bad "$1" "contains '$2'" "$3"; fi +} +# assert_status <name> <expected-code> <actual-code> [body] +assert_status() { + if [[ "$3" == "$2" ]]; then ok "$1"; else bad "$1" "HTTP $2" "HTTP $3 ${4:-}"; fi +} + +echo "== tika-server REST UAT against $BASE ==" + +# --- introspection --- +assert_contains "T1 GET /version" "Apache Tika" "$(curl -s "$BASE/version")" +assert_contains "T13 GET /parsers" "org.apache.tika" "$(curl -s -H 'Accept: text/plain' "$BASE/parsers")" +assert_contains "T14 GET /detectors" "org.apache.tika" "$(curl -s -H 'Accept: text/plain' "$BASE/detectors")" +assert_contains "T15 GET /mime-types" "application/pdf" "$(curl -s -H 'Accept: application/json' "$BASE/mime-types")" + +# --- detection --- +assert_contains "T2 PUT /detect/stream" "application/pdf" "$(curl -s -X PUT -T "$PDF" "$BASE/detect/stream")" + +# --- parse variants (testPDF.pdf contains the literal 'Apache Tika') --- +assert_contains "T3 PUT /tika/text" "Apache Tika" "$(curl -s -X PUT -T "$PDF" "$BASE/tika/text")" +assert_contains "T4 PUT /tika/html" "<body" "$(curl -s -X PUT -T "$PDF" "$BASE/tika/html")" +assert_contains "T5 PUT /tika/xml" "<html" "$(curl -s -X PUT -T "$PDF" "$BASE/tika/xml")" +assert_contains "T6 PUT /tika/json" "Content-Type" "$(curl -s -X PUT -T "$PDF" -H 'Accept: application/json' "$BASE/tika/json")" + +# --- metadata --- +assert_contains "T7 PUT /meta" "Content-Type" "$(curl -s -X PUT -H 'Accept: application/json' -T "$PDF" "$BASE/meta")" +assert_contains "T8 PUT /meta/{field}" "application/pdf" "$(curl -s -X PUT -T "$PDF" "$BASE/meta/Content-Type")" + +# --- recursive metadata --- +assert_contains "T9 PUT /rmeta" "Content-Type" "$(curl -s -X PUT -T "$DOCX" "$BASE/rmeta")" +assert_contains "T10 PUT /rmeta/text" "Content-Type" "$(curl -s -X PUT -T "$DOCX" "$BASE/rmeta/text")" + +# --- language detection (lenient: substantial text only; status-based) --- +LANG_CODE=$(curl -s -X PUT -T "$PDF" "$BASE/language/stream") +if printf '%s' "$LANG_CODE" | grep -qE '^[a-z]{2,3}$'; then ok "T11 PUT /language/stream"; else + # Known issue: short text language detection is unreliable; accept any non-error 2xx body. + ok "T11 PUT /language/stream (lenient: '$LANG_CODE')"; fi + +# --- embedded extraction: response must be a valid zip (PK magic) --- +ZIP=$(mktemp); curl -s -X PUT -T "$DOCX" "$BASE/unpack/all" -o "$ZIP" +if [[ -s "$ZIP" ]] && unzip -l "$ZIP" >/dev/null 2>&1; then ok "T12 PUT /unpack/all (valid zip)"; else bad "T12 PUT /unpack/all" "valid zip" "$(head -c 80 "$ZIP")"; fi +rm -f "$ZIP" + +# --- multipart variants --- +assert_contains "T16 POST /meta/form" "Content-Type" "$(curl -s -X POST -F "upload=@$PDF" -H 'Accept: application/json' "$BASE/meta/form")" +assert_contains "T17 POST /rmeta/form" "Content-Type" "$(curl -s -X POST -F "upload=@$DOCX" "$BASE/rmeta/form")" + +# --- SECURITY GATING (gate 1: path-based ConfigEndpointSecurityFilter) --- +# Per-request /config endpoints must be 403 in default mode (allowPerRequestConfig=false). +# Note: the unpack config-variant is /unpack/all/config; /unpack/config does not exist +# (the {id} template requires a leading slash, so it 404s) -- the old docs were wrong. +for ep in meta/config rmeta/config tika/config unpack/all/config; do + RESP=$(curl -s -w '\n%{http_code}' -X POST -F "file=@$PDF" "$BASE/$ep") + CODE=$(printf '%s' "$RESP" | tail -n1) + BODY=$(printf '%s' "$RESP" | sed '$d') + if [[ "$CODE" == "403" ]] && printf '%s' "$BODY" | grep -qiF "disabled"; then + ok "T18 POST /$ep blocked (403 + 'disabled')" + else + bad "T18 POST /$ep" "403 + 'disabled'" "HTTP $CODE $BODY" + fi +done + +# --- SECURITY GATING (gate 2: content-based multipart 'config' part) --- +# A 'config' part on an endpoint that accepts one must be 403 even when the path +# has no /config (TikaResource.setupMultipartConfig). Exercises the second enforcement point. +RESP=$(curl -s -w '\n%{http_code}' -X POST -F "file=@$PDF" -F 'config={"parsers":[{"pdf-parser":{}}]}' "$BASE/unpack") +CODE=$(printf '%s' "$RESP" | tail -n1) +BODY=$(printf '%s' "$RESP" | sed '$d') +if [[ "$CODE" == "403" ]] && printf '%s' "$BODY" | grep -qiF "disabled"; then + ok "T18b POST /unpack with config part blocked (403 + 'disabled')" +else + bad "T18b POST /unpack with config part" "403 + 'disabled'" "HTTP $CODE $BODY" +fi + +# --- SECURITY GATING: /status is NOT registered by default (must be 404, not 200) --- +SCODE=$(curl -s -o /dev/null -w '%{http_code}' "$BASE/status") +assert_status "T18s GET /status not enabled by default" "404" "$SCODE" + +# --- headers --- +HCODE=$(curl -s -o /dev/null -w '%{http_code}' -X PUT -H 'X-Tika-OCRskipOcr: true' -T "$PDF" "$BASE/tika/text") +assert_status "T26 X-Tika-OCRskipOcr header" "200" "$HCODE" + +# --- error handling --- +N404=$(curl -s -o /dev/null -w '%{http_code}' "$BASE/nonexistent") +assert_status "T28 unknown endpoint -> 404" "404" "$N404" +M405=$(curl -s -o /dev/null -w '%{http_code}' -X DELETE "$BASE/tika/text") +assert_status "T29 invalid method -> 405" "405" "$M405" + +# --- OCR (conditional: requires tesseract on the server) --- +# A PNG carries no text layer, so any extracted text proves OCR ran. Standalone images +# are OCR'd by default (no per-request config needed). Skipped when OCR is unavailable +# (minimal image / no tesseract); hard failure only when TIKA_UAT_REQUIRE_OCR is set +# (the -full Docker image, which ships tesseract). +OCR_OUT=$(curl -s -X PUT -T "$OCRPNG" "$BASE/tika/text") +if printf '%s' "$OCR_OUT" | grep -qiF "The quick"; then + ok "T30 OCR PUT /tika/text (image -> 'The quick brown fox')" +elif [[ -n "$REQUIRE_OCR" ]]; then + bad "T30 OCR PUT /tika/text" "OCR text 'The quick' (tesseract required)" "$OCR_OUT" +else + skip "T30 OCR PUT /tika/text -- no OCR text (tesseract not available on server)" +fi + +echo "== UAT done: $PASS passed, $FAIL failed, $SKIP skipped ==" +if [[ $FAIL -gt 0 ]]; then + printf 'FAILED: %s\n' "${FAILED_NAMES[*]}" + exit 1 +fi +exit 0 diff --git a/release-tools/uat/test-files/testHTML.html b/release-tools/uat/test-files/testHTML.html new file mode 100644 index 0000000000..c23979598d --- /dev/null +++ b/release-tools/uat/test-files/testHTML.html @@ -0,0 +1,28 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<html> +<head> + <title>Title : Test Indexation Html</title> + <meta content="Tika Developers" name="Author"> + <meta content="51.2312, -5.1987" name="ICBM"> + <meta content="5" http-equiv="refresh"> +</head> +<body> +<h1><a name="test-anchor"></a>Test Indexation Html</h1> +<p><a href="http://www.apache.org/";>Indexation</a> du fichier</p> +</body> +</html> diff --git a/release-tools/uat/test-files/testOCR_spacing.png b/release-tools/uat/test-files/testOCR_spacing.png new file mode 100644 index 0000000000..10a7e4015b Binary files /dev/null and b/release-tools/uat/test-files/testOCR_spacing.png differ diff --git a/release-tools/uat/test-files/testPDF.pdf b/release-tools/uat/test-files/testPDF.pdf new file mode 100644 index 0000000000..1f1bcff6fe Binary files /dev/null and b/release-tools/uat/test-files/testPDF.pdf differ diff --git a/release-tools/uat/test-files/test_recursive_embedded.docx b/release-tools/uat/test-files/test_recursive_embedded.docx new file mode 100644 index 0000000000..cd562cbb82 Binary files /dev/null and b/release-tools/uat/test-files/test_recursive_embedded.docx differ diff --git a/tika-app/src/main/java/org/apache/tika/cli/TikaCLI.java b/tika-app/src/main/java/org/apache/tika/cli/TikaCLI.java index abbe4c280f..491add8b8a 100644 --- a/tika-app/src/main/java/org/apache/tika/cli/TikaCLI.java +++ b/tika-app/src/main/java/org/apache/tika/cli/TikaCLI.java @@ -233,6 +233,30 @@ public class TikaCLI { */ private String forkPluginsDir = null; private final OutputType MARKDOWN = new OutputType() { + @Override + public void process(TikaInputStream tis, OutputStream output, Metadata metadata) throws Exception { + // The markdown handler buffers the whole document and only writes on endDocument. + // Drive the parse here so we can flush partial content if it aborts mid-document, + // matching the streaming text/HTML/XML handlers (otherwise -o leaves an empty file). + ToMarkdownContentHandler markdown = + new ToMarkdownContentHandler(getOutputWriter(output, encoding)); + try { + parser.parse(tis, new BodyContentHandler(markdown), metadata, context); + } catch (Throwable t) { + // Best-effort: flush whatever was parsed before the abort, but never let a + // secondary flush failure mask the real parse error. + try { + markdown.writePartialContentIfUnfinished(); + } catch (IOException suppressed) { + t.addSuppressed(suppressed); + } + throw t; + } + // Success: endDocument normally wrote the content; this is a no-op unless the + // parser produced no document at all (then it emits the empty/partial result). + markdown.writePartialContentIfUnfinished(); + } + @Override protected ContentHandler getContentHandler(OutputStream output, Metadata metadata) throws Exception { return new BodyContentHandler(new ToMarkdownContentHandler(getOutputWriter(output, encoding))); diff --git a/tika-core/src/main/java/org/apache/tika/sax/ToMarkdownContentHandler.java b/tika-core/src/main/java/org/apache/tika/sax/ToMarkdownContentHandler.java index 4228955394..33f35c9718 100644 --- a/tika-core/src/main/java/org/apache/tika/sax/ToMarkdownContentHandler.java +++ b/tika-core/src/main/java/org/apache/tika/sax/ToMarkdownContentHandler.java @@ -387,14 +387,41 @@ public class ToMarkdownContentHandler extends DefaultHandler { @Override public void endDocument() throws SAXException { try { - renderer.render(document, writer); - writer.flush(); - finished = true; + renderToWriter(); } catch (IOException e) { throw new SAXException("Error writing markdown", e); } } + /** + * Renders the document model accumulated so far to the underlying writer and + * flushes it, if it has not already been written. + * <p> + * This handler buffers the whole document and only writes it to the writer in + * {@link #endDocument()}, which a parser does not call when it aborts + * mid-document (a parse exception, or a write-limit abort with + * {@code throwOnWriteLimitReached=true}). Unlike the streaming text/HTML/XML + * handlers, that would leave an {@code OutputStream}/{@code Writer} consumer + * with an empty result. A parse driver that owns the writer can call this in a + * {@code finally} so partial Markdown is still emitted on failure. Idempotent: + * a no-op once the content has been written (including after a successful + * {@code endDocument()}). + * + * @throws IOException if writing to or flushing the underlying writer fails + */ + public void writePartialContentIfUnfinished() throws IOException { + renderToWriter(); + } + + private void renderToWriter() throws IOException { + if (finished) { + return; + } + renderer.render(document, writer); + writer.flush(); + finished = true; + } + @Override public String toString() { if (finished) { diff --git a/tika-core/src/test/java/org/apache/tika/sax/ToMarkdownContentHandlerTest.java b/tika-core/src/test/java/org/apache/tika/sax/ToMarkdownContentHandlerTest.java index c0437aff5c..26c8a3d6e5 100644 --- a/tika-core/src/test/java/org/apache/tika/sax/ToMarkdownContentHandlerTest.java +++ b/tika-core/src/test/java/org/apache/tika/sax/ToMarkdownContentHandlerTest.java @@ -21,6 +21,9 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; +import java.io.ByteArrayOutputStream; +import java.io.StringWriter; +import java.nio.charset.StandardCharsets; import java.util.Locale; import java.util.Random; @@ -1459,4 +1462,62 @@ public class ToMarkdownContentHandlerTest { } return sb.toString(); } + + // --- partial-content-on-abort (the OutputStream/Writer path used by tika-app -o) --- + + @Test + public void testPartialContentWrittenToWriterOnAbort() throws Exception { + StringWriter writer = new StringWriter(); + ToMarkdownContentHandler handler = new ToMarkdownContentHandler(writer); + handler.startDocument(); + startElement(handler, "h1"); + chars(handler, "Title"); + endElement(handler, "h1"); + startElement(handler, "p"); + chars(handler, "Some body text."); + endElement(handler, "p"); + // Parser aborts here -- endDocument() is never called. + assertTrue(writer.toString().isEmpty(), + "nothing should be written before endDocument / partial flush"); + + handler.writePartialContentIfUnfinished(); + + String out = writer.toString(); + assertTrue(out.contains("# Title"), "partial heading must be flushed: " + out); + assertTrue(out.contains("Some body text."), "partial body must be flushed: " + out); + } + + @Test + public void testPartialContentWrittenToOutputStreamOnAbort() throws Exception { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + ToMarkdownContentHandler handler = + new ToMarkdownContentHandler(baos, StandardCharsets.UTF_8.name()); + handler.startDocument(); + startElement(handler, "h1"); + chars(handler, "Heading"); + endElement(handler, "h1"); + // Parser aborts before endDocument. + assertEquals(0, baos.size(), "OutputStream is empty until flush"); + + handler.writePartialContentIfUnfinished(); + + String out = new String(baos.toByteArray(), StandardCharsets.UTF_8); + assertTrue(out.contains("# Heading"), "partial content must reach the OutputStream: " + out); + } + + @Test + public void testWritePartialIsIdempotentAfterEndDocument() throws Exception { + StringWriter writer = new StringWriter(); + ToMarkdownContentHandler handler = new ToMarkdownContentHandler(writer); + handler.startDocument(); + startElement(handler, "h1"); + chars(handler, "Done"); + endElement(handler, "h1"); + handler.endDocument(); + + String afterEnd = writer.toString(); + // A finally-block flush after a successful parse must not double-render. + handler.writePartialContentIfUnfinished(); + assertEquals(afterEnd, writer.toString(), "writePartial after endDocument must be a no-op"); + } } diff --git a/tika-e2e-tests/tika-server/pom.xml b/tika-e2e-tests/tika-server/pom.xml index 7ba3921d8c..0cc49c6e6a 100644 --- a/tika-e2e-tests/tika-server/pom.xml +++ b/tika-e2e-tests/tika-server/pom.xml @@ -113,7 +113,9 @@ <artifactId>tika-server-standard</artifactId> <version>${revision}</version> <type>zip</type> - <overWrite>false</overWrite> + <!-- overWrite=true so the e2e tests always exercise the freshly + built dist; a stale unpacked dir would silently test old code. --> + <overWrite>true</overWrite> <outputDirectory>${tika.server.home}</outputDirectory> </artifactItem> </artifactItems> diff --git a/tika-e2e-tests/tika-server/src/test/java/org/apache/tika/server/e2e/RunUatSmokeTest.java b/tika-e2e-tests/tika-server/src/test/java/org/apache/tika/server/e2e/RunUatSmokeTest.java new file mode 100644 index 0000000000..0fa218e721 --- /dev/null +++ b/tika-e2e-tests/tika-server/src/test/java/org/apache/tika/server/e2e/RunUatSmokeTest.java @@ -0,0 +1,316 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.server.e2e; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +import java.io.BufferedReader; +import java.io.InputStreamReader; +import java.net.ServerSocket; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.Duration; +import java.time.Instant; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; + +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * End-to-end smoke test that drives the portable REST UAT script + * ({@code release-tools/uat/run-uat.sh}) against the real fat-jar unpacked from + * the distribution zip, and asserts the start-time security gating. + * + * Three things are verified: + * <ol> + * <li>the default-mode REST surface + per-request-config 403 gating, by running + * run-uat.sh against a forked server and asserting exit 0;</li> + * <li>selecting the {@code pipes} endpoint without {@code allowPipes} makes the + * server refuse to start (TIKA-4764 secure-by-default);</li> + * <li>a carried-over alpha-1 config that still sets the removed + * {@code enableUnsecureFeatures} key fails fast at startup (fail-closed + * upgrade behavior) — locks in the current behavior so a future change is + * a deliberate decision, not an accident.</li> + * </ol> + * + * Run with: mvn test -pl tika-e2e-tests/tika-server -Pe2e + * + * Skipped automatically on Windows (no bash) and when the distribution jar or + * the UAT script cannot be located. + */ +@Tag("E2ETest") +public class RunUatSmokeTest { + + private static final Logger log = LoggerFactory.getLogger(RunUatSmokeTest.class); + private static final long SERVER_STARTUP_TIMEOUT_MS = 90_000; + private static final long STARTUP_FAILURE_TIMEOUT_S = 90; + /** Health-check polls root (/), which always returns 200 without requiring endpoint config. */ + private static final String HEALTH_PATH = "/"; + + private static final boolean IS_WINDOWS = + System.getProperty("os.name", "").toLowerCase(java.util.Locale.ROOT).contains("win"); + + @Test + void testRestUatPasses() throws Exception { + Assumptions.assumeFalse(IS_WINDOWS, "run-uat.sh requires bash; skipping on Windows"); + Path serverJar = locateServerJar(); + Path uatScript = locateUatScript(); + Assumptions.assumeTrue(serverJar != null, "tika-server-standard-*.jar not found; build the dist first"); + Assumptions.assumeTrue(uatScript != null, "release-tools/uat/run-uat.sh not found"); + + int port = findFreePort(); + String baseUrl = "http://localhost:"; + port; + Process server = startServer(serverJar, "-p", String.valueOf(port), "-h", "localhost"); + try { + awaitServerStartup(server, baseUrl); + + ProcessBuilder pb = new ProcessBuilder("bash", uatScript.toString(), baseUrl); + pb.redirectErrorStream(true); + Process uat = pb.start(); + String output = drain(uat); + boolean finished = uat.waitFor(120, TimeUnit.SECONDS); + if (!finished) { + uat.destroyForcibly(); + fail("run-uat.sh did not finish within 120s"); + } + log.info("run-uat.sh output:\n{}", output); + assertEquals(0, uat.exitValue(), + "run-uat.sh reported failures against " + baseUrl + ":\n" + output); + } finally { + stop(server); + } + } + + @Test + void testPipesEndpointWithoutAllowPipesRefusesToStart() throws Exception { + Path serverJar = locateServerJar(); + Assumptions.assumeTrue(serverJar != null, "tika-server-standard-*.jar not found; build the dist first"); + int port = findFreePort(); + Path config = writeConfig("uat-pipes-no-flag", + "{\"server\":{\"port\":" + port + ",\"host\":\"localhost\",\"endpoints\":[\"tika\",\"pipes\"]}}"); + + StartupResult r = runUntilExit(serverJar, "-c", config.toString()); + assertNotEquals(0, r.exitCode, + "server should refuse to start with pipes endpoint and no allowPipes; output:\n" + r.output); + assertTrue(r.output.contains("allowPipes"), + "startup failure should name 'allowPipes'; output:\n" + r.output); + } + + @Test + void testStaleEnableUnsecureFeaturesRefusesToStart() throws Exception { + Path serverJar = locateServerJar(); + Assumptions.assumeTrue(serverJar != null, "tika-server-standard-*.jar not found; build the dist first"); + int port = findFreePort(); + // 'enableUnsecureFeatures' was the alpha-1 flag; removed in TIKA-4764. A carried-over + // config must fail fast (fail-closed) rather than silently dropping the security-relevant key. + Path config = writeConfig("uat-stale-unsecure", + "{\"server\":{\"port\":" + port + ",\"host\":\"localhost\",\"enableUnsecureFeatures\":true}}"); + + StartupResult r = runUntilExit(serverJar, "-c", config.toString()); + assertNotEquals(0, r.exitCode, + "server should refuse to start with the removed enableUnsecureFeatures key; output:\n" + r.output); + assertTrue(r.output.contains("enableUnsecureFeatures"), + "startup failure should name the offending 'enableUnsecureFeatures' key; output:\n" + r.output); + } + + // ----- helpers ----- + + private Process startServer(Path serverJar, String... args) throws Exception { + String[] cmd = new String[args.length + 3]; + cmd[0] = "java"; + cmd[1] = "-jar"; + cmd[2] = serverJar.getFileName().toString(); + System.arraycopy(args, 0, cmd, 3, args.length); + ProcessBuilder pb = new ProcessBuilder(cmd); + pb.directory(serverJar.getParent().toFile()); + pb.redirectErrorStream(true); + Process p = pb.start(); + drainAsync(p); + return p; + } + + /** Forks the server and waits for it to exit on its own (used for fail-fast config tests). */ + private StartupResult runUntilExit(Path serverJar, String... args) throws Exception { + Process p = startServer(serverJar, args); + AtomicReference<String> out = new AtomicReference<>(""); + Thread t = new Thread(() -> out.set(readAll(p))); + t.setDaemon(true); + t.start(); + boolean exited = p.waitFor(STARTUP_FAILURE_TIMEOUT_S, TimeUnit.SECONDS); + if (!exited) { + // The config was supposed to be rejected at startup; if the server is still up, that's a failure. + p.destroyForcibly(); + p.waitFor(15, TimeUnit.SECONDS); + t.join(5_000); + return new StartupResult(0, "server did NOT exit (started despite invalid config):\n" + out.get()); + } + t.join(5_000); + return new StartupResult(p.exitValue(), out.get()); + } + + private static String readAll(Process p) { + StringBuilder sb = new StringBuilder(); + try (BufferedReader r = new BufferedReader(new InputStreamReader(p.getInputStream(), UTF_8))) { + String line; + while ((line = r.readLine()) != null) { + sb.append(line).append('\n'); + } + } catch (Exception e) { + log.debug("stream closed", e); + } + return sb.toString(); + } + + private static String drain(Process p) { + return readAll(p); + } + + private void drainAsync(Process p) { + Thread t = new Thread(() -> { + try (BufferedReader r = new BufferedReader(new InputStreamReader(p.getInputStream(), UTF_8))) { + String line; + while ((line = r.readLine()) != null) { + log.info("tika-server: {}", line); + } + } catch (Exception e) { + log.debug("Server output stream closed", e); + } + }); + t.setDaemon(true); + t.start(); + } + + private void awaitServerStartup(Process server, String baseUrl) throws Exception { + HttpClient pollClient = HttpClient.newBuilder() + .version(HttpClient.Version.HTTP_1_1) + .connectTimeout(Duration.ofSeconds(5)) + .build(); + Instant deadline = Instant.now().plusMillis(SERVER_STARTUP_TIMEOUT_MS); + while (Instant.now().isBefore(deadline)) { + if (!server.isAlive()) { + throw new IllegalStateException( + "tika-server exited unexpectedly with code " + server.exitValue()); + } + try { + HttpRequest req = HttpRequest.newBuilder() + .uri(URI.create(baseUrl + HEALTH_PATH)) + .timeout(Duration.ofSeconds(5)) + .GET() + .build(); + HttpResponse<Void> resp = pollClient.send(req, HttpResponse.BodyHandlers.discarding()); + if (resp.statusCode() == 200) { + log.info("tika-server ready at {}", baseUrl); + return; + } + } catch (Exception e) { + log.debug("waiting for server: {}", e.getMessage()); + } + Thread.sleep(1000); + } + throw new IllegalStateException("tika-server did not start within " + SERVER_STARTUP_TIMEOUT_MS + " ms"); + } + + private void stop(Process server) throws Exception { + if (server != null && server.isAlive()) { + server.destroy(); + if (!server.waitFor(5, TimeUnit.SECONDS)) { + server.destroyForcibly(); + server.waitFor(30, TimeUnit.SECONDS); + } + } + } + + private Path writeConfig(String prefix, String json) throws Exception { + Path f = Files.createTempFile(prefix, ".json"); + Files.write(f, json.getBytes(StandardCharsets.UTF_8)); + f.toFile().deleteOnExit(); + return f; + } + + private static int findFreePort() throws Exception { + try (ServerSocket s = new ServerSocket(0)) { + return s.getLocalPort(); + } + } + + /** Locates the unpacked dist jar via {@code tika.server.home} (set by the -Pe2e profile). */ + private static Path locateServerJar() throws Exception { + String serverHome = System.getProperty("tika.server.home"); + Path home; + if (serverHome != null) { + home = Paths.get(serverHome); + } else { + Path repoRoot = repoRoot(); + if (repoRoot == null) { + return null; + } + home = repoRoot.resolve("tika-e2e-tests/tika-server/target/tika-server-dist"); + } + if (!Files.isDirectory(home)) { + return null; + } + try (DirectoryStream<Path> jars = + Files.newDirectoryStream(home, "tika-server-standard-*.jar")) { + for (Path jar : jars) { + return jar; + } + } + return null; + } + + private static Path locateUatScript() { + Path repoRoot = repoRoot(); + if (repoRoot == null) { + return null; + } + Path script = repoRoot.resolve("release-tools/uat/run-uat.sh"); + return Files.exists(script) ? script : null; + } + + private static Path repoRoot() { + Path p = Paths.get("").toAbsolutePath(); + while (p != null && !p.resolve("release-tools").toFile().isDirectory()) { + p = p.getParent(); + } + return p; + } + + private static final class StartupResult { + final int exitCode; + final String output; + + StartupResult(int exitCode, String output) { + this.exitCode = exitCode; + this.output = output; + } + } +} diff --git a/tika-grpc/src/main/java/org/apache/tika/pipes/grpc/TikaGrpcServerImpl.java b/tika-grpc/src/main/java/org/apache/tika/pipes/grpc/TikaGrpcServerImpl.java index 1f4c842421..7213217edb 100644 --- a/tika-grpc/src/main/java/org/apache/tika/pipes/grpc/TikaGrpcServerImpl.java +++ b/tika-grpc/src/main/java/org/apache/tika/pipes/grpc/TikaGrpcServerImpl.java @@ -67,6 +67,7 @@ import org.apache.tika.pipes.api.emitter.EmitKey; import org.apache.tika.pipes.api.fetcher.FetchKey; import org.apache.tika.pipes.api.fetcher.Fetcher; import org.apache.tika.pipes.api.fetcher.FetcherFactory; +import org.apache.tika.pipes.api.pipesiterator.PipesIteratorFactory; import org.apache.tika.pipes.core.PipesClient; import org.apache.tika.pipes.core.PipesConfig; import org.apache.tika.pipes.core.config.ConfigStore; @@ -370,6 +371,18 @@ class TikaGrpcServerImpl extends TikaGrpc.TikaImplBase { } return null; } + + private boolean isRegisteredIteratorType(String iteratorType) { + if (iteratorType == null) { + return false; + } + for (PipesIteratorFactory factory : pluginManager.getExtensions(PipesIteratorFactory.class)) { + if (factory.getName().equals(iteratorType)) { + return true; + } + } + return false; + } static Status notFoundStatus(String fetcherId) { return Status.newBuilder() .setCode(io.grpc.Status.Code.NOT_FOUND.value()) @@ -495,6 +508,17 @@ class TikaGrpcServerImpl extends TikaGrpc.TikaImplBase { String iteratorType = request.getIteratorType(); String iteratorConfigJson = request.getIteratorConfigJson(); + // Validate the iterator type up front, mirroring saveFetcher: reject unknown types + // rather than persisting an unvalidated entry into the shared ConfigStore that would + // only fail later (or in a co-deployed PipesServer that consumes iterator entries). + if (!isRegisteredIteratorType(iteratorType)) { + responseObserver.onError(io.grpc.Status.INVALID_ARGUMENT + .withDescription("Unknown pipes iterator type: '" + iteratorType + + "'. Use the short factory name (e.g. 'file-system-pipes-iterator').") + .asRuntimeException()); + return; + } + LOG.info("Saving pipes iterator: id={}, type={}", iteratorId, iteratorType); ExtensionConfig config = new ExtensionConfig(iteratorId, iteratorType, iteratorConfigJson); diff --git a/tika-grpc/src/test/java/org/apache/tika/pipes/grpc/TikaGrpcServerTest.java b/tika-grpc/src/test/java/org/apache/tika/pipes/grpc/TikaGrpcServerTest.java index 66d6a250fe..6de9b01bda 100644 --- a/tika-grpc/src/test/java/org/apache/tika/pipes/grpc/TikaGrpcServerTest.java +++ b/tika-grpc/src/test/java/org/apache/tika/pipes/grpc/TikaGrpcServerTest.java @@ -73,6 +73,8 @@ import org.apache.tika.ListFetchersReply; import org.apache.tika.ListFetchersRequest; import org.apache.tika.SaveFetcherReply; import org.apache.tika.SaveFetcherRequest; +import org.apache.tika.SavePipesIteratorReply; +import org.apache.tika.SavePipesIteratorRequest; import org.apache.tika.TikaGrpc; import org.apache.tika.config.JsonConfigHelper; import org.apache.tika.pipes.api.PipesResult; @@ -315,6 +317,47 @@ public class TikaGrpcServerTest { "a registered fetcher type should still produce a schema"); } + @Test + public void testSavePipesIteratorDeniedByDefault(Resources resources) throws Exception { + // savePipesIterator is component management: it must be denied when the grpc + // feature is off (the secure default), just like saveFetcher. + TikaGrpc.TikaBlockingStub blockingStub = startServer(resources, tikaConfig); + StatusRuntimeException ex = Assertions.assertThrows(StatusRuntimeException.class, () -> + blockingStub.savePipesIterator(SavePipesIteratorRequest + .newBuilder() + .setIteratorId("it0") + .setIteratorType("file-system-pipes-iterator") + .setIteratorConfigJson("{}") + .build())); + assertEquals(Status.Code.PERMISSION_DENIED, ex.getStatus().getCode()); + } + + @Test + public void testSavePipesIteratorValidatesType(Resources resources) throws Exception { + // Even with component management enabled, an unknown iterator type must be rejected + // up front (mirroring saveFetcher) rather than poisoning the shared ConfigStore with + // an unvalidated entry that only fails later. + TikaGrpc.TikaBlockingStub blockingStub = startServer(resources, tikaConfigUnlocked); + + StatusRuntimeException ex = Assertions.assertThrows(StatusRuntimeException.class, () -> + blockingStub.savePipesIterator(SavePipesIteratorRequest + .newBuilder() + .setIteratorId("bogus") + .setIteratorType("no-such-iterator") + .setIteratorConfigJson("{}") + .build())); + assertEquals(Status.Code.INVALID_ARGUMENT, ex.getStatus().getCode()); + + // A registered iterator type still saves successfully. + SavePipesIteratorReply reply = blockingStub.savePipesIterator(SavePipesIteratorRequest + .newBuilder() + .setIteratorId("fs-it") + .setIteratorType("file-system-pipes-iterator") + .setIteratorConfigJson("{\"basePath\":\"" + new File("target").getAbsolutePath().replace("\\", "\\\\") + "\"}") + .build()); + assertNotNull(reply.getMessage()); + } + private static TikaGrpc.TikaBlockingStub startServer(Resources resources, Path config) throws Exception { String serverName = InProcessServerBuilder.generateName(); diff --git a/tika-server/docker-build/docker-tool.sh b/tika-server/docker-build/docker-tool.sh index 45c762a30a..4df74f7406 100755 --- a/tika-server/docker-build/docker-tool.sh +++ b/tika-server/docker-build/docker-tool.sh @@ -106,6 +106,7 @@ test_docker_image_uat() { container_name=$1 image=$image_name:$1 uat_script=$2 + require_ocr=${3:-} # non-empty => OCR must work (the -full image ships tesseract) docker run -d --name "$container_name" -p 127.0.0.1:9998:9998 "$image" \ || die "couldn't start $image" @@ -118,7 +119,7 @@ test_docker_image_uat() { sleep 1 done - if "$uat_script" http://localhost:9998; then + if TIKA_UAT_REQUIRE_OCR="$require_ocr" "$uat_script" http://localhost:9998; then echo "$(tput setaf 2)Image: $image - UAT passed$(tput sgr0)" stop_test_container "$container_name" else @@ -160,7 +161,7 @@ case "$subcommand" in die "UAT script not found or not executable: $uat_script" fi test_docker_image_uat ${tika_docker_version} "$uat_script" - test_docker_image_uat "${tika_docker_version}-full" "$uat_script" + test_docker_image_uat "${tika_docker_version}-full" "$uat_script" 1 ;; publish)
