This is an automated email from the ASF dual-hosted git repository.
tballison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git
The following commit(s) were added to refs/heads/main by this push:
new fe197f039a TIKA-4810 -- follow-on: single-pass Utf8Stats walker +
loose-end cleanup (#2995)
fe197f039a is described below
commit fe197f039ab7a0c2bcc0806334b9cf9c35dcd3cc
Author: Tim Allison <[email protected]>
AuthorDate: Fri Aug 7 18:47:30 2026 -0400
TIKA-4810 -- follow-on: single-pass Utf8Stats walker + loose-end cleanup
(#2995)
---
.skills/tika-eval-compare.md | 3 +
.../tika/ml/chardetect/CjkDecodeValidator.java | 28 +-
.../ml/chardetect/MojibusterEncodingDetector.java | 89 +------
.../ml/chardetect/StructuralEncodingRules.java | 284 +++++++--------------
.../ToleratedUtf8StructuralRegressionTest.java | 25 +-
.../apache/tika/ml/chardetect/Utf8StatsTest.java | 126 +++++++++
.../ml/junkdetect/JunkFilterEncodingDetector.java | 11 +-
7 files changed, 251 insertions(+), 315 deletions(-)
diff --git a/.skills/tika-eval-compare.md b/.skills/tika-eval-compare.md
index cdb581e386..17058187c4 100644
--- a/.skills/tika-eval-compare.md
+++ b/.skills/tika-eval-compare.md
@@ -22,6 +22,9 @@ Ask the user for:
zip archive containing `tika-app-*.jar`, `lib/`, and `plugins/`.
- A corpus of input files (a directory tree).
- tika-eval-app, built from `tika-eval/tika-eval-app` (use the zip).
+ The bare `target/tika-eval-app-*.jar` is thin (no bundled deps) and dies with
+ `NoClassDefFoundError: ...GzipCompressorOutputStream` (esp. with `-r`).
Always
+ run the jar from the unzipped `target/*.zip` dir, which carries its `lib/`.
- **Enable MD5 digesting** in both configs so tika-eval can match
embedded documents by content hash (not just index position).
Add to the config JSON:
diff --git
a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/CjkDecodeValidator.java
b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/CjkDecodeValidator.java
index 00f44dcee5..4dbe27977f 100644
---
a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/CjkDecodeValidator.java
+++
b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/CjkDecodeValidator.java
@@ -94,7 +94,7 @@ public final class CjkDecodeValidator {
i++;
continue;
}
- int ulen = utf8SequenceLength(bytes, i);
+ int ulen = StructuralEncodingRules.utf8SequenceLength(bytes, i);
if (ulen > 0) {
nUtf8Seqs++;
i += ulen; // embedded UTF-8 — not legacy content, skip
@@ -136,30 +136,4 @@ public final class CjkDecodeValidator {
|| name.contains("shift") || name.contains("jis") ||
name.contains("949");
}
- /** Length (2/3/4) of a valid UTF-8 multi-byte sequence starting at {@code
i},
- * or 0 if none. Lead-byte ranges exclude overlong 2-byte (C0/C1) and
- * out-of-range (≥F5) leads; continuations must be 0x80–0xBF. */
- static int utf8SequenceLength(byte[] b, int i) {
- int x = b[i] & 0xFF;
- int len;
- if (x >= 0xC2 && x <= 0xDF) {
- len = 2;
- } else if (x >= 0xE0 && x <= 0xEF) {
- len = 3;
- } else if (x >= 0xF0 && x <= 0xF4) {
- len = 4;
- } else {
- return 0;
- }
- if (i + len > b.length) {
- return 0;
- }
- for (int k = 1; k < len; k++) {
- int c = b[i + k] & 0xFF;
- if (c < 0x80 || c > 0xBF) {
- return 0;
- }
- }
- return len;
- }
}
diff --git
a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/MojibusterEncodingDetector.java
b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/MojibusterEncodingDetector.java
index a7f1909d9b..d0424b4573 100644
---
a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/MojibusterEncodingDetector.java
+++
b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/MojibusterEncodingDetector.java
@@ -140,7 +140,10 @@ public class MojibusterEncodingDetector implements
EncodingDetector {
* genuine false-CJK ≥5.3%, so ~2.5% separates them (see
CjkDecodeValidator). */
private static final double CJK_FAILURE_VETO_THRESHOLD = 0.025;
- /** Confidence for the windows-1252 fallback emitted on empty/ASCII
probes. */
+ /** Confidence for the windows-1252 fallback emitted on empty/ASCII probes.
+ * MUST equal JunkFilterEncodingDetector.NO_INFO_CONFIDENCE
(tika-ml-junkdetect):
+ * its strict {@code confidence > NO_INFO_CONFIDENCE} test treats exactly
this
+ * value as "statistical layer abstained" so a declaration can win. */
private static final float FALLBACK_CONFIDENCE = 0.1f;
/**
@@ -335,7 +338,8 @@ public class MojibusterEncodingDetector implements
EncodingDetector {
// because STRUCTURAL confidence outranks STATISTICAL.
// • AMBIGUOUS (pure ASCII or only truncated lead): no
// emission; NB + fallbacks handle it.
- StructuralEncodingRules.Utf8Result utf8 =
StructuralEncodingRules.checkUtf8(probe);
+ StructuralEncodingRules.Utf8Stats utf8Stats =
StructuralEncodingRules.utf8Stats(probe);
+ StructuralEncodingRules.Utf8Result utf8 = utf8Stats.toResult();
// TACTICAL: tolerate small corruption. If the grammar check returned
// NOT_UTF8 but the malformed-byte fraction is tiny, treat as UTF-8 —
// a single bad continuation byte in 2KB of CJK is nearly always
@@ -343,7 +347,7 @@ public class MojibusterEncodingDetector implements
EncodingDetector {
// replaced with a probabilistic decoder.
boolean utf8Tolerated = false;
if (utf8 == StructuralEncodingRules.Utf8Result.NOT_UTF8) {
- int errors = StructuralEncodingRules.countUtf8Errors(probe);
+ int errors = utf8Stats.errors();
// Length-aware: absolute floor for short probes, rate for long
ones.
int maxTolerated = Math.max(UTF8_MAX_TOLERATED_ERRORS,
(int) (probe.length * UTF8_MALFORMED_TOLERANCE));
@@ -371,7 +375,7 @@ public class MojibusterEncodingDetector implements
EncodingDetector {
// document its STRUCTURAL proof and JunkFilter has nothing to prefer
// over the declared charset.
boolean evidenceTolerated = utf8Tolerated
- && StructuralEncodingRules.countUtf8Sequences(probe) >=
MIN_TOLERATED_UTF8_SEQUENCES;
+ && utf8Stats.sequences() >= MIN_TOLERATED_UTF8_SEQUENCES;
if (utf8 == StructuralEncodingRules.Utf8Result.LIKELY_UTF8 ||
evidenceTolerated) {
pool.add(new EncodingResult(
java.nio.charset.StandardCharsets.UTF_8,
@@ -534,83 +538,6 @@ public class MojibusterEncodingDetector implements
EncodingDetector {
return true;
}
- /**
- * Resolve UTF-16 to LE or BE once NB has called it "UTF-16".
- *
- * <p>Two deterministic tests:
- * <ol>
- * <li>Null-density: count null bytes in even-offset positions
- * vs odd-offset positions. For ASCII-in-UTF-16-LE the
- * high byte is 0x00 at odd positions; for BE it's at even
- * positions. If one column is clearly null-dominant, that
- * column indicates the endianness.</li>
- * <li>Codepoint validity fallback: for ambiguous probes (pure
- * CJK UTF-16, no nulls in either column) count how many
- * 16-bit codepoints under LE vs BE interpretation land in
- * assigned Unicode BMP ranges (non-PUA, non-unassigned).
- * Whichever interpretation yields more valid codepoints
- * wins.</li>
- * </ol>
- *
- * <p>Also honors the {@code invalidUtf16Le}/{@code invalidUtf16Be}
- * flags from {@link WideUnicodeDetector} — if either endianness
- * is structurally invalid (surrogate-pair violation), the other
- * wins by default.
- *
- * @return the resolved charset, or {@code null} if the probe is
- * structurally invalid under both interpretations
- */
- private static java.nio.charset.Charset disambiguateUtf16(byte[] probe,
- boolean
invalidLe,
- boolean
invalidBe) {
- if (invalidLe && invalidBe) {
- return null;
- }
- if (invalidLe) {
- return java.nio.charset.Charset.forName("UTF-16BE");
- }
- if (invalidBe) {
- return java.nio.charset.Charset.forName("UTF-16LE");
- }
- int nullEven = 0;
- int nullOdd = 0;
- for (int i = 0; i + 1 < probe.length; i += 2) {
- if (probe[i] == 0) nullEven++;
- if (probe[i + 1] == 0) nullOdd++;
- }
- // Clear null-density winner: one column is ≥ 3× more
- // null-dominant than the other.
- if (nullEven >= 3 * Math.max(1, nullOdd)) {
- return java.nio.charset.Charset.forName("UTF-16BE");
- }
- if (nullOdd >= 3 * Math.max(1, nullEven)) {
- return java.nio.charset.Charset.forName("UTF-16LE");
- }
- // Ambiguous on null-density (CJK content). Count valid BMP
- // codepoints under each interpretation. A "valid" codepoint
- // is any non-zero codepoint outside the surrogate range
- // (0xD800-0xDFFF) — for CJK content most bytes map into
- // assigned blocks, and random-byte-interpreted-as-UTF-16
- // produces many surrogate-range halves.
- int validLe = 0;
- int validBe = 0;
- for (int i = 0; i + 1 < probe.length; i += 2) {
- int lo = probe[i] & 0xFF;
- int hi = probe[i + 1] & 0xFF;
- int leCp = (hi << 8) | lo;
- int beCp = (lo << 8) | hi;
- if (leCp != 0 && (leCp < 0xD800 || leCp > 0xDFFF)) {
- validLe++;
- }
- if (beCp != 0 && (beCp < 0xD800 || beCp > 0xDFFF)) {
- validBe++;
- }
- }
- return validLe >= validBe
- ? java.nio.charset.Charset.forName("UTF-16LE")
- : java.nio.charset.Charset.forName("UTF-16BE");
- }
-
/**
* Relabel the top result to windows-1252 when top is a non-1252
* member of {@link CharsetConfusables#SBCS_LATIN_FAMILY} and
diff --git
a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/StructuralEncodingRules.java
b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/StructuralEncodingRules.java
index 67eb7c7503..bbd9a371d2 100644
---
a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/StructuralEncodingRules.java
+++
b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/StructuralEncodingRules.java
@@ -656,97 +656,7 @@ public final class StructuralEncodingRules {
}
public static Utf8Result checkUtf8(byte[] bytes, int offset, int length) {
- int highByteCount = 0;
- // Count only multi-byte sequences whose continuations are all
- // present in the probe. A truncated lead at probe-end is
- // valid-so-far but provides no structural evidence — a single
- // 0xC3 byte alone should not be enough to claim UTF-8.
- int completeHighSeqCount = 0;
- int i = offset;
- int end = offset + length;
-
- while (i < end) {
- int b = bytes[i] & 0xFF;
-
- if (b < 0x80) {
- i++;
- continue;
- }
-
- highByteCount++;
-
- // Determine expected continuation count from the lead byte
- int seqLen;
- if (b >= 0xF8) {
- // 5-/6-byte sequences are not valid Unicode
- return Utf8Result.NOT_UTF8;
- } else if (b >= 0xF0) {
- seqLen = 4;
- } else if (b >= 0xE0) {
- seqLen = 3;
- } else if (b >= 0xC0) {
- seqLen = 2;
- } else {
- // 0x80–0xBF is a continuation byte without a lead → invalid
- return Utf8Result.NOT_UTF8;
- }
-
- // Overlong 2-byte sequence (C0 or C1 lead)
- if (seqLen == 2 && b <= 0xC1) {
- return Utf8Result.NOT_UTF8;
- }
-
- // Check that the right number of continuation bytes follow
- boolean truncated = false;
- for (int k = 1; k < seqLen; k++) {
- if (i + k >= end) {
- truncated = true;
- break;
- }
- int cb = bytes[i + k] & 0xFF;
- if (cb < 0x80 || cb > 0xBF) {
- return Utf8Result.NOT_UTF8;
- }
- }
-
- // Validate scalar value ranges for 3- and 4-byte sequences,
- // but only when the full sequence is present. Truncated sequences
- // at the end of a probe are not evidence of invalid UTF-8.
- if (!truncated) {
- if (seqLen == 3) {
- int cp = ((b & 0x0F) << 12)
- | ((bytes[i + 1] & 0xFF) & 0x3F) << 6
- | ((bytes[i + 2] & 0xFF) & 0x3F);
- if (cp < 0x0800 || (cp >= 0xD800 && cp <= 0xDFFF)) {
- return Utf8Result.NOT_UTF8;
- }
- } else if (seqLen == 4) {
- int cp = ((b & 0x07) << 18)
- | ((bytes[i + 1] & 0xFF) & 0x3F) << 12
- | ((bytes[i + 2] & 0xFF) & 0x3F) << 6
- | ((bytes[i + 3] & 0xFF) & 0x3F);
- if (cp < 0x10000 || cp > 0x10FFFF) {
- return Utf8Result.NOT_UTF8;
- }
- }
- completeHighSeqCount++;
- }
-
- i += seqLen;
- }
-
- // Grammar is valid. Require at least one COMPLETE multi-byte
- // sequence (not just a truncated lead at probe-end) to claim
- // LIKELY. A lone 0xC3 at the end of a 1-byte probe is
- // valid-so-far but provides no structural evidence.
- if (completeHighSeqCount > 0) {
- return Utf8Result.LIKELY_UTF8;
- }
- // highByteCount > 0 but completeHighSeqCount == 0 means every
- // high byte we saw was the start of a truncated sequence at
- // probe-end. That's not enough evidence — treat as ambiguous.
- // Zero high bytes = pure ASCII; caller handles this separately.
- return Utf8Result.AMBIGUOUS;
+ return utf8Stats(bytes, offset, length).toResult();
}
/**
@@ -787,30 +697,49 @@ public final class StructuralEncodingRules {
}
/**
- * Counts the number of malformed UTF-8 <em>sequences</em> in the sample —
- * one event per bad lead, orphaned continuation, overlong, surrogate, or
- * out-of-range codepoint, regardless of how many bytes the bad sequence
- * spans. Unlike {@link #checkUtf8}, this does not early-exit on the
- * first bad sequence; it scans the entire range, resyncing after each
- * error. Returns 0 for a clean UTF-8 stream.
+ * Single-pass tally of UTF-8 structure over the sample: malformed-sequence
+ * events and complete valid multi-byte sequences. Same walk as
+ * {@link #checkUtf8} but without the early exit — scans the entire range,
+ * resyncing after each error.
*
- * <p>Useful for "tolerant" UTF-8 acceptance: a real-world UTF-8 file with
- * a few corrupted sequences (copy-paste artefact, truncated upstream,
- * MIME transport flip) should still be recognized as UTF-8 rather than
- * rejected outright. Caller decides what error count is tolerable
- * (typically as a fraction of probe length).</p>
+ * <p>{@code errors} counts one event per bad lead, orphaned continuation,
+ * overlong, surrogate, or out-of-range codepoint regardless of how many
+ * bytes the bad sequence spans — matching Java's
+ * {@code new String(bytes, UTF_8)} U+FFFD-per-error semantics, the
+ * convention callers use for tolerance thresholds. A truncated sequence
+ * at probe-end is neither an error nor a sequence.</p>
*
- * <p>The count matches Java's {@code new String(bytes, UTF_8)}'s
- * U+FFFD-per-error semantics (one replacement per malformed sequence).</p>
+ * <p>{@code sequences} gauges how much genuine multi-byte UTF-8 evidence
+ * the probe carries independent of its error count.</p>
*
- * @return number of malformed UTF-8 sequence events
+ * <p>{@code truncatedTailInvalid}: the probe ends in a truncated sequence
+ * whose partial continuations are provably NOT UTF-8 (e.g. {@code E0 41}
+ * at probe-end). Not an error <em>event</em> (no U+FFFD equivalent — the
+ * decoder would ask for more input), but it disqualifies the sample from
+ * {@link Utf8Result#LIKELY_UTF8}/{@link Utf8Result#AMBIGUOUS}.</p>
*/
- public static int countUtf8Errors(byte[] bytes) {
- return countUtf8Errors(bytes, 0, bytes.length);
+ public record Utf8Stats(int errors, int sequences, boolean
truncatedTailInvalid) {
+
+ /** Collapse to the {@link #checkUtf8} tri-state: any invalidity →
+ * NOT_UTF8; else ≥1 complete multi-byte sequence → LIKELY_UTF8
+ * (a lone truncated lead is no structural evidence); else AMBIGUOUS
+ * (pure ASCII, or truncated-lead-only). */
+ public Utf8Result toResult() {
+ if (errors > 0 || truncatedTailInvalid) {
+ return Utf8Result.NOT_UTF8;
+ }
+ return sequences > 0 ? Utf8Result.LIKELY_UTF8 :
Utf8Result.AMBIGUOUS;
+ }
}
- public static int countUtf8Errors(byte[] bytes, int offset, int length) {
+ public static Utf8Stats utf8Stats(byte[] bytes) {
+ return utf8Stats(bytes, 0, bytes.length);
+ }
+
+ public static Utf8Stats utf8Stats(byte[] bytes, int offset, int length) {
int errors = 0;
+ int sequences = 0;
+ boolean truncatedTailInvalid = false;
int i = offset;
int end = offset + length;
while (i < end) {
@@ -843,10 +772,16 @@ public final class StructuralEncodingRules {
i++;
continue;
}
- int kEnd = Math.min(seqLen, end - i);
- // Truncated at probe-end is not an error — just stop here.
- if (kEnd < seqLen) {
- i = end;
+ // Truncated at probe-end is not an error event, but a provably-bad
+ // continuation in the partial tail still disqualifies UTF-8.
+ if (end - i < seqLen) {
+ for (int k = 1; k < end - i; k++) {
+ int cb = bytes[i + k] & 0xFF;
+ if (cb < 0x80 || cb > 0xBF) {
+ truncatedTailInvalid = true;
+ break;
+ }
+ }
break;
}
// Verify continuations are well-formed
@@ -860,11 +795,9 @@ public final class StructuralEncodingRules {
}
if (bad) {
errors++;
- // Skip the whole intended sequence. Advancing byte-by-byte
- // would re-count the orphaned continuations as additional
- // errors and inflate the count above Java's UTF-8 decoder's
- // U+FFFD-per-event semantics, which is the convention we
- // match for caller threshold comparisons.
+ // Skip the whole intended sequence: byte-by-byte advancement
+ // would re-count orphaned continuations and inflate the count
+ // above the U+FFFD-per-event convention.
i += seqLen;
continue;
}
@@ -889,84 +822,61 @@ public final class StructuralEncodingRules {
continue;
}
}
+ sequences++;
i += seqLen;
}
- return errors;
+ return new Utf8Stats(errors, sequences, truncatedTailInvalid);
+ }
+
+ /**
+ * Length (2/3/4) of a grammatically valid UTF-8 multi-byte sequence
+ * starting at {@code i}, or 0 if none. Lead ranges exclude overlong
+ * 2-byte (C0/C1) and out-of-range (≥F5) leads; continuations must be
+ * 0x80–0xBF. Deliberately grammar-only — NO codepoint-range/surrogate
+ * checks — so interleaved-decode callers (see
+ * {@link CjkDecodeValidator#strippedFailureRate}) classify a byte run as
+ * "UTF-8-shaped" cheaply without full validation.
+ */
+ public static int utf8SequenceLength(byte[] b, int i) {
+ int x = b[i] & 0xFF;
+ int len;
+ if (x >= 0xC2 && x <= 0xDF) {
+ len = 2;
+ } else if (x >= 0xE0 && x <= 0xEF) {
+ len = 3;
+ } else if (x >= 0xF0 && x <= 0xF4) {
+ len = 4;
+ } else {
+ return 0;
+ }
+ if (i + len > b.length) {
+ return 0;
+ }
+ for (int k = 1; k < len; k++) {
+ int c = b[i + k] & 0xFF;
+ if (c < 0x80 || c > 0xBF) {
+ return 0;
+ }
+ }
+ return len;
+ }
+
+ /** @see Utf8Stats#errors */
+ public static int countUtf8Errors(byte[] bytes) {
+ return utf8Stats(bytes).errors();
+ }
+
+ public static int countUtf8Errors(byte[] bytes, int offset, int length) {
+ return utf8Stats(bytes, offset, length).errors();
}
- /** Counts complete, valid multi-byte UTF-8 sequences — companion to
- * {@link #countUtf8Errors}, same walk, opposite tally. Gauges how much
- * genuine UTF-8 evidence a probe carries independent of its error count.
*/
+ /** @see Utf8Stats#sequences */
public static int countUtf8Sequences(byte[] bytes) {
- return countUtf8Sequences(bytes, 0, bytes.length);
+ return utf8Stats(bytes).sequences();
}
public static int countUtf8Sequences(byte[] bytes, int offset, int length)
{
- int sequences = 0;
- int i = offset;
- int end = offset + length;
- while (i < end) {
- int b = bytes[i] & 0xFF;
- if (b < 0x80) {
- i++;
- continue;
- }
- int seqLen;
- if (b >= 0xF8) {
- i++;
- continue;
- } else if (b >= 0xF0) {
- seqLen = 4;
- } else if (b >= 0xE0) {
- seqLen = 3;
- } else if (b >= 0xC0) {
- seqLen = 2;
- } else {
- i++;
- continue;
- }
- if (seqLen == 2 && b <= 0xC1) {
- i++;
- continue;
- }
- int kEnd = Math.min(seqLen, end - i);
- if (kEnd < seqLen) {
- break;
- }
- boolean bad = false;
- for (int k = 1; k < seqLen; k++) {
- int cb = bytes[i + k] & 0xFF;
- if (cb < 0x80 || cb > 0xBF) {
- bad = true;
- break;
- }
- }
- if (bad) {
- i += seqLen;
- continue;
- }
- if (seqLen == 3) {
- int cp = ((b & 0x0F) << 12)
- | ((bytes[i + 1] & 0xFF) & 0x3F) << 6
- | ((bytes[i + 2] & 0xFF) & 0x3F);
- if (cp < 0x0800 || (cp >= 0xD800 && cp <= 0xDFFF)) {
- i += seqLen;
- continue;
- }
- } else if (seqLen == 4) {
- int cp = ((b & 0x07) << 18)
- | ((bytes[i + 1] & 0xFF) & 0x3F) << 12
- | ((bytes[i + 2] & 0xFF) & 0x3F) << 6
- | ((bytes[i + 3] & 0xFF) & 0x3F);
- if (cp < 0x10000 || cp > 0x10FFFF) {
- i += seqLen;
- continue;
- }
- }
- sequences++;
- i += seqLen;
- }
- return sequences;
+ return utf8Stats(bytes, offset, length).sequences();
}
// -----------------------------------------------------------------------
diff --git
a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/test/java/org/apache/tika/ml/chardetect/ToleratedUtf8StructuralRegressionTest.java
b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/test/java/org/apache/tika/ml/chardetect/ToleratedUtf8StructuralRegressionTest.java
index 18ccefa994..f645b9bb46 100644
---
a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/test/java/org/apache/tika/ml/chardetect/ToleratedUtf8StructuralRegressionTest.java
+++
b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/test/java/org/apache/tika/ml/chardetect/ToleratedUtf8StructuralRegressionTest.java
@@ -55,10 +55,7 @@ public class ToleratedUtf8StructuralRegressionTest {
public void longDocumentWithOneStrayByteIsStillUtf8() throws IOException {
byte[] probe = buildProbe(30);
List<EncodingResult> results = newDetector().detect(probe);
- boolean hasStructuralUtf8 = results.stream().anyMatch(r ->
- "UTF-8".equals(r.getCharset().name())
- && r.getResultType() ==
EncodingResult.ResultType.STRUCTURAL);
- assertTrue(hasStructuralUtf8,
+ assertTrue(hasStructuralUtf8(results),
"A long, overwhelmingly UTF-8 document with a single tolerated
"
+ "error byte must still yield a STRUCTURAL UTF-8
candidate; "
+ "results were: " + results);
@@ -73,10 +70,7 @@ public class ToleratedUtf8StructuralRegressionTest {
byte[] probe = bo.toByteArray();
List<EncodingResult> results = newDetector().detect(probe);
- boolean hasStructuralUtf8 = results.stream().anyMatch(r ->
- "UTF-8".equals(r.getCharset().name())
- && r.getResultType() ==
EncodingResult.ResultType.STRUCTURAL);
- assertFalse(hasStructuralUtf8,
+ assertFalse(hasStructuralUtf8(results),
"A short probe shaped like a zip entry name must not be
promoted "
+ "to STRUCTURAL UTF-8 on a single tolerated error
alone; "
+ "results were: " + results);
@@ -87,10 +81,7 @@ public class ToleratedUtf8StructuralRegressionTest {
public void chineseGbkFilenameIsNotPromotedToUtf8() {
byte[] probe = "说明.txt".getBytes(Charset.forName("GBK"));
List<EncodingResult> results = newDetector().detect(probe);
- boolean hasStructuralUtf8 = results.stream().anyMatch(r ->
- "UTF-8".equals(r.getCharset().name())
- && r.getResultType() ==
EncodingResult.ResultType.STRUCTURAL);
- assertFalse(hasStructuralUtf8,
+ assertFalse(hasStructuralUtf8(results),
"A short GBK filename must not be promoted to STRUCTURAL UTF-8
"
+ "on a single tolerated error alone; results were: "
+ results);
assertTrue(results.stream().anyMatch(r ->
r.getCharset().name().startsWith("GB")),
@@ -102,14 +93,16 @@ public class ToleratedUtf8StructuralRegressionTest {
public void sauteFilenameIsNotPromotedToUtf8() {
byte[] probe = "Sauté.txt".getBytes(Charset.forName("windows-1252"));
List<EncodingResult> results = newDetector().detect(probe);
- boolean hasStructuralUtf8 = results.stream().anyMatch(r ->
- "UTF-8".equals(r.getCharset().name())
- && r.getResultType() ==
EncodingResult.ResultType.STRUCTURAL);
- assertFalse(hasStructuralUtf8,
+ assertFalse(hasStructuralUtf8(results),
"A short windows-1252 filename must not be promoted to
STRUCTURAL "
+ "UTF-8 on a single tolerated error alone; results
were: " + results);
}
+ private static boolean hasStructuralUtf8(List<EncodingResult> results) {
+ return results.stream().anyMatch(r ->
"UTF-8".equals(r.getCharset().name())
+ && r.getResultType() == EncodingResult.ResultType.STRUCTURAL);
+ }
+
/** Declared-windows-1252 HTML page, genuinely UTF-8, one stray raw ©
byte. */
private static byte[] buildProbe(int repeatCount) throws IOException {
StringBuilder body = new StringBuilder();
diff --git
a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/test/java/org/apache/tika/ml/chardetect/Utf8StatsTest.java
b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/test/java/org/apache/tika/ml/chardetect/Utf8StatsTest.java
new file mode 100644
index 0000000000..348b7d7988
--- /dev/null
+++
b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/test/java/org/apache/tika/ml/chardetect/Utf8StatsTest.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.ml.chardetect;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.nio.charset.StandardCharsets;
+
+import org.junit.jupiter.api.Test;
+
+import org.apache.tika.ml.chardetect.StructuralEncodingRules.Utf8Result;
+import org.apache.tika.ml.chardetect.StructuralEncodingRules.Utf8Stats;
+
+/**
+ * Pins the single-pass {@link StructuralEncodingRules#utf8Stats} contract —
+ * including the edges where the pre-consolidation {@code checkUtf8} and
+ * {@code countUtf8Errors} deliberately differed (a provably-bad truncated
+ * tail is NOT_UTF8 but not an error <em>event</em>).
+ */
+public class Utf8StatsTest {
+
+ private static Utf8Stats stats(int... unsignedBytes) {
+ byte[] b = new byte[unsignedBytes.length];
+ for (int i = 0; i < b.length; i++) {
+ b[i] = (byte) unsignedBytes[i];
+ }
+ return StructuralEncodingRules.utf8Stats(b);
+ }
+
+ @Test
+ public void pureAsciiIsAmbiguousWithNoCounts() {
+ Utf8Stats s = StructuralEncodingRules.utf8Stats(
+ "plain ascii only".getBytes(StandardCharsets.US_ASCII));
+ assertEquals(0, s.errors());
+ assertEquals(0, s.sequences());
+ assertEquals(Utf8Result.AMBIGUOUS, s.toResult());
+ }
+
+ @Test
+ public void completeSequencesAreLikelyAndCounted() {
+ Utf8Stats s = StructuralEncodingRules.utf8Stats(
+ "héllo wörld 中文 😀".getBytes(StandardCharsets.UTF_8));
+ assertEquals(0, s.errors());
+ assertEquals(5, s.sequences()); // é ö 中 文 😀
+ assertEquals(Utf8Result.LIKELY_UTF8, s.toResult());
+ }
+
+ @Test
+ public void truncatedCleanLeadAtEndIsAmbiguousNotError() {
+ // lone C3 at probe-end: valid-so-far, no structural evidence
+ Utf8Stats s = stats('a', 'b', 0xC3);
+ assertEquals(0, s.errors());
+ assertEquals(0, s.sequences());
+ assertFalse(s.truncatedTailInvalid());
+ assertEquals(Utf8Result.AMBIGUOUS, s.toResult());
+ }
+
+ @Test
+ public void provablyBadTruncatedTailIsNotUtf8ButNotAnErrorEvent() {
+ // E0 41 at probe-end: 3-byte lead + non-continuation — cannot be
UTF-8,
+ // but per U+FFFD-event semantics it is not counted as an error.
+ Utf8Stats s = stats('a', 0xE0, 0x41);
+ assertEquals(0, s.errors());
+ assertTrue(s.truncatedTailInvalid());
+ assertEquals(Utf8Result.NOT_UTF8, s.toResult());
+ }
+
+ @Test
+ public void classicErrorEventsEachCountOnce() {
+ // F8 lead + orphan continuation + overlong C0 + bad-continuation seq
+ Utf8Stats s = stats(0xF8, 'a', 0x80, 'b', 0xC0, 'c', 0xE0, 0x41, 0x41,
'd');
+ assertEquals(4, s.errors());
+ assertEquals(0, s.sequences());
+ assertEquals(Utf8Result.NOT_UTF8, s.toResult());
+ }
+
+ @Test
+ public void surrogateAndOverlongThreeByteAreErrors() {
+ // ED A0 80 = U+D800 surrogate; E0 80 80 = overlong (cp < 0x0800)
+ Utf8Stats s = stats(0xED, 0xA0, 0x80, 0xE0, 0x80, 0x80);
+ assertEquals(2, s.errors());
+ assertEquals(0, s.sequences());
+ }
+
+ @Test
+ public void mixedErrorsAndSequencesTallyIndependently() {
+ // one stray legacy byte before genuine multi-byte content (TIKA-4810
shape)
+ byte[] bengali = "সেমি".getBytes(StandardCharsets.UTF_8);
+ byte[] probe = new byte[1 + bengali.length];
+ probe[0] = (byte) 0xA9;
+ System.arraycopy(bengali, 0, probe, 1, bengali.length);
+ Utf8Stats s = StructuralEncodingRules.utf8Stats(probe);
+ assertEquals(1, s.errors());
+ assertEquals(4, s.sequences());
+ assertEquals(Utf8Result.NOT_UTF8, s.toResult());
+ }
+
+ @Test
+ public void sequenceLengthIsGrammarOnly() {
+ // grammar-only by design: overlong E0 80 80 still reports length 3
+ byte[] overlong = {(byte) 0xE0, (byte) 0x80, (byte) 0x80};
+ assertEquals(3, StructuralEncodingRules.utf8SequenceLength(overlong,
0));
+ byte[] twoByte = "é".getBytes(StandardCharsets.UTF_8);
+ assertEquals(2, StructuralEncodingRules.utf8SequenceLength(twoByte,
0));
+ byte[] c0Lead = {(byte) 0xC0, (byte) 0x80};
+ assertEquals(0, StructuralEncodingRules.utf8SequenceLength(c0Lead, 0));
+ byte[] truncated = {(byte) 0xE0, (byte) 0xA6};
+ assertEquals(0, StructuralEncodingRules.utf8SequenceLength(truncated,
0));
+ }
+}
diff --git
a/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/JunkFilterEncodingDetector.java
b/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/JunkFilterEncodingDetector.java
index 46f100e63d..0253502c32 100644
---
a/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/JunkFilterEncodingDetector.java
+++
b/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/JunkFilterEncodingDetector.java
@@ -83,10 +83,13 @@ public class JunkFilterEncodingDetector implements
MetaEncodingDetector {
private static final int DEFAULT_READ_LIMIT = 16384;
/** A STATISTICAL candidate at or below this confidence carries no real
- * signal — it's the "I don't know" level (matches Mojibuster's
- * windows-1252 fallback confidence). When the statistical layer offers
- * nothing above this, the junk-filter defers to a DECLARATIVE/STRUCTURAL
- * anchor instead of arbitrating near-identical decodes by quality. */
+ * signal — it's the "I don't know" level. MUST equal
+ * MojibusterEncodingDetector.FALLBACK_CONFIDENCE (tika-encoding-detector-
+ * mojibuster); the strict {@code >} in hasConfidentNonDeclarative is
+ * load-bearing: at exactly this value the fallback counts as abstention.
+ * When the statistical layer offers nothing above this, the junk-filter
+ * defers to a DECLARATIVE/STRUCTURAL anchor instead of arbitrating
+ * near-identical decodes by quality. */
private static final float NO_INFO_CONFIDENCE = 0.1f;
// Adaptive candidate band (TIKA speed lever). The tournament only needs
