This is an automated email from the ASF dual-hosted git repository. tballison pushed a commit to branch TIKA-4810-take2 in repository https://gitbox.apache.org/repos/asf/tika.git
commit b0900ac8f9df283d2a3252f5da52a7243ceeb834 Author: tallison <[email protected]> AuthorDate: Fri Aug 7 16:12:32 2026 -0400 TIKA-4810 -- follow-on: single-pass Utf8Stats walker + loose-end cleanup --- .skills/tika-eval-compare.md | 3 + .../tika/ml/chardetect/CjkDecodeValidator.java | 28 +- .../ml/chardetect/MojibusterEncodingDetector.java | 89 +------ .../ml/chardetect/StructuralEncodingRules.java | 284 +++++++-------------- .../ToleratedUtf8StructuralRegressionTest.java | 25 +- .../apache/tika/ml/chardetect/Utf8StatsTest.java | 126 +++++++++ .../ml/junkdetect/JunkFilterEncodingDetector.java | 11 +- 7 files changed, 251 insertions(+), 315 deletions(-) diff --git a/.skills/tika-eval-compare.md b/.skills/tika-eval-compare.md index cdb581e386..17058187c4 100644 --- a/.skills/tika-eval-compare.md +++ b/.skills/tika-eval-compare.md @@ -22,6 +22,9 @@ Ask the user for: zip archive containing `tika-app-*.jar`, `lib/`, and `plugins/`. - A corpus of input files (a directory tree). - tika-eval-app, built from `tika-eval/tika-eval-app` (use the zip). + The bare `target/tika-eval-app-*.jar` is thin (no bundled deps) and dies with + `NoClassDefFoundError: ...GzipCompressorOutputStream` (esp. with `-r`). Always + run the jar from the unzipped `target/*.zip` dir, which carries its `lib/`. - **Enable MD5 digesting** in both configs so tika-eval can match embedded documents by content hash (not just index position). Add to the config JSON: diff --git a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/CjkDecodeValidator.java b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/CjkDecodeValidator.java index 00f44dcee5..4dbe27977f 100644 --- a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/CjkDecodeValidator.java +++ b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/CjkDecodeValidator.java @@ -94,7 +94,7 @@ public final class CjkDecodeValidator { i++; continue; } - int ulen = utf8SequenceLength(bytes, i); + int ulen = StructuralEncodingRules.utf8SequenceLength(bytes, i); if (ulen > 0) { nUtf8Seqs++; i += ulen; // embedded UTF-8 — not legacy content, skip @@ -136,30 +136,4 @@ public final class CjkDecodeValidator { || name.contains("shift") || name.contains("jis") || name.contains("949"); } - /** Length (2/3/4) of a valid UTF-8 multi-byte sequence starting at {@code i}, - * or 0 if none. Lead-byte ranges exclude overlong 2-byte (C0/C1) and - * out-of-range (≥F5) leads; continuations must be 0x80–0xBF. */ - static int utf8SequenceLength(byte[] b, int i) { - int x = b[i] & 0xFF; - int len; - if (x >= 0xC2 && x <= 0xDF) { - len = 2; - } else if (x >= 0xE0 && x <= 0xEF) { - len = 3; - } else if (x >= 0xF0 && x <= 0xF4) { - len = 4; - } else { - return 0; - } - if (i + len > b.length) { - return 0; - } - for (int k = 1; k < len; k++) { - int c = b[i + k] & 0xFF; - if (c < 0x80 || c > 0xBF) { - return 0; - } - } - return len; - } } diff --git a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/MojibusterEncodingDetector.java b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/MojibusterEncodingDetector.java index a7f1909d9b..d0424b4573 100644 --- a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/MojibusterEncodingDetector.java +++ b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/MojibusterEncodingDetector.java @@ -140,7 +140,10 @@ public class MojibusterEncodingDetector implements EncodingDetector { * genuine false-CJK ≥5.3%, so ~2.5% separates them (see CjkDecodeValidator). */ private static final double CJK_FAILURE_VETO_THRESHOLD = 0.025; - /** Confidence for the windows-1252 fallback emitted on empty/ASCII probes. */ + /** Confidence for the windows-1252 fallback emitted on empty/ASCII probes. + * MUST equal JunkFilterEncodingDetector.NO_INFO_CONFIDENCE (tika-ml-junkdetect): + * its strict {@code confidence > NO_INFO_CONFIDENCE} test treats exactly this + * value as "statistical layer abstained" so a declaration can win. */ private static final float FALLBACK_CONFIDENCE = 0.1f; /** @@ -335,7 +338,8 @@ public class MojibusterEncodingDetector implements EncodingDetector { // because STRUCTURAL confidence outranks STATISTICAL. // • AMBIGUOUS (pure ASCII or only truncated lead): no // emission; NB + fallbacks handle it. - StructuralEncodingRules.Utf8Result utf8 = StructuralEncodingRules.checkUtf8(probe); + StructuralEncodingRules.Utf8Stats utf8Stats = StructuralEncodingRules.utf8Stats(probe); + StructuralEncodingRules.Utf8Result utf8 = utf8Stats.toResult(); // TACTICAL: tolerate small corruption. If the grammar check returned // NOT_UTF8 but the malformed-byte fraction is tiny, treat as UTF-8 — // a single bad continuation byte in 2KB of CJK is nearly always @@ -343,7 +347,7 @@ public class MojibusterEncodingDetector implements EncodingDetector { // replaced with a probabilistic decoder. boolean utf8Tolerated = false; if (utf8 == StructuralEncodingRules.Utf8Result.NOT_UTF8) { - int errors = StructuralEncodingRules.countUtf8Errors(probe); + int errors = utf8Stats.errors(); // Length-aware: absolute floor for short probes, rate for long ones. int maxTolerated = Math.max(UTF8_MAX_TOLERATED_ERRORS, (int) (probe.length * UTF8_MALFORMED_TOLERANCE)); @@ -371,7 +375,7 @@ public class MojibusterEncodingDetector implements EncodingDetector { // document its STRUCTURAL proof and JunkFilter has nothing to prefer // over the declared charset. boolean evidenceTolerated = utf8Tolerated - && StructuralEncodingRules.countUtf8Sequences(probe) >= MIN_TOLERATED_UTF8_SEQUENCES; + && utf8Stats.sequences() >= MIN_TOLERATED_UTF8_SEQUENCES; if (utf8 == StructuralEncodingRules.Utf8Result.LIKELY_UTF8 || evidenceTolerated) { pool.add(new EncodingResult( java.nio.charset.StandardCharsets.UTF_8, @@ -534,83 +538,6 @@ public class MojibusterEncodingDetector implements EncodingDetector { return true; } - /** - * Resolve UTF-16 to LE or BE once NB has called it "UTF-16". - * - * <p>Two deterministic tests: - * <ol> - * <li>Null-density: count null bytes in even-offset positions - * vs odd-offset positions. For ASCII-in-UTF-16-LE the - * high byte is 0x00 at odd positions; for BE it's at even - * positions. If one column is clearly null-dominant, that - * column indicates the endianness.</li> - * <li>Codepoint validity fallback: for ambiguous probes (pure - * CJK UTF-16, no nulls in either column) count how many - * 16-bit codepoints under LE vs BE interpretation land in - * assigned Unicode BMP ranges (non-PUA, non-unassigned). - * Whichever interpretation yields more valid codepoints - * wins.</li> - * </ol> - * - * <p>Also honors the {@code invalidUtf16Le}/{@code invalidUtf16Be} - * flags from {@link WideUnicodeDetector} — if either endianness - * is structurally invalid (surrogate-pair violation), the other - * wins by default. - * - * @return the resolved charset, or {@code null} if the probe is - * structurally invalid under both interpretations - */ - private static java.nio.charset.Charset disambiguateUtf16(byte[] probe, - boolean invalidLe, - boolean invalidBe) { - if (invalidLe && invalidBe) { - return null; - } - if (invalidLe) { - return java.nio.charset.Charset.forName("UTF-16BE"); - } - if (invalidBe) { - return java.nio.charset.Charset.forName("UTF-16LE"); - } - int nullEven = 0; - int nullOdd = 0; - for (int i = 0; i + 1 < probe.length; i += 2) { - if (probe[i] == 0) nullEven++; - if (probe[i + 1] == 0) nullOdd++; - } - // Clear null-density winner: one column is ≥ 3× more - // null-dominant than the other. - if (nullEven >= 3 * Math.max(1, nullOdd)) { - return java.nio.charset.Charset.forName("UTF-16BE"); - } - if (nullOdd >= 3 * Math.max(1, nullEven)) { - return java.nio.charset.Charset.forName("UTF-16LE"); - } - // Ambiguous on null-density (CJK content). Count valid BMP - // codepoints under each interpretation. A "valid" codepoint - // is any non-zero codepoint outside the surrogate range - // (0xD800-0xDFFF) — for CJK content most bytes map into - // assigned blocks, and random-byte-interpreted-as-UTF-16 - // produces many surrogate-range halves. - int validLe = 0; - int validBe = 0; - for (int i = 0; i + 1 < probe.length; i += 2) { - int lo = probe[i] & 0xFF; - int hi = probe[i + 1] & 0xFF; - int leCp = (hi << 8) | lo; - int beCp = (lo << 8) | hi; - if (leCp != 0 && (leCp < 0xD800 || leCp > 0xDFFF)) { - validLe++; - } - if (beCp != 0 && (beCp < 0xD800 || beCp > 0xDFFF)) { - validBe++; - } - } - return validLe >= validBe - ? java.nio.charset.Charset.forName("UTF-16LE") - : java.nio.charset.Charset.forName("UTF-16BE"); - } - /** * Relabel the top result to windows-1252 when top is a non-1252 * member of {@link CharsetConfusables#SBCS_LATIN_FAMILY} and diff --git a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/StructuralEncodingRules.java b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/StructuralEncodingRules.java index 67eb7c7503..bbd9a371d2 100644 --- a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/StructuralEncodingRules.java +++ b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/StructuralEncodingRules.java @@ -656,97 +656,7 @@ public final class StructuralEncodingRules { } public static Utf8Result checkUtf8(byte[] bytes, int offset, int length) { - int highByteCount = 0; - // Count only multi-byte sequences whose continuations are all - // present in the probe. A truncated lead at probe-end is - // valid-so-far but provides no structural evidence — a single - // 0xC3 byte alone should not be enough to claim UTF-8. - int completeHighSeqCount = 0; - int i = offset; - int end = offset + length; - - while (i < end) { - int b = bytes[i] & 0xFF; - - if (b < 0x80) { - i++; - continue; - } - - highByteCount++; - - // Determine expected continuation count from the lead byte - int seqLen; - if (b >= 0xF8) { - // 5-/6-byte sequences are not valid Unicode - return Utf8Result.NOT_UTF8; - } else if (b >= 0xF0) { - seqLen = 4; - } else if (b >= 0xE0) { - seqLen = 3; - } else if (b >= 0xC0) { - seqLen = 2; - } else { - // 0x80–0xBF is a continuation byte without a lead → invalid - return Utf8Result.NOT_UTF8; - } - - // Overlong 2-byte sequence (C0 or C1 lead) - if (seqLen == 2 && b <= 0xC1) { - return Utf8Result.NOT_UTF8; - } - - // Check that the right number of continuation bytes follow - boolean truncated = false; - for (int k = 1; k < seqLen; k++) { - if (i + k >= end) { - truncated = true; - break; - } - int cb = bytes[i + k] & 0xFF; - if (cb < 0x80 || cb > 0xBF) { - return Utf8Result.NOT_UTF8; - } - } - - // Validate scalar value ranges for 3- and 4-byte sequences, - // but only when the full sequence is present. Truncated sequences - // at the end of a probe are not evidence of invalid UTF-8. - if (!truncated) { - if (seqLen == 3) { - int cp = ((b & 0x0F) << 12) - | ((bytes[i + 1] & 0xFF) & 0x3F) << 6 - | ((bytes[i + 2] & 0xFF) & 0x3F); - if (cp < 0x0800 || (cp >= 0xD800 && cp <= 0xDFFF)) { - return Utf8Result.NOT_UTF8; - } - } else if (seqLen == 4) { - int cp = ((b & 0x07) << 18) - | ((bytes[i + 1] & 0xFF) & 0x3F) << 12 - | ((bytes[i + 2] & 0xFF) & 0x3F) << 6 - | ((bytes[i + 3] & 0xFF) & 0x3F); - if (cp < 0x10000 || cp > 0x10FFFF) { - return Utf8Result.NOT_UTF8; - } - } - completeHighSeqCount++; - } - - i += seqLen; - } - - // Grammar is valid. Require at least one COMPLETE multi-byte - // sequence (not just a truncated lead at probe-end) to claim - // LIKELY. A lone 0xC3 at the end of a 1-byte probe is - // valid-so-far but provides no structural evidence. - if (completeHighSeqCount > 0) { - return Utf8Result.LIKELY_UTF8; - } - // highByteCount > 0 but completeHighSeqCount == 0 means every - // high byte we saw was the start of a truncated sequence at - // probe-end. That's not enough evidence — treat as ambiguous. - // Zero high bytes = pure ASCII; caller handles this separately. - return Utf8Result.AMBIGUOUS; + return utf8Stats(bytes, offset, length).toResult(); } /** @@ -787,30 +697,49 @@ public final class StructuralEncodingRules { } /** - * Counts the number of malformed UTF-8 <em>sequences</em> in the sample — - * one event per bad lead, orphaned continuation, overlong, surrogate, or - * out-of-range codepoint, regardless of how many bytes the bad sequence - * spans. Unlike {@link #checkUtf8}, this does not early-exit on the - * first bad sequence; it scans the entire range, resyncing after each - * error. Returns 0 for a clean UTF-8 stream. + * Single-pass tally of UTF-8 structure over the sample: malformed-sequence + * events and complete valid multi-byte sequences. Same walk as + * {@link #checkUtf8} but without the early exit — scans the entire range, + * resyncing after each error. * - * <p>Useful for "tolerant" UTF-8 acceptance: a real-world UTF-8 file with - * a few corrupted sequences (copy-paste artefact, truncated upstream, - * MIME transport flip) should still be recognized as UTF-8 rather than - * rejected outright. Caller decides what error count is tolerable - * (typically as a fraction of probe length).</p> + * <p>{@code errors} counts one event per bad lead, orphaned continuation, + * overlong, surrogate, or out-of-range codepoint regardless of how many + * bytes the bad sequence spans — matching Java's + * {@code new String(bytes, UTF_8)} U+FFFD-per-error semantics, the + * convention callers use for tolerance thresholds. A truncated sequence + * at probe-end is neither an error nor a sequence.</p> * - * <p>The count matches Java's {@code new String(bytes, UTF_8)}'s - * U+FFFD-per-error semantics (one replacement per malformed sequence).</p> + * <p>{@code sequences} gauges how much genuine multi-byte UTF-8 evidence + * the probe carries independent of its error count.</p> * - * @return number of malformed UTF-8 sequence events + * <p>{@code truncatedTailInvalid}: the probe ends in a truncated sequence + * whose partial continuations are provably NOT UTF-8 (e.g. {@code E0 41} + * at probe-end). Not an error <em>event</em> (no U+FFFD equivalent — the + * decoder would ask for more input), but it disqualifies the sample from + * {@link Utf8Result#LIKELY_UTF8}/{@link Utf8Result#AMBIGUOUS}.</p> */ - public static int countUtf8Errors(byte[] bytes) { - return countUtf8Errors(bytes, 0, bytes.length); + public record Utf8Stats(int errors, int sequences, boolean truncatedTailInvalid) { + + /** Collapse to the {@link #checkUtf8} tri-state: any invalidity → + * NOT_UTF8; else ≥1 complete multi-byte sequence → LIKELY_UTF8 + * (a lone truncated lead is no structural evidence); else AMBIGUOUS + * (pure ASCII, or truncated-lead-only). */ + public Utf8Result toResult() { + if (errors > 0 || truncatedTailInvalid) { + return Utf8Result.NOT_UTF8; + } + return sequences > 0 ? Utf8Result.LIKELY_UTF8 : Utf8Result.AMBIGUOUS; + } } - public static int countUtf8Errors(byte[] bytes, int offset, int length) { + public static Utf8Stats utf8Stats(byte[] bytes) { + return utf8Stats(bytes, 0, bytes.length); + } + + public static Utf8Stats utf8Stats(byte[] bytes, int offset, int length) { int errors = 0; + int sequences = 0; + boolean truncatedTailInvalid = false; int i = offset; int end = offset + length; while (i < end) { @@ -843,10 +772,16 @@ public final class StructuralEncodingRules { i++; continue; } - int kEnd = Math.min(seqLen, end - i); - // Truncated at probe-end is not an error — just stop here. - if (kEnd < seqLen) { - i = end; + // Truncated at probe-end is not an error event, but a provably-bad + // continuation in the partial tail still disqualifies UTF-8. + if (end - i < seqLen) { + for (int k = 1; k < end - i; k++) { + int cb = bytes[i + k] & 0xFF; + if (cb < 0x80 || cb > 0xBF) { + truncatedTailInvalid = true; + break; + } + } break; } // Verify continuations are well-formed @@ -860,11 +795,9 @@ public final class StructuralEncodingRules { } if (bad) { errors++; - // Skip the whole intended sequence. Advancing byte-by-byte - // would re-count the orphaned continuations as additional - // errors and inflate the count above Java's UTF-8 decoder's - // U+FFFD-per-event semantics, which is the convention we - // match for caller threshold comparisons. + // Skip the whole intended sequence: byte-by-byte advancement + // would re-count orphaned continuations and inflate the count + // above the U+FFFD-per-event convention. i += seqLen; continue; } @@ -889,84 +822,61 @@ public final class StructuralEncodingRules { continue; } } + sequences++; i += seqLen; } - return errors; + return new Utf8Stats(errors, sequences, truncatedTailInvalid); + } + + /** + * Length (2/3/4) of a grammatically valid UTF-8 multi-byte sequence + * starting at {@code i}, or 0 if none. Lead ranges exclude overlong + * 2-byte (C0/C1) and out-of-range (≥F5) leads; continuations must be + * 0x80–0xBF. Deliberately grammar-only — NO codepoint-range/surrogate + * checks — so interleaved-decode callers (see + * {@link CjkDecodeValidator#strippedFailureRate}) classify a byte run as + * "UTF-8-shaped" cheaply without full validation. + */ + public static int utf8SequenceLength(byte[] b, int i) { + int x = b[i] & 0xFF; + int len; + if (x >= 0xC2 && x <= 0xDF) { + len = 2; + } else if (x >= 0xE0 && x <= 0xEF) { + len = 3; + } else if (x >= 0xF0 && x <= 0xF4) { + len = 4; + } else { + return 0; + } + if (i + len > b.length) { + return 0; + } + for (int k = 1; k < len; k++) { + int c = b[i + k] & 0xFF; + if (c < 0x80 || c > 0xBF) { + return 0; + } + } + return len; + } + + /** @see Utf8Stats#errors */ + public static int countUtf8Errors(byte[] bytes) { + return utf8Stats(bytes).errors(); + } + + public static int countUtf8Errors(byte[] bytes, int offset, int length) { + return utf8Stats(bytes, offset, length).errors(); } - /** Counts complete, valid multi-byte UTF-8 sequences — companion to - * {@link #countUtf8Errors}, same walk, opposite tally. Gauges how much - * genuine UTF-8 evidence a probe carries independent of its error count. */ + /** @see Utf8Stats#sequences */ public static int countUtf8Sequences(byte[] bytes) { - return countUtf8Sequences(bytes, 0, bytes.length); + return utf8Stats(bytes).sequences(); } public static int countUtf8Sequences(byte[] bytes, int offset, int length) { - int sequences = 0; - int i = offset; - int end = offset + length; - while (i < end) { - int b = bytes[i] & 0xFF; - if (b < 0x80) { - i++; - continue; - } - int seqLen; - if (b >= 0xF8) { - i++; - continue; - } else if (b >= 0xF0) { - seqLen = 4; - } else if (b >= 0xE0) { - seqLen = 3; - } else if (b >= 0xC0) { - seqLen = 2; - } else { - i++; - continue; - } - if (seqLen == 2 && b <= 0xC1) { - i++; - continue; - } - int kEnd = Math.min(seqLen, end - i); - if (kEnd < seqLen) { - break; - } - boolean bad = false; - for (int k = 1; k < seqLen; k++) { - int cb = bytes[i + k] & 0xFF; - if (cb < 0x80 || cb > 0xBF) { - bad = true; - break; - } - } - if (bad) { - i += seqLen; - continue; - } - if (seqLen == 3) { - int cp = ((b & 0x0F) << 12) - | ((bytes[i + 1] & 0xFF) & 0x3F) << 6 - | ((bytes[i + 2] & 0xFF) & 0x3F); - if (cp < 0x0800 || (cp >= 0xD800 && cp <= 0xDFFF)) { - i += seqLen; - continue; - } - } else if (seqLen == 4) { - int cp = ((b & 0x07) << 18) - | ((bytes[i + 1] & 0xFF) & 0x3F) << 12 - | ((bytes[i + 2] & 0xFF) & 0x3F) << 6 - | ((bytes[i + 3] & 0xFF) & 0x3F); - if (cp < 0x10000 || cp > 0x10FFFF) { - i += seqLen; - continue; - } - } - sequences++; - i += seqLen; - } - return sequences; + return utf8Stats(bytes, offset, length).sequences(); } // ----------------------------------------------------------------------- diff --git a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/test/java/org/apache/tika/ml/chardetect/ToleratedUtf8StructuralRegressionTest.java b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/test/java/org/apache/tika/ml/chardetect/ToleratedUtf8StructuralRegressionTest.java index 18ccefa994..f645b9bb46 100644 --- a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/test/java/org/apache/tika/ml/chardetect/ToleratedUtf8StructuralRegressionTest.java +++ b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/test/java/org/apache/tika/ml/chardetect/ToleratedUtf8StructuralRegressionTest.java @@ -55,10 +55,7 @@ public class ToleratedUtf8StructuralRegressionTest { public void longDocumentWithOneStrayByteIsStillUtf8() throws IOException { byte[] probe = buildProbe(30); List<EncodingResult> results = newDetector().detect(probe); - boolean hasStructuralUtf8 = results.stream().anyMatch(r -> - "UTF-8".equals(r.getCharset().name()) - && r.getResultType() == EncodingResult.ResultType.STRUCTURAL); - assertTrue(hasStructuralUtf8, + assertTrue(hasStructuralUtf8(results), "A long, overwhelmingly UTF-8 document with a single tolerated " + "error byte must still yield a STRUCTURAL UTF-8 candidate; " + "results were: " + results); @@ -73,10 +70,7 @@ public class ToleratedUtf8StructuralRegressionTest { byte[] probe = bo.toByteArray(); List<EncodingResult> results = newDetector().detect(probe); - boolean hasStructuralUtf8 = results.stream().anyMatch(r -> - "UTF-8".equals(r.getCharset().name()) - && r.getResultType() == EncodingResult.ResultType.STRUCTURAL); - assertFalse(hasStructuralUtf8, + assertFalse(hasStructuralUtf8(results), "A short probe shaped like a zip entry name must not be promoted " + "to STRUCTURAL UTF-8 on a single tolerated error alone; " + "results were: " + results); @@ -87,10 +81,7 @@ public class ToleratedUtf8StructuralRegressionTest { public void chineseGbkFilenameIsNotPromotedToUtf8() { byte[] probe = "说明.txt".getBytes(Charset.forName("GBK")); List<EncodingResult> results = newDetector().detect(probe); - boolean hasStructuralUtf8 = results.stream().anyMatch(r -> - "UTF-8".equals(r.getCharset().name()) - && r.getResultType() == EncodingResult.ResultType.STRUCTURAL); - assertFalse(hasStructuralUtf8, + assertFalse(hasStructuralUtf8(results), "A short GBK filename must not be promoted to STRUCTURAL UTF-8 " + "on a single tolerated error alone; results were: " + results); assertTrue(results.stream().anyMatch(r -> r.getCharset().name().startsWith("GB")), @@ -102,14 +93,16 @@ public class ToleratedUtf8StructuralRegressionTest { public void sauteFilenameIsNotPromotedToUtf8() { byte[] probe = "Sauté.txt".getBytes(Charset.forName("windows-1252")); List<EncodingResult> results = newDetector().detect(probe); - boolean hasStructuralUtf8 = results.stream().anyMatch(r -> - "UTF-8".equals(r.getCharset().name()) - && r.getResultType() == EncodingResult.ResultType.STRUCTURAL); - assertFalse(hasStructuralUtf8, + assertFalse(hasStructuralUtf8(results), "A short windows-1252 filename must not be promoted to STRUCTURAL " + "UTF-8 on a single tolerated error alone; results were: " + results); } + private static boolean hasStructuralUtf8(List<EncodingResult> results) { + return results.stream().anyMatch(r -> "UTF-8".equals(r.getCharset().name()) + && r.getResultType() == EncodingResult.ResultType.STRUCTURAL); + } + /** Declared-windows-1252 HTML page, genuinely UTF-8, one stray raw © byte. */ private static byte[] buildProbe(int repeatCount) throws IOException { StringBuilder body = new StringBuilder(); diff --git a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/test/java/org/apache/tika/ml/chardetect/Utf8StatsTest.java b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/test/java/org/apache/tika/ml/chardetect/Utf8StatsTest.java new file mode 100644 index 0000000000..348b7d7988 --- /dev/null +++ b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/test/java/org/apache/tika/ml/chardetect/Utf8StatsTest.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.ml.chardetect; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.nio.charset.StandardCharsets; + +import org.junit.jupiter.api.Test; + +import org.apache.tika.ml.chardetect.StructuralEncodingRules.Utf8Result; +import org.apache.tika.ml.chardetect.StructuralEncodingRules.Utf8Stats; + +/** + * Pins the single-pass {@link StructuralEncodingRules#utf8Stats} contract — + * including the edges where the pre-consolidation {@code checkUtf8} and + * {@code countUtf8Errors} deliberately differed (a provably-bad truncated + * tail is NOT_UTF8 but not an error <em>event</em>). + */ +public class Utf8StatsTest { + + private static Utf8Stats stats(int... unsignedBytes) { + byte[] b = new byte[unsignedBytes.length]; + for (int i = 0; i < b.length; i++) { + b[i] = (byte) unsignedBytes[i]; + } + return StructuralEncodingRules.utf8Stats(b); + } + + @Test + public void pureAsciiIsAmbiguousWithNoCounts() { + Utf8Stats s = StructuralEncodingRules.utf8Stats( + "plain ascii only".getBytes(StandardCharsets.US_ASCII)); + assertEquals(0, s.errors()); + assertEquals(0, s.sequences()); + assertEquals(Utf8Result.AMBIGUOUS, s.toResult()); + } + + @Test + public void completeSequencesAreLikelyAndCounted() { + Utf8Stats s = StructuralEncodingRules.utf8Stats( + "héllo wörld 中文 😀".getBytes(StandardCharsets.UTF_8)); + assertEquals(0, s.errors()); + assertEquals(5, s.sequences()); // é ö 中 文 😀 + assertEquals(Utf8Result.LIKELY_UTF8, s.toResult()); + } + + @Test + public void truncatedCleanLeadAtEndIsAmbiguousNotError() { + // lone C3 at probe-end: valid-so-far, no structural evidence + Utf8Stats s = stats('a', 'b', 0xC3); + assertEquals(0, s.errors()); + assertEquals(0, s.sequences()); + assertFalse(s.truncatedTailInvalid()); + assertEquals(Utf8Result.AMBIGUOUS, s.toResult()); + } + + @Test + public void provablyBadTruncatedTailIsNotUtf8ButNotAnErrorEvent() { + // E0 41 at probe-end: 3-byte lead + non-continuation — cannot be UTF-8, + // but per U+FFFD-event semantics it is not counted as an error. + Utf8Stats s = stats('a', 0xE0, 0x41); + assertEquals(0, s.errors()); + assertTrue(s.truncatedTailInvalid()); + assertEquals(Utf8Result.NOT_UTF8, s.toResult()); + } + + @Test + public void classicErrorEventsEachCountOnce() { + // F8 lead + orphan continuation + overlong C0 + bad-continuation seq + Utf8Stats s = stats(0xF8, 'a', 0x80, 'b', 0xC0, 'c', 0xE0, 0x41, 0x41, 'd'); + assertEquals(4, s.errors()); + assertEquals(0, s.sequences()); + assertEquals(Utf8Result.NOT_UTF8, s.toResult()); + } + + @Test + public void surrogateAndOverlongThreeByteAreErrors() { + // ED A0 80 = U+D800 surrogate; E0 80 80 = overlong (cp < 0x0800) + Utf8Stats s = stats(0xED, 0xA0, 0x80, 0xE0, 0x80, 0x80); + assertEquals(2, s.errors()); + assertEquals(0, s.sequences()); + } + + @Test + public void mixedErrorsAndSequencesTallyIndependently() { + // one stray legacy byte before genuine multi-byte content (TIKA-4810 shape) + byte[] bengali = "সেমি".getBytes(StandardCharsets.UTF_8); + byte[] probe = new byte[1 + bengali.length]; + probe[0] = (byte) 0xA9; + System.arraycopy(bengali, 0, probe, 1, bengali.length); + Utf8Stats s = StructuralEncodingRules.utf8Stats(probe); + assertEquals(1, s.errors()); + assertEquals(4, s.sequences()); + assertEquals(Utf8Result.NOT_UTF8, s.toResult()); + } + + @Test + public void sequenceLengthIsGrammarOnly() { + // grammar-only by design: overlong E0 80 80 still reports length 3 + byte[] overlong = {(byte) 0xE0, (byte) 0x80, (byte) 0x80}; + assertEquals(3, StructuralEncodingRules.utf8SequenceLength(overlong, 0)); + byte[] twoByte = "é".getBytes(StandardCharsets.UTF_8); + assertEquals(2, StructuralEncodingRules.utf8SequenceLength(twoByte, 0)); + byte[] c0Lead = {(byte) 0xC0, (byte) 0x80}; + assertEquals(0, StructuralEncodingRules.utf8SequenceLength(c0Lead, 0)); + byte[] truncated = {(byte) 0xE0, (byte) 0xA6}; + assertEquals(0, StructuralEncodingRules.utf8SequenceLength(truncated, 0)); + } +} diff --git a/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/JunkFilterEncodingDetector.java b/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/JunkFilterEncodingDetector.java index 46f100e63d..0253502c32 100644 --- a/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/JunkFilterEncodingDetector.java +++ b/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/JunkFilterEncodingDetector.java @@ -83,10 +83,13 @@ public class JunkFilterEncodingDetector implements MetaEncodingDetector { private static final int DEFAULT_READ_LIMIT = 16384; /** A STATISTICAL candidate at or below this confidence carries no real - * signal — it's the "I don't know" level (matches Mojibuster's - * windows-1252 fallback confidence). When the statistical layer offers - * nothing above this, the junk-filter defers to a DECLARATIVE/STRUCTURAL - * anchor instead of arbitrating near-identical decodes by quality. */ + * signal — it's the "I don't know" level. MUST equal + * MojibusterEncodingDetector.FALLBACK_CONFIDENCE (tika-encoding-detector- + * mojibuster); the strict {@code >} in hasConfidentNonDeclarative is + * load-bearing: at exactly this value the fallback counts as abstention. + * When the statistical layer offers nothing above this, the junk-filter + * defers to a DECLARATIVE/STRUCTURAL anchor instead of arbitrating + * near-identical decodes by quality. */ private static final float NO_INFO_CONFIDENCE = 0.1f; // Adaptive candidate band (TIKA speed lever). The tournament only needs
