This is an automated email from the ASF dual-hosted git repository. tballison pushed a commit to branch TIKA-4810-fix-mojibuster-utf8-tolerance-regression in repository https://gitbox.apache.org/repos/asf/tika.git
commit 31c6651d855e1d626bd40891b6eddbb5e6724b6c Author: tallison <[email protected]> AuthorDate: Fri Aug 7 14:16:04 2026 -0400 TIKA-4810 -- Restore tolerated-UTF-8 structural promotion, gated on evidence volume --- .../ml/chardetect/MojibusterEncodingDetector.java | 40 +++-- .../ml/chardetect/StructuralEncodingRules.java | 83 ++++++++++ .../ToleratedUtf8StructuralRegressionTest.java | 175 +++++++++++++++++++++ 3 files changed, 288 insertions(+), 10 deletions(-) diff --git a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/MojibusterEncodingDetector.java b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/MojibusterEncodingDetector.java index f3e5417731..72af572733 100644 --- a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/MojibusterEncodingDetector.java +++ b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/MojibusterEncodingDetector.java @@ -174,6 +174,18 @@ public class MojibusterEncodingDetector implements EncodingDetector { */ private static final int UTF8_MAX_TOLERATED_ERRORS = 1; + /** + * Minimum count of complete, valid multi-byte UTF-8 sequences required before + * a tolerated (NOT_UTF8-but-within-error-budget) probe is promoted to a + * STRUCTURAL UTF-8 candidate. Tolerance alone isn't enough evidence at any + * length — 1 error in a 20-byte zip entry name is a 5% error rate, easily a + * coincidentally-valid legacy-encoded string, not corrupted UTF-8. Requiring + * substantial genuine multi-byte evidence (mirrors {@link + * CjkDecodeValidator#MIN_HIGH_BYTES}) separates that short-probe false-positive + * risk from the long-document case this tolerance mechanism exists for. + */ + private static final int MIN_TOLERATED_UTF8_SEQUENCES = 30; + /** Windows-1252: the WHATWG-canonical default for unlabeled Western content. */ private static final String WIN1252 = "windows-1252"; @@ -355,16 +367,24 @@ public class MojibusterEncodingDetector implements EncodingDetector { } } LOG.trace("mojibuster utf8Check={} tolerated={}", utf8, utf8Tolerated); - // Emit a structural UTF-8 candidate only when the grammar is definitively - // clean (LIKELY_UTF8). When the probe is NOT_UTF8 but within the error - // tolerance (utf8Tolerated), NB's UTF-8 result is already kept as a - // STATISTICAL candidate (see NOT_UTF8 disqualifier above) — promoting it - // to STRUCTURAL here would cause the "return only top-1 STRUCTURAL" path - // to short-circuit JunkFilter, preventing it from comparing UTF-8 against - // windows-1252. For short probes a single bad byte in otherwise-ASCII - // content is more likely a genuine Latin-1/windows-1252 byte than a - // corrupt UTF-8 sequence; JunkFilter has enough signal to arbitrate. - if (utf8 == StructuralEncodingRules.Utf8Result.LIKELY_UTF8) { + // Emit a structural UTF-8 candidate when the grammar is definitively clean + // (LIKELY_UTF8), OR when it's tolerated AND backed by abundant genuine + // multi-byte evidence (evidenceTolerated below). Bare tolerance is not + // promoted: on a short probe (e.g. a zip entry name — ZipParser routes + // entry-name bytes through this same detector) NB's UTF-8 result is + // already kept as a STATISTICAL candidate (see NOT_UTF8 disqualifier + // above), and a single tolerated error there is more likely a + // coincidentally-valid legacy-encoded string than corrupted UTF-8 — regr- + // ession-tested in ToleratedUtf8StructuralRegressionTest. On a long, + // overwhelmingly-UTF-8 document a single stray legacy byte (e.g. a raw + // 0xA9 copyright sign) must not cost the whole document its STRUCTURAL + // proof: NB can come back with an empty pool for some scripts, leaving + // nothing for JunkFilter to prefer over the declared charset — real-world + // regression from commit 360b3d354 (2026-06-10), which dropped this + // branch entirely on the assumption that an NB fallback always exists. + boolean evidenceTolerated = utf8Tolerated + && StructuralEncodingRules.countUtf8Sequences(probe) >= MIN_TOLERATED_UTF8_SEQUENCES; + if (utf8 == StructuralEncodingRules.Utf8Result.LIKELY_UTF8 || evidenceTolerated) { pool.add(new EncodingResult( java.nio.charset.StandardCharsets.UTF_8, UTF8_STRUCTURAL_CONF, "UTF-8", diff --git a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/StructuralEncodingRules.java b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/StructuralEncodingRules.java index 2302f110d4..6bdb2be441 100644 --- a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/StructuralEncodingRules.java +++ b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/StructuralEncodingRules.java @@ -894,6 +894,89 @@ public final class StructuralEncodingRules { return errors; } + /** + * Counts complete, valid multi-byte UTF-8 sequences in the sample — + * companion to {@link #countUtf8Errors}, same walk, opposite tally. Used + * to gauge how much genuine UTF-8 evidence a probe carries independent of + * its error count: a probe with one tolerated error and hundreds of valid + * sequences is overwhelmingly UTF-8; a probe with one tolerated error and + * two or three valid sequences (a short filename, say) is not distinguishable + * from a coincidentally-valid legacy-encoded string. + * + * @return number of complete, well-formed multi-byte UTF-8 sequences + */ + public static int countUtf8Sequences(byte[] bytes) { + return countUtf8Sequences(bytes, 0, bytes.length); + } + + public static int countUtf8Sequences(byte[] bytes, int offset, int length) { + int sequences = 0; + int i = offset; + int end = offset + length; + while (i < end) { + int b = bytes[i] & 0xFF; + if (b < 0x80) { + i++; + continue; + } + int seqLen; + if (b >= 0xF8) { + i++; + continue; + } else if (b >= 0xF0) { + seqLen = 4; + } else if (b >= 0xE0) { + seqLen = 3; + } else if (b >= 0xC0) { + seqLen = 2; + } else { + i++; + continue; + } + if (seqLen == 2 && b <= 0xC1) { + i++; + continue; + } + int kEnd = Math.min(seqLen, end - i); + if (kEnd < seqLen) { + break; + } + boolean bad = false; + for (int k = 1; k < seqLen; k++) { + int cb = bytes[i + k] & 0xFF; + if (cb < 0x80 || cb > 0xBF) { + bad = true; + break; + } + } + if (bad) { + i += seqLen; + continue; + } + if (seqLen == 3) { + int cp = ((b & 0x0F) << 12) + | ((bytes[i + 1] & 0xFF) & 0x3F) << 6 + | ((bytes[i + 2] & 0xFF) & 0x3F); + if (cp < 0x0800 || (cp >= 0xD800 && cp <= 0xDFFF)) { + i += seqLen; + continue; + } + } else if (seqLen == 4) { + int cp = ((b & 0x07) << 18) + | ((bytes[i + 1] & 0xFF) & 0x3F) << 12 + | ((bytes[i + 2] & 0xFF) & 0x3F) << 6 + | ((bytes[i + 3] & 0xFF) & 0x3F); + if (cp < 0x10000 || cp > 0x10FFFF) { + i += seqLen; + continue; + } + } + sequences++; + i += seqLen; + } + return sequences; + } + // ----------------------------------------------------------------------- // Result type // ----------------------------------------------------------------------- diff --git a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/test/java/org/apache/tika/ml/chardetect/ToleratedUtf8StructuralRegressionTest.java b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/test/java/org/apache/tika/ml/chardetect/ToleratedUtf8StructuralRegressionTest.java new file mode 100644 index 0000000000..28b5cedb75 --- /dev/null +++ b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/test/java/org/apache/tika/ml/chardetect/ToleratedUtf8StructuralRegressionTest.java @@ -0,0 +1,175 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.ml.chardetect; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.util.List; + +import org.junit.jupiter.api.Test; + +import org.apache.tika.detect.EncodingResult; + +/** + * Regression test for a real-world failure: a genuinely UTF-8 HTML page whose + * only single-byte "legacy" artifact (a stray {@code ©} written as raw + * {@code 0xA9} rather than an entity) sits before the bulk of the document's + * real multi-byte content. {@link StructuralEncodingRules#checkUtf8} correctly + * reports {@code NOT_UTF8} for the whole probe (one malformed lead byte), and + * the tolerance mechanism in {@link MojibusterEncodingDetector} is supposed to + * recognize this as "essentially UTF-8" when there's abundant genuine + * multi-byte evidence. + * + * <p>Commit 360b3d354 ("merge conflict and flaky test", 2026-06-10) dropped the + * {@code || utf8Tolerated} branch that used to promote this case to a + * STRUCTURAL UTF-8 candidate, on the assumption that the NB statistical layer + * would independently propose UTF-8 as a fallback. That assumption doesn't + * hold for every script/corpus (verified against a real Bengali-language news + * page): NB's own candidate pool can come back completely empty, leaving + * Mojibuster with nothing but the {@code windows-1252} "give up" default — + * silent, complete mojibake on an otherwise-clean UTF-8 document.</p> + * + * <p>The companion {@link #shortProbeWithOneStrayByteIsNotPromoted()} test + * guards the reason that branch was narrowed in the first place: zip entry + * names are typically 9-30 bytes, and {@link + * org.apache.tika.parser.pkg.ZipParser} runs them through this same detector + * (see {@code ZipParser#isDetectCharsetsInEntryNames}). A single coincidental + * error byte in a short, genuinely-legacy-encoded filename must NOT be enough + * to promote it to STRUCTURAL UTF-8 — that would re-open the false-positive + * this detector is relied on to avoid for filenames.</p> + */ +public class ToleratedUtf8StructuralRegressionTest { + + private static final String BENGALI_SENTENCE = + "সেমিতে ক্রোয়েশিয়া টাইব্রেকারে রাশিয়াকে হারিয়ে ফাইনালে উঠেছে। "; + + private static MojibusterEncodingDetector newDetector() { + try { + return new MojibusterEncodingDetector(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + /** + * Long document, abundant genuine multi-byte UTF-8 evidence, exactly one + * tolerated error byte before it. Must still be recognized as UTF-8. + */ + @Test + public void longDocumentWithOneStrayByteIsStillUtf8() throws IOException { + byte[] probe = buildProbe(30); + List<EncodingResult> results = newDetector().detect(probe); + boolean hasStructuralUtf8 = results.stream().anyMatch(r -> + "UTF-8".equals(r.getCharset().name()) + && r.getResultType() == EncodingResult.ResultType.STRUCTURAL); + assertTrue(hasStructuralUtf8, + "A long, overwhelmingly UTF-8 document with a single tolerated " + + "error byte must still yield a STRUCTURAL UTF-8 candidate; " + + "results were: " + results); + } + + /** + * Short probe (the zip-entry-name shape), exactly one error byte, only a + * handful of genuine multi-byte sequences. Must NOT be promoted to + * STRUCTURAL UTF-8 on the strength of tolerance alone — that's the + * false-positive TIKA-4752-era filename detection depends on avoiding. + */ + @Test + public void shortProbeWithOneStrayByteIsNotPromoted() throws IOException { + // ~20 bytes: one legacy high byte + a couple of genuine multi-byte + // UTF-8 chars — the shape of a real (short) zip entry name, not a + // full document. + ByteArrayOutputStream bo = new ByteArrayOutputStream(); + bo.write(0xA9); // stray legacy byte, invalid as a UTF-8 lead + bo.writeBytes("café-Köln.txt".getBytes(StandardCharsets.UTF_8)); + byte[] probe = bo.toByteArray(); + + List<EncodingResult> results = newDetector().detect(probe); + boolean hasStructuralUtf8 = results.stream().anyMatch(r -> + "UTF-8".equals(r.getCharset().name()) + && r.getResultType() == EncodingResult.ResultType.STRUCTURAL); + assertFalse(hasStructuralUtf8, + "A short probe shaped like a zip entry name must not be promoted " + + "to STRUCTURAL UTF-8 on a single tolerated error alone; " + + "results were: " + results); + } + + /** + * Real embedded-file-name regression from {@code attachment_name_diffs.xlsx} + * (commoncrawl3/5D/5DXWH7R4A5Q6VAWBAMBSUZM5PNEVAE63): a GBK zip entry name + * ({@code 说明.txt}) must stay GB18030, not get pulled toward STRUCTURAL + * UTF-8 by tolerance — the same false-positive risk as the Latin case, + * CJK-flavored. + */ + @Test + public void chineseGbkFilenameIsNotPromotedToUtf8() { + byte[] probe = "说明.txt".getBytes(Charset.forName("GBK")); + List<EncodingResult> results = newDetector().detect(probe); + boolean hasStructuralUtf8 = results.stream().anyMatch(r -> + "UTF-8".equals(r.getCharset().name()) + && r.getResultType() == EncodingResult.ResultType.STRUCTURAL); + assertFalse(hasStructuralUtf8, + "A short GBK filename must not be promoted to STRUCTURAL UTF-8 " + + "on a single tolerated error alone; results were: " + results); + assertTrue(results.stream().anyMatch(r -> r.getCharset().name().startsWith("GB")), + "Expected a GB18030/GBK candidate; results were: " + results); + } + + /** + * Real embedded-file-name regression from {@code attachment_name_diffs.xlsx} + * (bug_trackers/MOZILLA/240463-316268/MOZILLA-296795-4.zip): a windows-1252 + * zip entry name ({@code Sauté.txt}) must stay legacy SBCS, not get promoted + * to STRUCTURAL UTF-8 by tolerance. + */ + @Test + public void sauteFilenameIsNotPromotedToUtf8() { + byte[] probe = "Sauté.txt".getBytes(Charset.forName("windows-1252")); + List<EncodingResult> results = newDetector().detect(probe); + boolean hasStructuralUtf8 = results.stream().anyMatch(r -> + "UTF-8".equals(r.getCharset().name()) + && r.getResultType() == EncodingResult.ResultType.STRUCTURAL); + assertFalse(hasStructuralUtf8, + "A short windows-1252 filename must not be promoted to STRUCTURAL " + + "UTF-8 on a single tolerated error alone; results were: " + results); + } + + /** HTML wrapper + {@code repeatCount} copies of a real Bengali sentence, + * with a single raw {@code 0xA9} (not a UTF-8 encoded {@code ©}) planted + * in a meta tag before the real content — matches the real-world + * failure exactly (declared windows-1252, genuinely UTF-8 body). */ + private static byte[] buildProbe(int repeatCount) throws IOException { + StringBuilder body = new StringBuilder(); + for (int i = 0; i < repeatCount; i++) { + body.append(BENGALI_SENTENCE); + } + ByteArrayOutputStream bo = new ByteArrayOutputStream(); + bo.writeBytes(("<html><head><meta http-equiv=\"Content-Type\" " + + "content=\"text/html; charset=windows-1252\">") + .getBytes(StandardCharsets.US_ASCII)); + bo.writeBytes("<meta name=\"copyright\" content=\"".getBytes(StandardCharsets.US_ASCII)); + bo.write(0xA9); // stray legacy byte, invalid as a UTF-8 lead + bo.writeBytes(" 2013\"></head><body><title>".getBytes(StandardCharsets.US_ASCII)); + bo.writeBytes(body.toString().getBytes(StandardCharsets.UTF_8)); + bo.writeBytes("</title></body></html>".getBytes(StandardCharsets.US_ASCII)); + return bo.toByteArray(); + } +}
