This is an automated email from the ASF dual-hosted git repository.
tballison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git
The following commit(s) were added to refs/heads/main by this push:
new 8cd84d687a TIKA-5027: report the windows-125x sibling when C1
punctuation proves it (#3554)
8cd84d687a is described below
commit 8cd84d687a278cba7abafd35c2c7eacd8ff4bd69
Author: Tim Allison <[email protected]>
AuthorDate: Fri Nov 6 16:11:06 2026 -0500
TIKA-5027: report the windows-125x sibling when C1 punctuation proves it
(#3554)
---
CHANGES.txt | 11 ++--
.../ROOT/pages/advanced/junk-detection.adoc | 6 ++
.../ml/junkdetect/JunkFilterEncodingDetector.java | 71 +++++++++++++++++++++-
.../junkdetect/JunkFilterEncodingDetectorTest.java | 56 +++++++++++++++++
4 files changed, 137 insertions(+), 7 deletions(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index b5804055c7..cfb61d1e4a 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -345,10 +345,13 @@ Release 4.3.0 - ???
is honored only when the bytes after it decode in that charset, and a
rejected BOM
is dropped rather than left in the text as junk characters (TIKA-5019).
- * Charset detection keeps the statistical detector's pick when an HTML or
HTTP charset
- declaration names the same charset, rather than letting text-quality
scoring override
- two agreeing signals; and UTF-8 with sparse corruption (up to one bad
sequence per
- ten valid ones, existing U+FFFD not counted) is read as UTF-8 (TIKA-5019).
+ * Charset detection reads UTF-8 with sparse corruption (up to one bad
sequence per
+ ten valid ones, existing U+FFFD not counted) as UTF-8 (TIKA-5019).
+
+ * Charset detection reports the windows-125x sibling of an ISO-8859 charset
(e.g.
+ windows-1250 for ISO-8859-2) when the page's 0x80-0x9F bytes are
punctuation that
+ only the windows charset prints (… “ ” €) and no letters decode
differently, even
+ when the page declares the ISO charset (TIKA-5027).
* Charset detection uses a hashed byte n-gram model in place of the
byte-bigram Naive
Bayes model, with UTF-16 and UTF-32 detected by the same model instead of
a separate
diff --git a/docs/modules/ROOT/pages/advanced/junk-detection.adoc
b/docs/modules/ROOT/pages/advanced/junk-detection.adoc
index fd5d2473e3..fafd8e4316 100644
--- a/docs/modules/ROOT/pages/advanced/junk-detection.adoc
+++ b/docs/modules/ROOT/pages/advanced/junk-detection.adoc
@@ -112,6 +112,12 @@ amount of evidence: two decodes that differ in one
character can still have a la
structural charset when the margin is below the model's `getDeferThreshold()`
(chosen per model
by the trainer).
+Whatever decides, an ISO-8859 pick becomes its windows-125x sibling
(ISO-8859-2 -> windows-1250,
+ISO-8859-1 or -15 -> windows-1252, and so on) when the two decodes differ only
at 0x80-0x9F bytes
+that the windows charset prints: those bytes are windows punctuation, never
ISO-8859 text. A
+0x80-0x9F byte after another high byte is not counted, since it is more likely
a UTF-8
+continuation byte.
+
== tika-eval fields
[cols="1,3"]
diff --git
a/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/JunkFilterEncodingDetector.java
b/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/JunkFilterEncodingDetector.java
index 04771f7045..c6f6599101 100644
---
a/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/JunkFilterEncodingDetector.java
+++
b/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/JunkFilterEncodingDetector.java
@@ -165,8 +165,7 @@ public class JunkFilterEncodingDetector implements
MetaEncodingDetector {
context.setArbitrationInfo("junk-filter-prefer-declarative");
LOG.trace("junk-filter -> {} (declarative with equivalent
decode)",
prepared.getRuleCharset().name());
- return List.of(new EncodingResult(prepared.getRuleCharset(),
-
context.getTopConfidenceFor(prepared.getRuleCharset())));
+ return result(prepared.getRuleCharset(), bytes, context);
case IDENTICAL_DECODINGS: {
// Text cannot tell them apart. A validated BOM is then the
only discriminating
// evidence (it also tells the reader to drop the BOM bytes);
else the legacy-page
@@ -196,7 +195,73 @@ public class JunkFilterEncodingDetector implements
MetaEncodingDetector {
context.setArbitrationInfo(deferred ? "junk-filter-hashed-deferred" :
"junk-filter-hashed");
LOG.trace("junk-filter -> {} (champion {}, margin {}, anchor {})",
pick.name(),
decision.getChampion().name(), decision.getLogit(),
decision.getAnchor());
- return List.of(new EncodingResult(pick,
context.getTopConfidenceFor(pick)));
+ return result(pick, bytes, context);
+ }
+
+ /** {@code pick}, or its windows-125x sibling when the probe's C1 bytes
prove it (see {@link #windowsSibling}). */
+ private static List<EncodingResult> result(Charset pick, byte[] bytes,
EncodingDetectorContext context) {
+ float confidence = context.getTopConfidenceFor(pick);
+ Charset sibling = windowsSibling(pick, bytes);
+ if (sibling != null) {
+ context.setArbitrationInfo(context.getArbitrationInfo() +
"-c1-sibling");
+ return List.of(new EncodingResult(sibling, confidence));
+ }
+ return List.of(new EncodingResult(pick, confidence));
+ }
+
+ private static final Map<String, String> WINDOWS_SIBLING = Map.of(
+ "ISO-8859-1", "windows-1252", "ISO-8859-15", "windows-1252",
"ISO-8859-2", "windows-1250",
+ "ISO-8859-7", "windows-1253", "ISO-8859-8", "windows-1255",
"ISO-8859-9", "windows-1254",
+ "ISO-8859-13", "windows-1257");
+
+ /**
+ * The windows-125x sibling of an ISO-8859 {@code pick}, or {@code null}.
It applies when the two
+ * cleaned decodes differ only where the ISO side has a C1 control
(0x80-0x9F) that the sibling
+ * prints: such a byte is punctuation typed in windows-125x (… “ ” €),
never ISO-8859 text, and
+ * a declaration does not outweigh it (gold rule). A letter difference
means the text decides,
+ * so the pick stands. A C1 byte after a high byte is likely a UTF-8
continuation (Ã + 0x9F for
+ * ß), which is mojibake, not evidence.
+ */
+ static Charset windowsSibling(Charset pick, byte[] bytes) {
+ String name = WINDOWS_SIBLING.get(pick.name());
+ if (name == null || !hasC1Evidence(bytes)) {
+ return null;
+ }
+ Charset windows;
+ try {
+ windows = Charset.forName(name);
+ } catch (IllegalArgumentException e) {
+ return null;
+ }
+ String iso = HtmlContentCleaner.clean(new String(bytes, pick));
+ String win = HtmlContentCleaner.clean(new String(bytes, windows));
+ if (iso.length() != win.length()) {
+ return null;
+ }
+ boolean differs = false;
+ for (int i = 0; i < iso.length(); i++) {
+ char a = iso.charAt(i);
+ char b = win.charAt(i);
+ if (a == b) {
+ continue;
+ }
+ if (a < 0x80 || a > 0x9F || b == '�' || Character.isISOControl(b))
{
+ return null;
+ }
+ differs = true;
+ }
+ return differs ? windows : null;
+ }
+
+ /** A 0x80-0x9F byte that follows an ASCII byte (or starts the probe). */
+ private static boolean hasC1Evidence(byte[] bytes) {
+ for (int i = 0; i < bytes.length; i++) {
+ int v = bytes[i] & 0xFF;
+ if (v >= 0x80 && v <= 0x9F && (i == 0 || (bytes[i - 1] & 0xFF) <
0x80)) {
+ return true;
+ }
+ }
+ return false;
}
private static boolean isCjkCharset(String name) {
diff --git
a/tika-ml/tika-ml-junkdetect/src/test/java/org/apache/tika/ml/junkdetect/JunkFilterEncodingDetectorTest.java
b/tika-ml/tika-ml-junkdetect/src/test/java/org/apache/tika/ml/junkdetect/JunkFilterEncodingDetectorTest.java
index 1686a9f8fb..18902ad7f0 100644
---
a/tika-ml/tika-ml-junkdetect/src/test/java/org/apache/tika/ml/junkdetect/JunkFilterEncodingDetectorTest.java
+++
b/tika-ml/tika-ml-junkdetect/src/test/java/org/apache/tika/ml/junkdetect/JunkFilterEncodingDetectorTest.java
@@ -17,6 +17,7 @@
package org.apache.tika.ml.junkdetect;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.nio.charset.Charset;
@@ -348,4 +349,59 @@ public class JunkFilterEncodingDetectorTest {
assertEquals(utf8, new
JunkFilterEncodingDetector(MODEL).detect(tis, new Metadata(),
pc).get(0).getCharset());
}
}
+
+ private static final Charset ISO_8859_2 = Charset.forName("ISO-8859-2");
+ private static final Charset WIN_1250 = Charset.forName("windows-1250");
+
+ /** Polish whose letters (ł ę ó ż) read the same in ISO-8859-2 and
windows-1250; only the ellipsis differs. */
+ private static byte[] polishWithEllipsis() {
+ return "<p>Z\u0142oty \u017c\u00f3\u0142w pe\u0142z\u0142 bardzo
wolno\u2026 a\u017c do rzeki.</p>".getBytes(WIN_1250);
+ }
+
+ @Test
+ public void c1PunctuationUpgradesIsoToWindowsSibling() {
+ assertEquals(WIN_1250,
JunkFilterEncodingDetector.windowsSibling(ISO_8859_2, polishWithEllipsis()));
+ assertEquals(Charset.forName("windows-1252"),
JunkFilterEncodingDetector.windowsSibling(
+ Charset.forName("ISO-8859-15"), "Il a dit
\u201coui\u201d.".getBytes(Charset.forName("windows-1252"))));
+ }
+
+ /** A letter that differs between the siblings means the text decides, so
a stray C1 byte does not. */
+ @Test
+ public void c1ByteDoesNotOverrideDifferingLetters() {
+ byte[] bytes = "Zielona \u0142\u0105ka
\u015bwieci\u2026".getBytes(ISO_8859_2);
+ bytes[bytes.length - 1] = (byte) 0x85;
+ assertNull(JunkFilterEncodingDetector.windowsSibling(ISO_8859_2,
bytes));
+ }
+
+ /** UTF-8 \u00df (C3 9F) read as Latin is mojibake; its 0x9F is not
windows-1252 evidence. */
+ @Test
+ public void utf8ContinuationByteIsNotC1Evidence() {
+
assertNull(JunkFilterEncodingDetector.windowsSibling(Charset.forName("ISO-8859-15"),
+ "Schlie\u00dfen".getBytes(StandardCharsets.UTF_8)));
+ }
+
+ /** 0x8D is undefined in windows-1253, so the sibling would not read
better. */
+ @Test
+ public void c1ByteUndefinedInSiblingIsNotEvidence() {
+
assertNull(JunkFilterEncodingDetector.windowsSibling(Charset.forName("ISO-8859-7"),
+ new byte[] {'a', ' ', (byte) 0x8D, ' ', 'b'}));
+ }
+
+ @Test
+ public void onlyIsoPicksHaveASibling() {
+ assertNull(JunkFilterEncodingDetector.windowsSibling(WIN_1250,
polishWithEllipsis()));
+ assertNull(JunkFilterEncodingDetector.windowsSibling(ISO_8859_2,
"plain ascii".getBytes(StandardCharsets.US_ASCII)));
+ }
+
+ /** A declared ISO-8859-2 page whose only high-byte difference is a
windows-1250 ellipsis is windows-1250. */
+ @Test
+ public void declaredIsoWithWindowsEllipsisDetectsAsWindows() throws
Exception {
+ ParseContext pc = contextWith(
+ new EncodingResult(ISO_8859_2, 1.0f, "ISO-8859-2",
EncodingResult.ResultType.DECLARATIVE),
+ new EncodingResult(ISO_8859_2, 0.9f, "ISO-8859-2",
EncodingResult.ResultType.STATISTICAL),
+ new EncodingResult(WIN_1250, 0.8f, "windows-1250",
EncodingResult.ResultType.STATISTICAL));
+ try (TikaInputStream tis = TikaInputStream.get(polishWithEllipsis())) {
+ assertEquals(WIN_1250, new
JunkFilterEncodingDetector(MODEL).detect(tis, new Metadata(),
pc).get(0).getCharset());
+ }
+ }
}
