This is an automated email from the ASF dual-hosted git repository.
tballison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git
The following commit(s) were added to refs/heads/main by this push:
new 88f5737122 Decode legacy single-byte charsets as their windows
supersets (#3593)
88f5737122 is described below
commit 88f5737122222e7c8def2e5808f6fcbbabaa3b39
Author: Tim Allison <[email protected]>
AuthorDate: Mon Nov 9 15:38:55 2026 -0500
Decode legacy single-byte charsets as their windows supersets (#3593)
---
CHANGES.txt | 5 ++++
.../org/apache/tika/detect/CharsetSupersets.java | 9 +++++++
.../apache/tika/detect/CharsetSupersetsTest.java | 29 ++++++++++++++++++++++
3 files changed, 43 insertions(+)
diff --git a/CHANGES.txt b/CHANGES.txt
index b92cbac3a0..b8f5f3a343 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -104,6 +104,11 @@ Release 4.3.0 - ???
EXTRACTION AND FORMATS
+ * ISO-2022-JP is decoded as x-windows-50221, so NEC/IBM extension
characters (circled
+ numbers, Roman numerals) no longer become U+FFFD. ISO-8859-1, ISO-8859-9,
TIS-620 and
+ ISO-8859-11 decode as windows-1252, windows-1254 and windows-874, as
browsers do, when a
+ detector other than the default chain reports them (TIKA-5045).
+
* Binary files are no longer detected as text because of their name or a
text header.
When content has no magic and at least 5% control bytes, and does not
decode as
UTF-16 or UTF-32, a text type from the file name or Content-Type no
longer wins: Terminal Reality
diff --git
a/tika-core/src/main/java/org/apache/tika/detect/CharsetSupersets.java
b/tika-core/src/main/java/org/apache/tika/detect/CharsetSupersets.java
index c9935feb3a..f862e37595 100644
--- a/tika-core/src/main/java/org/apache/tika/detect/CharsetSupersets.java
+++ b/tika-core/src/main/java/org/apache/tika/detect/CharsetSupersets.java
@@ -43,6 +43,10 @@ import java.util.Map;
* <li>GBK → GB18030 (GB18030 is a strict superset; enables 4-byte extension
sequences)</li>
* <li>Shift_JIS → windows-31j (MS932 is a strict superset with NEC/IBM
extensions)</li>
* <li>EUC-JP → x-eucJP-Open (EUC packing of the NEC/IBM vendor
extensions)</li>
+ * <li>ISO-2022-JP → x-windows-50221 (CP50221 adds the NEC/IBM
extensions)</li>
+ * <li>ISO-8859-1 → windows-1252, ISO-8859-9 → windows-1254, TIS-620 and
ISO-8859-11 →
+ * windows-874 (what browsers do: 0x80-0x9F are punctuation, not C1
controls; the few
+ * bytes the windows code page leaves undefined decode as U+FFFD)</li>
* </ul>
*/
public final class CharsetSupersets {
@@ -61,6 +65,11 @@ public final class CharsetSupersets {
m.put("GBK", "GB18030");
m.put("Shift_JIS", "windows-31j");
m.put("EUC-JP", "x-eucJP-Open");
+ m.put("ISO-2022-JP", "x-windows-50221");
+ m.put("ISO-8859-1", "windows-1252");
+ m.put("ISO-8859-9", "windows-1254");
+ m.put("TIS-620", "x-windows-874");
+ m.put("x-iso-8859-11", "x-windows-874");
SUPERSET_MAP = Collections.unmodifiableMap(m);
}
diff --git
a/tika-core/src/test/java/org/apache/tika/detect/CharsetSupersetsTest.java
b/tika-core/src/test/java/org/apache/tika/detect/CharsetSupersetsTest.java
index 738fad6196..d8cfd6d9e5 100644
--- a/tika-core/src/test/java/org/apache/tika/detect/CharsetSupersetsTest.java
+++ b/tika-core/src/test/java/org/apache/tika/detect/CharsetSupersetsTest.java
@@ -39,6 +39,7 @@ public class CharsetSupersetsTest {
assertEquals("GB18030", name("GBK"));
assertEquals("windows-31j", name("Shift_JIS"));
assertEquals("x-eucJP-Open", name("EUC-JP"));
+ assertEquals("x-windows-50221", name("ISO-2022-JP"));
}
@Test
@@ -63,5 +64,33 @@ public class CharsetSupersetsTest {
byte[] eucjp = {(byte) 0xAD, (byte) 0xA1};
assertEquals('\uFFFD', new String(eucjp,
Charset.forName("EUC-JP")).charAt(0));
assertEquals("\u2460", new String(eucjp,
Charset.forName(name("EUC-JP"))));
+
+ byte[] jis =
"\u65e5\u2460".getBytes(Charset.forName("x-windows-50221"));
+ assertEquals('\uFFFD', new String(jis,
Charset.forName("ISO-2022-JP")).charAt(1));
+ assertEquals("\u65e5\u2460", new String(jis,
Charset.forName(name("ISO-2022-JP"))));
+ }
+
+ @Test
+ public void singleByteDecodeAsBrowserCodePage() {
+ assertEquals("windows-1252", name("ISO-8859-1"));
+ assertEquals("windows-1254", name("ISO-8859-9"));
+ assertEquals("x-windows-874", name("TIS-620"));
+ assertEquals("x-windows-874", name("ISO-8859-11"));
+ // 0x92 is a C1 control under ISO-8859-1, a right single quote on real
pages
+ assertEquals("\u2019", new String(new byte[]{(byte) 0x92},
Charset.forName(name("ISO-8859-1"))));
+ for (String narrow : new String[]{"ISO-8859-1", "ISO-8859-9",
"TIS-620", "ISO-8859-11"}) {
+ Charset n = Charset.forName(narrow);
+ Charset w = Charset.forName(name(narrow));
+ for (int b = 0; b < 256; b++) {
+ if (b >= 0x80 && b < 0xA0) {
+ continue;
+ }
+ byte[] one = {(byte) b};
+ String expected = new String(one, n);
+ if (expected.charAt(0) != '\uFFFD') {
+ assertEquals(expected, new String(one, w), narrow + " byte
" + b);
+ }
+ }
+ }
}
}
