This is an automated email from the ASF dual-hosted git repository.
tballison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git
The following commit(s) were added to refs/heads/main by this push:
new 7bba21fae1 TIKA-5052: UTF-32 and UTF-16 structural checks read the
whole probe (#3632)
7bba21fae1 is described below
commit 7bba21fae1c70259f3ef0da74d5199356d04a5c9
Author: Tim Allison <[email protected]>
AuthorDate: Tue Nov 17 11:21:21 2026 -0500
TIKA-5052: UTF-32 and UTF-16 structural checks read the whole probe (#3632)
utf32Invalid and the surrogate scan sampled 512 bytes, so ar symbol tables,
zero-led
disk images and NUL-led logs passed as UTF-32BE and decoded to
replacement characters;
an all-NUL prefix no longer counts as UTF-32.
Co-Authored-By: Claude Fable 5.1 <[email protected]>
---
CHANGES.txt | 6 ++
.../tika/ml/chardetect/WideUnicodeDetector.java | 27 +++++---
.../ml/chardetect/WideUnicodeDetectionTest.java | 78 +++++++++++++++++++++-
3 files changed, 100 insertions(+), 11 deletions(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index 55287541b6..a8f0c3d840 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -519,6 +519,12 @@ Release 4.3.0 - ???
UTF-32 code points: binary with a few NULs in its header (a shader file
of ASCII 0/1
after an 80 37 00 00 header) was reported as UTF-32BE (TIKA-5052).
+ * The structural UTF-32 check and the UTF-16 surrogate check read the whole
detection
+ sample, not its first 512 bytes, and an all-NUL sample is not UTF-32: ar
symbol
+ tables, zero-led disk images and NUL-led logs were reported as UTF-32BE
and decoded
+ to replacement characters, and binary with a text-like head kept a UTF-16
label
+ (TIKA-5052).
+
* Charset detection no longer reads ASCII text with a few control
characters (logs with
escape or form-feed bytes, IRC colour codes, record separators, banners)
as BOM-less
UTF-16: when the model returns UTF-16 or UTF-32 as its only candidate,
the junk filter
diff --git
a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/WideUnicodeDetector.java
b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/WideUnicodeDetector.java
index cb948f30c4..fbd273d605 100644
---
a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/WideUnicodeDetector.java
+++
b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/WideUnicodeDetector.java
@@ -96,10 +96,11 @@ final class WideUnicodeDetector {
int[] utf32 = utf32Invalid(bytes, offset, length);
boolean invalidBe = utf32[0] > 0;
boolean invalidLe = utf32[1] > 0;
- if (!invalidBe && utf32[2] >= UTF32_MIN_VALID) {
+ // all-NUL groups are code points under both orders but evidence of
nothing (NUL-led logs)
+ if (!invalidBe && utf32[3] >= UTF32_MIN_VALID) {
return new Result(Charset.forName("UTF-32BE"), false, false,
false, invalidLe);
}
- if (!invalidLe && utf32[2] >= UTF32_MIN_VALID) {
+ if (!invalidLe && utf32[3] >= UTF32_MIN_VALID) {
return new Result(Charset.forName("UTF-32LE"), false, false,
invalidBe, false);
}
Result utf16 = tryUtf16(bytes, offset, length);
@@ -124,7 +125,7 @@ final class WideUnicodeDetector {
// -----------------------------------------------------------------------
/**
- * Minimum number of valid codepoints required for detection. At 8 bytes
+ * Minimum number of valid non-NUL codepoints required for detection. At 8
bytes
* we get exactly 2 groups; the probability that 2 random 4-byte values
* both fall in 0x000000–0x10FFFF (excluding surrogates) is ~1.6e-11,
* so 2 is safe against false positives on real-world byte data.
@@ -141,12 +142,17 @@ final class WideUnicodeDetector {
* immediately disqualify it. No null-byte density threshold is needed,
* which means non-BMP content (emoji, historic scripts) works
perfectly.</p>
*/
- /** {invalid BE groups, invalid LE groups, groups} over the first 512
bytes. */
+ /**
+ * {invalid BE groups, invalid LE groups, groups, non-NUL groups} over the
whole probe. A
+ * 512-byte sample let ar symbol tables (big-endian small ints) and
zero-led disk images through
+ * as UTF-32BE.
+ */
private static int[] utf32Invalid(byte[] bytes, int offset, int length) {
- int sampleLen = (Math.min(length, 512) / 4) * 4;
+ int sampleLen = (length / 4) * 4;
int groups = sampleLen / 4;
int invalidBe = 0;
int invalidLe = 0;
+ int nonNul = 0;
for (int g = 0; g < groups; g++) {
int base = offset + g * 4;
@@ -154,6 +160,9 @@ final class WideUnicodeDetector {
int b1 = bytes[base + 1] & 0xFF;
int b2 = bytes[base + 2] & 0xFF;
int b3 = bytes[base + 3] & 0xFF;
+ if ((b0 | b1 | b2 | b3) != 0) {
+ nonNul++;
+ }
int cpBe = (b0 << 24) | (b1 << 16) | (b2 << 8) | b3;
if (!(cpBe >= 0 && cpBe <= 0x10FFFF && (cpBe < 0xD800 || cpBe >
0xDFFF))) {
@@ -165,7 +174,7 @@ final class WideUnicodeDetector {
invalidLe++;
}
}
- return new int[] {invalidBe, invalidLe, groups};
+ return new int[] {invalidBe, invalidLe, groups, nonNul};
}
// -----------------------------------------------------------------------
@@ -180,7 +189,7 @@ final class WideUnicodeDetector {
* the main statistical model on probes that cannot be valid UTF-16.
*/
private static Result tryUtf16(byte[] bytes, int offset, int length) {
- int sampleLen = (Math.min(length, 512) / 2) * 2;
+ int sampleLen = (length / 2) * 2;
if (sampleLen < 8) {
return Result.EMPTY;
}
@@ -225,8 +234,8 @@ final class WideUnicodeDetector {
}
}
}
- // A high surrogate ending the sample is checked against the next
unit; at the
- // probe end the pair was cut, which is no evidence either way.
+ // A high surrogate ending the scan is checked against an odd trailing
byte's unit only when one
+ // exists; at the probe end the pair was cut, which is no evidence
either way.
int next = offset + sampleLen;
if (next + 1 < offset + length) {
int even = bytes[next] & 0xFF;
diff --git
a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/test/java/org/apache/tika/ml/chardetect/WideUnicodeDetectionTest.java
b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/test/java/org/apache/tika/ml/chardetect/WideUnicodeDetectionTest.java
index 28eb750f15..d2029cde91 100644
---
a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/test/java/org/apache/tika/ml/chardetect/WideUnicodeDetectionTest.java
+++
b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/test/java/org/apache/tika/ml/chardetect/WideUnicodeDetectionTest.java
@@ -18,6 +18,7 @@ package org.apache.tika.ml.chardetect;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.nio.charset.Charset;
@@ -210,6 +211,79 @@ public class WideUnicodeDetectionTest {
"Grüße, мир, 世界".getBytes(Charset.forName(charset)));
assertEquals(charset, wide.charset.name());
assertFalse("UTF-32BE".equals(charset) ? wide.invalidUtf32Be :
wide.invalidUtf32Le);
+ // longer than any sample window, with a group cut at the probe end
+ byte[] longer = "Grüße, мир, 世界.
".repeat(60).getBytes(Charset.forName(charset));
+ assertEquals(charset, top(Arrays.copyOf(longer, longer.length - 2)));
+ }
+
+ /**
+ * GNU ar symbol table ({@code /} member of a {@code gcc -c; ar rcs}
archive): big-endian count and
+ * offsets, then NUL-terminated names. Every header int is a code point,
so a check over the first
+ * bytes only called the 1.2M corpus's .a/.deb members UTF-32BE and
decoded the names to U+FFFD.
+ */
+ @Test
+ public void arSymbolTableIsNotUtf32() {
+ int[] header = {7, 0xcc, 0xcc, 0xcc, 0xcc, 0x6e8, 0x6e8, 0x6e8};
+ String names =
"alpha_function\0beta_function\0gamma_function\0delta_variable\0"
+ + "epsilon_function\0zeta_function\0eta_variable\0\0";
+ byte[] bytes = new byte[header.length * 4 + names.length()];
+ for (int i = 0; i < header.length; i++) {
+ bytes[i * 4 + 2] = (byte) (header[i] >>> 8);
+ bytes[i * 4 + 3] = (byte) header[i];
+ }
+ System.arraycopy(names.getBytes(StandardCharsets.US_ASCII), 0, bytes,
header.length * 4,
+ names.length());
+ WideUnicodeDetector.Result wide = WideUnicodeDetector.analyze(bytes);
+ assertTrue(wide.invalidUtf32Be);
+ assertTrue(wide.invalidUtf32Le);
+ assertFalse(top(bytes).startsWith("UTF-32"));
+ // 128 header ints fill the old 512-byte sample on their own
+ byte[] big = new byte[128 * 4 + names.length()];
+ for (int i = 0; i < 128; i++) {
+ big[i * 4 + 2] = (byte) (i >>> 6);
+ big[i * 4 + 3] = (byte) (i * 24);
+ }
+ System.arraycopy(names.getBytes(StandardCharsets.US_ASCII), 0, big,
512, names.length());
+ assertTrue(WideUnicodeDetector.analyze(big).invalidUtf32Be);
+ assertFalse(top(big).startsWith("UTF-32"));
+ }
+
+ /** A zero-led partition image: 512 NUL bytes, then data no 4-byte group
of which is a code point. */
+ @Test
+ public void zeroLedImageIsNotUtf32() {
+ byte[] bytes = new byte[8192];
+ Random r = new Random(5062L);
+ for (int i = 512; i < bytes.length; i++) {
+ bytes[i] = (byte) (0x80 | r.nextInt(128));
+ }
+ WideUnicodeDetector.Result wide = WideUnicodeDetector.analyze(bytes);
+ assertTrue(wide.invalidUtf32Be);
+ assertTrue(wide.invalidUtf32Le);
+ String top = top(bytes);
+ assertFalse(top.startsWith("UTF-32") || top.startsWith("UTF-16"), top);
+ }
+
+ /** 17 KB of NUL then an ASCII log (a 1.2M MOZILLA .zip member): the
all-NUL prefix is not UTF-32. */
+ @Test
+ public void nulLedLogIsNotUtf32() {
+ byte[] nuls = new byte[17412];
+ assertNull(WideUnicodeDetector.analyze(nuls).charset);
+ assertFalse(top(nuls).startsWith("UTF-32"), top(nuls));
+ byte[] log = "1074222328[40404160]: http request
[\n".repeat(400).getBytes(StandardCharsets.US_ASCII);
+ byte[] bytes = Arrays.copyOf(nuls, nuls.length + log.length);
+ System.arraycopy(log, 0, bytes, nuls.length, log.length);
+ assertTrue(WideUnicodeDetector.analyze(bytes).invalidUtf32Be);
+ assertFalse(top(bytes).startsWith("UTF-32"), top(bytes));
+ }
+
+ /** A lone surrogate past byte 512 (binary with a text-like head) still
vetoes UTF-16. */
+ @Test
+ public void loneSurrogateDeepInTheProbeIsInvalid() {
+ String prefix = "Full thread dump Java HotSpot(TM) 64-Bit Server VM
".repeat(12);
+ assertTrue(prefix.length() > 512);
+ assertTrue(WideUnicodeDetector.analyze(le(prefix + "x",
prefix.length(), 0xD83D)).invalidUtf16Le);
+ assertFalse(WideUnicodeDetector.analyze(le(prefix + "x",
prefix.length(), 0xD83D, 0xDE00))
+ .invalidUtf16Le);
}
/** UTF-16LE of {@code text} with raw code {@code units} inserted at char
index {@code at}. */
@@ -235,9 +309,9 @@ public class WideUnicodeDetectionTest {
assertTrue(WideUnicodeDetector.analyze(le("abcdefghx", 8,
0xD83D)).invalidUtf16Le);
}
- /** The surrogate check samples 512 bytes; a pair cut there is checked
against the next unit. */
+ /** A pair straddling byte 512 is one pair; a lone high surrogate there is
still invalid. */
@Test
- public void surrogateCutAtSampleEndLooksAhead() {
+ public void surrogateStraddlingByte512() {
String prefix = "x".repeat(255);
assertFalse(WideUnicodeDetector.analyze(le(prefix + " tail", 255,
0xD83D, 0xDE00))
.invalidUtf16Le);