This is an automated email from the ASF dual-hosted git repository.
tballison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git
The following commit(s) were added to refs/heads/main by this push:
new fee8015695 TIKA-5062: classify magic-less content as text or binary
with TextBinaryClassifier (#3598)
fee8015695 is described below
commit fee8015695e795424a17383098695db3bd74fe67
Author: Tim Allison <[email protected]>
AuthorDate: Thu Nov 12 14:02:33 2026 -0500
TIKA-5062: classify magic-less content as text or binary with
TextBinaryClassifier (#3598)
---
CHANGES.txt | 10 +
.../apache/tika/detect/TextBinaryClassifier.java | 597 +++++++++++++++++++++
.../main/java/org/apache/tika/mime/MimeTypes.java | 10 +-
.../tika/detect/TextBinaryClassifierTest.java | 159 ++++++
.../org/apache/tika/mime/MimeDetectionTest.java | 49 +-
5 files changed, 819 insertions(+), 6 deletions(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index fa3dd54880..4db7145361 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -130,6 +130,16 @@ Release 4.3.0 - ???
A UTF-8 BOM followed by binary magic (JPEG, PNG, GIF, ID3, zip, SQLite)
is detected by
that magic instead of as text (TIKA-5029).
+ * Content with no magic is judged text or binary by TextBinaryClassifier, a
small
+ model over byte statistics of the 64 KB prefix, instead of TextDetector's
control-byte
+ check, which only accepted mostly-ASCII or valid UTF-8. Extensionless
text in legacy
+ encodings (windows-1251, GB18030, Shift_JIS, EUC-JP, EUC-KR, KOI8-R and
others), logs
+ with a few control bytes, zero-filled holes or backspace progress bars
are now
+ text/plain; 3.x returned application/octet-stream for these, and 4.0-4.2
rescued
+ some only through a 512-byte re-sniff. Printable ASCII is always text;
inputs under
+ 16 bytes keep the old check. A text name or Content-Type still loses to
content with
+ 5% or more control bytes that the classifier calls binary (TIKA-5062).
+
* New or fixed magic: Emacs bytecode from every Emacs version (only version
19 matched
before, and *.elc is no longer also an octet-stream glob), Lua bytecode
(application/x-lua-bytecode), Qt .qm translations
(application/x-qt-translation, were
diff --git
a/tika-core/src/main/java/org/apache/tika/detect/TextBinaryClassifier.java
b/tika-core/src/main/java/org/apache/tika/detect/TextBinaryClassifier.java
new file mode 100644
index 0000000000..bbd75bb31b
--- /dev/null
+++ b/tika-core/src/main/java/org/apache/tika/detect/TextBinaryClassifier.java
@@ -0,0 +1,597 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.detect;
+
+/**
+ * Decides whether magic-less bytes are text or binary. A binned logistic
model (a scorecard)
+ * over byte statistics of the prefix: control-byte classes, NUL structure,
printable runs, line
+ * shape, byte-class periodicity, entropy, UTF-8 validity and UTF-16
plausibility. It needs no
+ * charset detector, so legacy 8-bit and CJK text count as text, and so do
text with a few
+ * controls, zero-filled holes and backspace progress bars.
+ * <p>
+ * Trained outside this repository on typed text in many encodings (with
control, NUL-hole and
+ * separator augmentations) against mid-file windows of typed binaries,
magic-less binaries and
+ * synthesized fills, samples, image rows, tables and object code. The arrays
are generated; do
+ * not edit them by hand. Inputs shorter than {@link #MIN_LENGTH} bytes carry
too little evidence
+ * and are not classified.
+ *
+ * @since Apache Tika 4.3.0
+ */
+public final class TextBinaryClassifier {
+
+ /** Shortest input the model decides. */
+ public static final int MIN_LENGTH = 16;
+
+ // generated by the training scripts (35 features, 8 quantile bins); do
not edit by hand
+ static final String[] FEATURES = {
+ "logn", "nul", "bs", "esc", "c0", "del", "c1", "ff", "hi",
"ctlNoNul", "nulRuns", "nulRunMax",
+ "nulRunMean", "nulMod2", "nulMod4", "nulAfterStr", "nulStr",
"cov8a", "nlKB", "lineMean", "lineStd",
+ "lineMax", "bsCtx", "ac1", "ac2", "ac4", "ac8", "ac16", "ent",
"distinct", "topB", "u8ok",
+ "u8badKB", "u16", "u16asc"
+ };
+ private static final double[][] EDGES = {
+ { // logn
+ 5.754887502163469, 7.149747119504682, 8.396604781181859,
9.505811553919594, 10.689997971419444,
+ 11.870846298570346, 13.406005627992485
+ },
+ { // nul
+ 0.019995632340095118, 0.39709629708827826
+ },
+ { // bs
+ 0.004129666834030079
+ },
+ { // esc
+ 0.002523128679562658
+ },
+ { // c0
+ 0.0568959761244983, 0.1076923076923077
+ },
+ { // del
+ 0.001491465316532892
+ },
+ { // c1
+ 0.001404371180760891, 0.03362508201802201,
0.122322099538035
+ },
+ { // ff
+ 0.00473533621330323
+ },
+ { // hi
+ 0.0038095238095238095, 0.046875, 0.22629537237525812,
0.4921052631578947, 0.6565917638464245
+ },
+ { // ctlNoNul
+ 0.006213592366837315, 0.07654376183463077,
0.12507791780548536
+ },
+ { // nulRuns
+ 8.258064516129032, 87.31782945736434
+ },
+ { // nulRunMax
+ 1.0, 6.0
+ },
+ { // nulRunMean
+ 1.0, 2.2860107190286474
+ },
+ { // nulMod2
+ 0.5116279069767442, 0.6199380165289257
+ },
+ { // nulMod4
+ 0.2769366379930956, 0.4
+ },
+ { // nulAfterStr
+ 0.3915307971014493
+ },
+ { // nulStr
+ 0.28871331948096
+ },
+ { // cov8a
+ 0.06691937157305528, 0.3911804509845034,
0.8625976971706535, 0.9880588755860057
+ },
+ { // nlKB
+ 4.308630945998772, 11.407705673074272, 19.729053904812396,
28.779619808866116, 50.25766871165644
+ },
+ { // lineMean
+ 14.766729152465349, 24.37291666666667, 34.875, 46.0, 65.0,
106.66641025641026, 203.703125
+ },
+ { // lineStd
+ 4.499728862888747, 14.648999369161627, 24.874367443102166,
38.066371371830726, 62.818474464887444,
+ 124.01573926622027
+ },
+ { // lineMax
+ 34.0, 63.0, 89.0, 138.0, 236.0, 437.0, 842.0
+ },
+ { // bsCtx
+ 0.5
+ },
+ { // ac1
+ -0.014645433732129898, -6.090915087741955e-06, 0.0,
0.001156642184634793, 0.02923237875234655,
+ 0.1125418152391145, 0.23822469498055948
+ },
+ { // ac2
+ -0.0023151378088784383, 0.0, 0.005819579583405954,
0.05187415943023149, 0.14083653862136364,
+ 0.272730637787365
+ },
+ { // ac4
+ -0.003707768722685391, 0.0, 0.002868279756737202,
0.03738516597302008, 0.1123482449642961,
+ 0.2386099113685777
+ },
+ { // ac8
+ -0.0033944490193483015, 0.0, 0.0028463679233658723,
0.031422249339583846, 0.09236417472085226,
+ 0.21638970257424897
+ },
+ { // ac16
+ -0.0071732051701677915, 0.0, 0.0009436492081413839,
0.017246907046564786, 0.06897695306082857,
+ 0.18520800220809217
+ },
+ { // ent
+ 3.2225571723615913, 4.033423537031595, 4.560147688419784,
4.886989850387823, 5.1522139434457435,
+ 5.4761576479361, 6.398792231420389
+ },
+ { // distinct
+ 19.0, 33.0, 49.0, 64.0, 80.0, 108.0, 180.0
+ },
+ { // topB
+ 0.034482758620689655, 0.05834046056850623,
0.07209393453379019, 0.08769149498151083,
+ 0.11184713998273697, 0.17624892670160258,
0.49778684328405964
+ },
+ { // u8ok
+ 0.10157727153361344, 0.19414620217018846,
0.45454545454545453
+ },
+ { // u8badKB
+ 0.4183080190778277, 38.12977237838997, 187.60011920317265,
423.73148812770967, 517.8892901075354
+ },
+ { // u16
+ 0.5000750610134621, 0.7641788885433941,
0.8219416554456218, 0.9028357862511449, 0.9428571428571428,
+ 0.967741935483871, 0.9984012636289396
+ },
+ { // u16asc
+ 0.002168507640888958, 0.037037037037037035
+ },
+ };
+ private static final double[][] WEIGHTS = {
+ { // logn
+ -0.2310552191574832, -0.21454777147190596,
-0.18720285013968138, 0.04043538182810273,
+ -0.0011347842473640145, 0.5098808748046423,
0.029937496947774873, -0.19424181002493615
+ },
+ { // nul
+ -0.5534189838509629, -0.08852605572828375,
0.3940163581184254
+ },
+ { // bs
+ 0.941705619863977, -1.1896343013248327
+ },
+ { // esc
+ 0.36700426674966113, -0.6149329482105139
+ },
+ { // c0
+ 1.287796145711941, -0.5192731668499568, -1.0164516603229834
+ },
+ { // del
+ 0.586816300178283, -0.8347449816391818
+ },
+ { // c1
+ 1.3015976755135448, 0.2136216550141253,
-0.5161059143125614, -1.2470420976759873
+ },
+ { // ff
+ 0.12174317826350628, -0.3696718597243321
+ },
+ { // hi
+ 0.14965444085693166, 0.39476484585952565,
0.49127485791474845, -0.6483281868735459,
+ -0.6504227957585362, 0.015128156540113825
+ },
+ { // ctlNoNul
+ 1.4289802643302496, -0.517994133979678,
-0.7021363665739889, -0.4567784452375745
+ },
+ { // nulRuns
+ -0.2564679234529926, -0.18592516182962976,
0.19446440382178015
+ },
+ { // nulRunMax
+ 1.9328632906237004, -0.31834881031328466,
-1.8624431617713795
+ },
+ { // nulRunMean
+ 1.9328632906237004, -0.5923858075934905,
-1.5884061644911696
+ },
+ { // nulMod2
+ -0.099713586614645, 0.0553038928090881, -0.2035189876553327
+ },
+ { // nulMod4
+ -0.24256003738837195, -0.48982048848977344,
0.4844518444172544
+ },
+ { // nulAfterStr
+ -0.48829944089571653, 0.2403707594349252
+ },
+ { // nulStr
+ -0.9037103991283869, 0.6557817176675811
+ },
+ { // cov8a
+ -0.6880486969786129, -0.49266761809079457,
-0.922031077877108, 0.06470512914568709,
+ 1.790113582340125
+ },
+ { // nlKB
+ -0.9060205391104922, -0.9380839356393983,
-0.16574032676983722, 0.15709931992082168,
+ 0.6350006412382854, 0.9698161588998079
+ },
+ { // lineMean
+ 0.5535694600631904, 0.14193797040403583,
0.38109679748301095, 0.48135015034424317,
+ 0.36036675494371245, -0.03584748431925305,
-0.8152337160441433, -1.3151686143356447
+ },
+ { // lineStd
+ -0.7022035733239248, -0.017669433007049717,
0.17268461707830748, 0.4208108855370347,
+ -0.0344312513639432, 0.2593099080938668,
-0.3464298344751901
+ },
+ { // lineMax
+ -0.4337125077993913, -0.08815454486161163,
0.06141618379144683, -0.04202722613795733,
+ 0.07121522082134446, 0.26525175243975657,
0.12090224080376981, -0.20281980051821652
+ },
+ { // bsCtx
+ -0.4745241498412075, 0.22659546838031475
+ },
+ { // ac1
+ 0.9940394538443768, 0.5541923503449178,
0.08048984474590326, -0.646209691683989,
+ 0.28041195331372165, -0.600813029811643,
-0.4254032648676291, -0.48463629734650665
+ },
+ { // ac2
+ -0.028937001938560303, -0.2902408813910141,
-0.41246666619131167, -0.2128073200748067,
+ 0.06428475002166702, 0.6435252301020274,
-0.01128679198883952
+ },
+ { // ac4
+ -0.1963096243653022, -0.722004648858803,
-0.5702104194327913, -0.3201496153710012,
+ 0.06811452345543635, 0.2946020742824756, 1.1980290288291586
+ },
+ { // ac8
+ -0.2737512069109041, -0.7329743062802071,
-0.4364693017213012, 0.061542494945294283,
+ 0.0698718810231144, 0.3678820055016415, 0.6959697519815295
+ },
+ { // ac16
+ -0.12311810995138345, -0.3447769275861189,
-0.15218494178719055, 0.30226365606927824,
+ 0.032025834015999774, 0.03991020286289722,
-0.002048395084322618
+ },
+ { // ent
+ -0.31972906512940474, 0.6303748680407597,
0.21202965361914433, -0.1607538910083972,
+ 0.21406369819694765, 0.2975981081786166,
-0.18325798844721863, -0.9382540649112939
+ },
+ { // distinct
+ -2.1659173174584496, -1.14341367167224,
-0.38619106903632294, -0.1777969091663309,
+ 0.5302827815999743, 0.5894067720107392,
1.2356538328103999, 1.2700468994513778
+ },
+ { // topB
+ -1.5281763000720574, -0.5288595602473192,
0.1300143991973351, 0.7603002812320029, 0.53037160171491,
+ 0.8424515460638258, 0.6464507608068175, -1.1004814101563625
+ },
+ { // u8ok
+ -0.9270157212910567, -0.5388113512829429,
0.6593092993660308, 0.558589091747253
+ },
+ { // u8badKB
+ 0.6191097559302005, 0.5257182368960559,
0.5988965161121214, -0.35933937610436184,
+ -0.7089828107082297, -0.9233310035865361
+ },
+ { // u16
+ 0.6021631366019026, 0.053120195059327815,
-0.41057615520909246, -0.6069578121702731,
+ -0.23291750610992767, 0.15458659610644337,
0.6607799310113494, -0.46812706675057586
+ },
+ { // u16asc
+ 0.041099389670909856, -0.06469694129777277,
-0.22433112983399436
+ },
+ };
+ private static final double INTERCEPT = -0.3937321772678982;
+ static final double THRESHOLD = 0.8377535594988522;
+
+ private static final int NFEAT = FEATURES.length;
+
+ private TextBinaryClassifier() {
+ }
+
+ /**
+ * @return true if the bytes are printable ASCII or the model reads them
as text; false for
+ * binary or input under {@link #MIN_LENGTH}
+ */
+ public static boolean isText(byte[] data) {
+ if (data.length < MIN_LENGTH) {
+ return false;
+ }
+ // printable ASCII only: text by definition (the trainer dropped such
binary windows)
+ return isPrintableAscii(data) || probability(data) >= THRESHOLD;
+ }
+
+ private static boolean isPrintableAscii(byte[] data) {
+ for (byte b : data) {
+ int c = b & 0xFF;
+ if (c >= 0x7F || (c < 0x20 && c != 9 && c != 10 && c != 12 && c !=
13)) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ /**
+ * @return estimated probability that the bytes are text (calibrated on
held-out data)
+ */
+ public static double probability(byte[] data) {
+ if (data.length == 0) {
+ return 0;
+ }
+ return score(features(data));
+ }
+
+ static double score(double[] f) {
+ double z = INTERCEPT;
+ for (int j = 0; j < NFEAT; j++) {
+ z += WEIGHTS[j][bin(EDGES[j], f[j])];
+ }
+ return 1.0 / (1.0 + Math.exp(-z));
+ }
+
+ // number of interior edges <= x, as the trainer bins
+ private static int bin(double[] interior, double x) {
+ int k = 0;
+ while (k < interior.length && interior[k] <= x) {
+ k++;
+ }
+ return k;
+ }
+
+ private static boolean texty(int c) {
+ return c == 9 || c == 10 || c == 13 || (c >= 0x20 && c < 0x7F) || c >=
0x80;
+ }
+
+ private static boolean asciiPrintable(int c) {
+ return c == 9 || c == 10 || c == 13 || (c >= 0x20 && c < 0x7F);
+ }
+
+ // 0 NUL, 1 control, 2 other ASCII, 3 high byte
+ private static final int[] CLASS = new int[256];
+
+ static {
+ for (int c = 0; c < 256; c++) {
+ CLASS[c] = c == 0 ? 0
+ : (c < 0x20 && c != 9 && c != 10 && c != 13 && c != 12) ||
c == 0x7F ? 1
+ : c < 0x80 ? 2 : 3;
+ }
+ }
+
+ private static boolean u16Text(int u) {
+ return u == 9 || u == 10 || u == 13 || (u >= 0x20 && u < 0x7F) || (u
>= 0xA0 && u < 0x2C00)
+ || (u >= 0x3000 && u < 0xA000) || (u >= 0xAC00 && u < 0xD7A4)
|| (u >= 0xFF00 && u < 0xFFF0);
+ }
+
+ private static final int[] LAGS = {1, 2, 4, 8, 16};
+
+ /** Feature vector in {@link #FEATURES} order; definitions must match the
trainer exactly. */
+ static double[] features(byte[] b) {
+ int n = b.length;
+ double nd = n;
+ int[] h = new int[256];
+ int[] nulPos4 = new int[4];
+ int[] cls = new int[4];
+ int[] agree = new int[LAGS.length];
+ int[] u16ok = new int[2];
+ int[] u16asc = new int[2];
+ int nulRuns = 0;
+ int nulRunMax = 0;
+ long nulRunTot = 0;
+ int nulAfterStr = 0;
+ int nulStr = 0;
+ int run = 0;
+ int tRun = 0;
+ int aRun = 0;
+ int lastTextyRun = 0;
+ long cov8a = 0;
+ int bsCtx = 0;
+ int nl = 0;
+ int lineMax = 0;
+ int cur = 0;
+ double lsum = 0;
+ double lsq = 0;
+ int lcount = 0;
+ for (int i = 0; i < n; i++) {
+ int c = b[i] & 0xFF;
+ h[c]++;
+ int k = CLASS[c];
+ cls[k]++;
+ for (int l = 0; l < LAGS.length; l++) {
+ if (i >= LAGS[l] && CLASS[b[i - LAGS[l]] & 0xFF] == k) {
+ agree[l]++;
+ }
+ }
+ if (c == 0) {
+ nulPos4[i & 3]++;
+ if (run == 0) {
+ nulRuns++;
+ if (tRun >= 4) {
+ nulAfterStr++;
+ }
+ lastTextyRun = tRun;
+ }
+ run++;
+ } else if (run > 0) {
+ nulRunTot += run;
+ nulRunMax = Math.max(nulRunMax, run);
+ if (lastTextyRun >= 4 && texty(c)) {
+ nulStr++;
+ }
+ run = 0;
+ }
+ if (c != 0 && texty(c)) {
+ tRun++;
+ } else {
+ tRun = 0;
+ }
+ if (asciiPrintable(c)) {
+ aRun++;
+ } else {
+ if (aRun >= 8) {
+ cov8a += aRun;
+ }
+ aRun = 0;
+ }
+ if (c == 8) {
+ int pv = i > 0 ? b[i - 1] & 0xFF : 0;
+ int nx = i + 1 < n ? b[i + 1] & 0xFF : 0;
+ if ((pv == 8 || (pv != 0 && texty(pv))) && (nx == 8 || (nx !=
0 && texty(nx)))) {
+ bsCtx++;
+ }
+ }
+ if (c == '\n') {
+ nl++;
+ }
+ if (c == '\n' || (c == '\r' && (i + 1 >= n || b[i + 1] != '\n'))) {
+ lsum += cur;
+ lsq += (double) cur * cur;
+ lcount++;
+ lineMax = Math.max(lineMax, cur);
+ cur = 0;
+ } else if (c != '\r') {
+ cur++;
+ }
+ if ((i & 1) == 1) {
+ int lo = b[i - 1] & 0xFF;
+ int le = lo | c << 8;
+ int be = lo << 8 | c;
+ if (u16Text(le)) {
+ u16ok[0]++;
+ }
+ if (u16Text(be)) {
+ u16ok[1]++;
+ }
+ if (c == 0 && asciiPrintable(lo)) {
+ u16asc[0]++;
+ }
+ if (lo == 0 && asciiPrintable(c)) {
+ u16asc[1]++;
+ }
+ }
+ }
+ if (run > 0) {
+ nulRunTot += run;
+ nulRunMax = Math.max(nulRunMax, run);
+ }
+ if (aRun >= 8) {
+ cov8a += aRun;
+ }
+ lineMax = Math.max(lineMax, cur);
+ lsum += cur;
+ lsq += (double) cur * cur;
+ lcount++;
+
+ int u8ok = 0;
+ int u8bad = 0;
+ for (int i = 0; i < n; ) {
+ int c = b[i] & 0xFF;
+ if (c < 0x80) {
+ i++;
+ continue;
+ }
+ int k = c >= 0xF0 && c <= 0xF4 ? 3 : c >= 0xE0 ? 2 : c >= 0xC2 &&
c < 0xE0 ? 1 : -1;
+ if (c > 0xF4) {
+ k = -1;
+ }
+ if (k < 0) {
+ u8bad++;
+ i++;
+ continue;
+ }
+ if (i + k >= n) {
+ // sequence cut by the end of the prefix
+ u8ok += n - i;
+ break;
+ }
+ boolean ok = true;
+ for (int j = 1; j <= k; j++) {
+ if ((b[i + j] & 0xC0) != 0x80) {
+ ok = false;
+ break;
+ }
+ }
+ if (ok) {
+ u8ok += k + 1;
+ i += k + 1;
+ } else {
+ u8bad++;
+ i++;
+ }
+ }
+
+ int nul = h[0];
+ int hi = cls[3];
+ int c0 = 0;
+ for (int c = 1; c < 32; c++) {
+ if (c != 9 && c != 10 && c != 13 && c != 12 && c != 8 && c != 27) {
+ c0 += h[c];
+ }
+ }
+ int c1 = 0;
+ for (int c = 0x80; c < 0xA0; c++) {
+ c1 += h[c];
+ }
+ double ent = 0;
+ int distinct = 0;
+ int top = 0;
+ for (int c = 0; c < 256; c++) {
+ if (h[c] > 0) {
+ distinct++;
+ double p = h[c] / nd;
+ ent -= p * Math.log(p) / Math.log(2);
+ if (c != 32 && c != 10 && c != 13 && c != 9) {
+ top = Math.max(top, h[c]);
+ }
+ }
+ }
+ double exp = 0;
+ for (int k = 0; k < 4; k++) {
+ exp += Math.pow(cls[k] / nd, 2);
+ }
+ double lm = lsum / lcount;
+ int units = n / 2;
+ double[] ac = new double[LAGS.length];
+ for (int l = 0; l < LAGS.length; l++) {
+ int m = Math.max(0, n - LAGS[l]);
+ ac[l] = m == 0 ? 0.0 : agree[l] / (double) m - exp;
+ }
+ int nm2 = Math.max(nulPos4[0] + nulPos4[2], nulPos4[1] + nulPos4[3]);
+ int nm4 = Math.max(Math.max(nulPos4[0], nulPos4[1]),
Math.max(nulPos4[2], nulPos4[3]));
+ return new double[] {
+ Math.log(n) / Math.log(2),
+ nul / nd,
+ h[8] / nd,
+ h[27] / nd,
+ c0 / nd,
+ h[0x7F] / nd,
+ c1 / nd,
+ h[0xFF] / nd,
+ hi / nd,
+ (h[8] + h[27] + c0 + h[0x7F]) / nd,
+ nulRuns * 1024.0 / nd,
+ nulRunMax,
+ nulRuns == 0 ? 0.0 : (double) nulRunTot / nulRuns,
+ nul == 0 ? 0.5 : (double) nm2 / nul,
+ nul == 0 ? 0.25 : (double) nm4 / nul,
+ nulRuns == 0 ? 0.0 : (double) nulAfterStr / nulRuns,
+ nulRuns == 0 ? 0.0 : (double) nulStr / nulRuns,
+ cov8a / nd,
+ nl * 1024.0 / nd,
+ lm,
+ Math.sqrt(Math.max(0, lsq / lcount - lm * lm)),
+ lineMax,
+ h[8] == 0 ? 0.0 : (double) bsCtx / h[8],
+ ac[0], ac[1], ac[2], ac[3], ac[4],
+ ent,
+ distinct,
+ top / nd,
+ hi == 0 ? 1.0 : u8ok / (double) (u8ok + u8bad),
+ u8bad * 1024.0 / nd,
+ units == 0 ? 0 : Math.max(u16ok[0], u16ok[1]) / (double) units,
+ units == 0 ? 0 : Math.max(u16asc[0], u16asc[1]) / (double)
units,
+ };
+ }
+}
diff --git a/tika-core/src/main/java/org/apache/tika/mime/MimeTypes.java
b/tika-core/src/main/java/org/apache/tika/mime/MimeTypes.java
index 70910abd66..c4a7f48a1c 100644
--- a/tika-core/src/main/java/org/apache/tika/mime/MimeTypes.java
+++ b/tika-core/src/main/java/org/apache/tika/mime/MimeTypes.java
@@ -37,6 +37,7 @@ import javax.xml.namespace.QName;
import org.apache.tika.Tika;
import org.apache.tika.annotation.TikaComponent;
import org.apache.tika.detect.Detector;
+import org.apache.tika.detect.TextBinaryClassifier;
import org.apache.tika.detect.TextDetector;
import org.apache.tika.detect.TextStatistics;
import org.apache.tika.detect.XmlRootExtractor;
@@ -306,10 +307,15 @@ public final class MimeTypes implements Detector,
Serializable {
return result;
}
- // Finally, assume plain text if no control bytes are found
+ byte[] content = withoutNulTail(data);
+ if (content.length >= TextBinaryClassifier.MIN_LENGTH) {
+ return TextBinaryClassifier.isText(content) ?
Collections.singletonList(textMimeType)
+ : rootMimeTypeL;
+ }
+ // too short for the classifier: plain text only if no control bytes
are found
try {
TextDetector detector = new TextDetector(getMinLength());
- try (TikaInputStream tis =
TikaInputStream.get(withoutNulTail(data))) {
+ try (TikaInputStream tis = TikaInputStream.get(content)) {
MimeType type = forName(detector.detect(tis, new Metadata(),
new ParseContext()).toString());
return Collections.singletonList(type);
}
diff --git
a/tika-core/src/test/java/org/apache/tika/detect/TextBinaryClassifierTest.java
b/tika-core/src/test/java/org/apache/tika/detect/TextBinaryClassifierTest.java
new file mode 100644
index 0000000000..f8acc6a8dd
--- /dev/null
+++
b/tika-core/src/test/java/org/apache/tika/detect/TextBinaryClassifierTest.java
@@ -0,0 +1,159 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.detect;
+
+import static java.nio.charset.StandardCharsets.US_ASCII;
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.io.ByteArrayOutputStream;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.charset.Charset;
+import java.util.Arrays;
+import java.util.Random;
+
+import org.junit.jupiter.api.Test;
+
+import org.apache.tika.io.TikaInputStream;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.mime.MediaType;
+import org.apache.tika.mime.MimeTypes;
+import org.apache.tika.parser.ParseContext;
+
+public class TextBinaryClassifierTest {
+
+ private static final String RUSSIAN = "Банк России сообщает, что с 1 июля
изменяются правила расчёта "
+ + "обязательных резервов для кредитных организаций.\r\nПросим
довести настоящее письмо до "
+ + "сведения всех заинтересованных лиц и обеспечить его исполнение
в установленный срок.\r\n";
+ private static final String CHINESE = "关于做好二〇一九年度预算执行情况审计工作的通知\r\n各区县人民政府,"
+ + "市政府各部门:为进一步加强财政资金管理,提高资金使用效益,现将有关事项通知如下。\r\n"
+ + "一、高度重视审计工作,切实落实主体责任。二、严格按照规定时限报送材料。\r\n";
+ private static final String JAPANESE =
"第三十回全国大会のご案内\n日時:二〇一八年十月十三日(土)午前十時より\n"
+ + "会場:東京国際フォーラム ホールB\n参加をご希望の方は、事務局までお申し込みください。\n"
+ + "詳しくは当会ホームページをご覧ください。皆様のご参加をお待ちしております。\n";
+ private static final String ENGLISH = "Release notes for version 2.4.1\n\n"
+ + "Fixed a crash when the configuration file contains an empty
section.\n"
+ + "The installer now keeps existing settings when upgrading in
place.\n"
+ + "Thanks to everyone who reported problems on the mailing
list.\n";
+
+ private static byte[] repeat(byte[] unit, int times) {
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ for (int i = 0; i < times; i++) {
+ out.writeBytes(unit);
+ }
+ return out.toByteArray();
+ }
+
+ private static void assertText(byte[] data) {
+ assertTrue(TextBinaryClassifier.isText(data), "p=" +
TextBinaryClassifier.probability(data));
+ }
+
+ private static void assertBinary(byte[] data) {
+ assertFalse(TextBinaryClassifier.isText(data), "p=" +
TextBinaryClassifier.probability(data));
+ }
+
+ @Test
+ public void legacyAndUnicodeText() {
+ assertText(repeat(RUSSIAN.getBytes(Charset.forName("windows-1251")),
4));
+ assertText(repeat(CHINESE.getBytes(Charset.forName("GB18030")), 4));
+ assertText(repeat(JAPANESE.getBytes(Charset.forName("EUC-JP")), 4));
+ assertText(repeat(JAPANESE.getBytes(Charset.forName("Shift_JIS")), 4));
+ assertText(repeat((RUSSIAN + CHINESE + ENGLISH).getBytes(UTF_8), 3));
+ }
+
+ @Test
+ public void textWithControls() {
+ // a stray bell and form feed in an otherwise plain log
+ String log = "2019-03-04 10:15:02 INFO starting worker pool size=8\n"
+ + "2019-03-04 10:15:03 WARN disk usage at 91% on /var\u0007\n"
+ + "2019-03-04 10:15:09 INFO checkpoint written in 412 ms\n\f"
+ + "2019-03-04 10:16:00 INFO shutting down\n";
+ assertText(repeat(log.getBytes(US_ASCII), 6));
+
+ // a preallocated log whose middle block was never written
+ byte[] hole = repeat(log.getBytes(US_ASCII), 12);
+ Arrays.fill(hole, hole.length / 3, hole.length / 3 + 600, (byte) 0);
+ assertText(hole);
+
+ // an installer log with a backspace-driven progress counter
+ StringBuilder sb = new StringBuilder("Extracting files to C:\\Program
Files\\Example\r\n");
+ for (int pct = 0; pct <= 100; pct += 5) {
+ sb.append(String.format(java.util.Locale.ROOT, "%3d%%",
pct)).append("\b\b\b\b");
+ }
+ sb.append("\r\nInstallation finished. Restart the computer to complete
setup.\r\n");
+ assertText(repeat(sb.toString().getBytes(US_ASCII), 4));
+ }
+
+ @Test
+ public void binary() {
+ byte[] random = new byte[8192];
+ new Random(5029).nextBytes(random);
+ assertBinary(random);
+
+ byte[] fill = repeat(new byte[] {(byte) 0xC7, (byte) 0xC7, (byte)
0xC7, (byte) 0xC7, (byte) 0xC7,
+ (byte) 0xC7, (byte) 0xC7, (byte) 0xC7, (byte) 0xC7, (byte)
0xC7, '\n'}, 40);
+ assertBinary(fill);
+
+ byte[] pcm = new byte[4000];
+ for (int i = 0; i < pcm.length; i++) {
+ pcm[i] = (byte) (128 + 90 * Math.sin(2 * Math.PI * 440 * i /
8000.0));
+ }
+ assertBinary(pcm);
+
+ // Elf64_Sym records: name, info, other, shndx, value, size
+ ByteBuffer sym = ByteBuffer.allocate(24 *
64).order(ByteOrder.LITTLE_ENDIAN);
+ for (int i = 0; i < 64; i++) {
+ sym.putInt(1 + 17 * i).put((byte) 0x12).put((byte)
0).putShort((short) 14)
+ .putLong(0x401000L + 0x40L * i).putLong(32 + i);
+ }
+ assertBinary(sym.array());
+ }
+
+ @Test
+ public void printableAsciiIsText() {
+ // a low-diversity payload the model alone scores binary (p 0.79)
+ byte[] runs = "XXXXXXXXXXYYYYYYYYYY".getBytes(US_ASCII);
+ assertTrue(TextBinaryClassifier.probability(runs) < 0.837);
+ assertText(runs);
+ }
+
+ @Test
+ public void shortInputIsNotClassified() {
+ assertFalse(TextBinaryClassifier.isText("fifteen
bytes!!".getBytes(US_ASCII)));
+ }
+
+ @Test
+ public void mimeTypesUsesClassifier() throws Exception {
+ MimeTypes types = MimeTypes.getDefaultMimeTypes();
+ byte[] russian =
repeat(RUSSIAN.getBytes(Charset.forName("windows-1251")), 4);
+ assertEquals(MediaType.TEXT_PLAIN, detect(types, russian));
+ byte[] fill = repeat(new byte[] {(byte) 0xC7, (byte) 0xC7, (byte)
0xC7, (byte) 0xC7, '\n'}, 60);
+ assertEquals(MediaType.OCTET_STREAM, detect(types, fill));
+ // 15 bytes stay on the control-byte check
+ assertEquals(MediaType.TEXT_PLAIN, detect(types, "fifteen
bytes!!".getBytes(US_ASCII)));
+ assertEquals(MediaType.OCTET_STREAM, detect(types, new byte[] {1, 2,
3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}));
+ }
+
+ private static MediaType detect(MimeTypes types, byte[] data) throws
Exception {
+ try (TikaInputStream tis = TikaInputStream.get(data)) {
+ return types.detect(tis, new Metadata(), new ParseContext());
+ }
+ }
+}
diff --git
a/tika-core/src/test/java/org/apache/tika/mime/MimeDetectionTest.java
b/tika-core/src/test/java/org/apache/tika/mime/MimeDetectionTest.java
index 0800216d86..c98a7c076c 100644
--- a/tika-core/src/test/java/org/apache/tika/mime/MimeDetectionTest.java
+++ b/tika-core/src/test/java/org/apache/tika/mime/MimeDetectionTest.java
@@ -16,6 +16,7 @@
*/
package org.apache.tika.mime;
+import static java.nio.charset.StandardCharsets.ISO_8859_1;
import static java.nio.charset.StandardCharsets.US_ASCII;
import static java.nio.charset.StandardCharsets.UTF_16BE;
import static java.nio.charset.StandardCharsets.UTF_16LE;
@@ -27,6 +28,11 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.util.Arrays;
+import java.util.Locale;
+import java.util.Random;
import org.apache.commons.io.ByteOrderMark;
import org.junit.jupiter.api.BeforeAll;
@@ -451,10 +457,6 @@ public class MimeDetectionTest {
public void testBinaryContentRefusesTextGlob() throws IOException {
// headers copied from real files that a text glob used to claim
String[][] cases = {
- // Terminal Reality POD archive
- {"Bulldog.pod",
"2f000000436f6d70696c65642062792042696e456469742056657273696f6e20"
- +
"322e322e38207da62d2900000000000000000000000000000000000000000000"
- +
"00000000000000000000000000000000000000004152545c4241545453352e41"},
// Mozilla startupCache XDR bytecode, named for its source
{"AccountManager.js",
"0000001c000000b9000000c5fcc6bf73b9000000000000000000000000000000"
@@ -482,6 +484,45 @@ public class MimeDetectionTest {
}
}
+ @Test
+ public void testTerminalRealityPodIsBinary() throws IOException {
+ byte[] pod = terminalRealityPod();
+ assertEquals(MediaType.OCTET_STREAM, detect(pod, "Bulldog.pod"));
+ Metadata metadata = new Metadata();
+ metadata.set(HttpHeaders.CONTENT_TYPE, "text/plain");
+ try (TikaInputStream tis = TikaInputStream.get(pod)) {
+ assertEquals(MediaType.OCTET_STREAM, MIME_TYPES.detect(tis,
metadata, new ParseContext()));
+ }
+ }
+
+ // shaped like real BinEdit .pod archives: entry count, comment, 40-byte
directory entries, indexed pixels
+ private static byte[] terminalRealityPod() {
+ ByteBuffer bb =
ByteBuffer.allocate(16384).order(ByteOrder.LITTLE_ENDIAN);
+ int entries = 48;
+ bb.putInt(entries);
+ bb.put("Compiled by BinEdit Version 2.2.8 }".getBytes(ISO_8859_1));
+ bb.put(new byte[] {(byte) 0xA6, '-', ')'});
+ bb.position(0x54);
+ Random random = new Random(5029);
+ int offset = 0x54 + 40 * entries;
+ for (int i = 0; i < entries; i++) {
+ String name = String.format(Locale.ROOT, "ART\\%02dBF%02d.%s",
random.nextInt(100), i / 2,
+ i % 2 == 0 ? "ACT" : "RAW");
+ byte[] padded = Arrays.copyOf(name.getBytes(US_ASCII), 32);
+ int size = i % 2 == 0 ? 0x300 : 0x10000;
+ bb.put(padded).putInt(size).putInt(offset);
+ offset += size;
+ }
+ while (bb.hasRemaining()) {
+ byte value = (byte) random.nextInt(256);
+ int run = 1 + random.nextInt(24);
+ for (int k = 0; k < run && bb.hasRemaining(); k++) {
+ bb.put(value);
+ }
+ }
+ return bb.array();
+ }
+
@Test
public void testTextGlobStillWinsForText() throws IOException {
String latin = "The quick brown fox jumps over the lazy dog.\n";