This is an automated email from the ASF dual-hosted git repository.
tballison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git
The following commit(s) were added to refs/heads/main by this push:
new d55249ac07 TIKA-5029: Binary as text detection (#3592)
d55249ac07 is described below
commit d55249ac073e1fd5d721ad1e166a8f6d56307464
Author: Tim Allison <[email protected]>
AuthorDate: Mon Nov 9 08:58:50 2026 -0500
TIKA-5029: Binary as text detection (#3592)
* don't detect binary content as text
A text glob or Content-Type no longer wins over magic-less content with
5%+
control bytes that is not UTF-16; DefaultDetector drops its 512-byte text
re-sniff; zero fill after text is ignored; UTF-8 BOM + binary magic
detects
the magic. New/fixed magic: Emacs elc (all versions), Lua bytecode, Qt qm,
SELinux policy package, PCP archive, binary FrameMaker (<MakerFile).
Co-Authored-By: Claude Opus 5.5 <[email protected]>
---
CHANGES.txt | 19 +++
.../org/apache/tika/detect/DefaultDetector.java | 23 +--
.../main/java/org/apache/tika/mime/MimeTypes.java | 179 +++++++++++++++++++--
.../org/apache/tika/mime/tika-mimetypes.xml | 43 ++++-
.../java/org/apache/tika/TikaDetectionTest.java | 2 +-
.../org/apache/tika/mime/MimeDetectionTest.java | 128 +++++++++++++++
.../apache/tika/mime/NicheFormatDetectionTest.java | 40 ++++-
.../java/org/apache/tika/mime/TestMimeTypes.java | 7 +-
.../org/apache/tika/parser/pkg/ZipParserTest.java | 5 +-
9 files changed, 403 insertions(+), 43 deletions(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index bdb9ab009f..b92cbac3a0 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -104,6 +104,25 @@ Release 4.3.0 - ???
EXTRACTION AND FORMATS
+ * Binary files are no longer detected as text because of their name or a
text header.
+ When content has no magic and at least 5% control bytes, and does not
decode as
+ UTF-16 or UTF-32, a text type from the file name or Content-Type no
longer wins: Terminal Reality
+ .pod archives, Mozilla startupCache bytecode named .js or .xul, CODESYS
.meta files and
+ similar are now application/octet-stream. DefaultDetector no longer
re-sniffs the first
+ 512 bytes for text after MimeTypes found the 64 KB prefix binary, so a
binary body
+ behind a short text header (Sass caches, BSB/KAP charts) is no longer
text/plain;
+ zero fill after lines of text (a crashed or preallocated log) does not
count as binary.
+ A UTF-8 BOM followed by binary magic (JPEG, PNG, GIF, ID3, zip, SQLite)
is detected by
+ that magic instead of as text (TIKA-5029).
+
+ * New or fixed magic: Emacs bytecode from every Emacs version (only version
19 matched
+ before, and *.elc is no longer also an octet-stream glob), Lua bytecode
+ (application/x-lua-bytecode), Qt .qm translations
(application/x-qt-translation, were
+ text/html), SELinux policy packages
(application/x-selinux-policy-package, were
+ text/x-pascal by *.pp), Performance Co-Pilot archives
(application/x-pcp-archive), and
+ binary FrameMaker documents ("<MakerFile"), now
application/vnd.framemaker instead of
+ the text MIF type application/vnd.mif (TIKA-5029).
+
* Markdown output keeps document code inside its code spans and blocks: a
line of
backticks inside <pre> no longer closes the fenced code block, and a line
break inside
inline code, alt text or a link destination, or a lone CR inside <pre>,
no longer starts
diff --git
a/tika-core/src/main/java/org/apache/tika/detect/DefaultDetector.java
b/tika-core/src/main/java/org/apache/tika/detect/DefaultDetector.java
index e6cd160446..5f242dd7c6 100644
--- a/tika-core/src/main/java/org/apache/tika/detect/DefaultDetector.java
+++ b/tika-core/src/main/java/org/apache/tika/detect/DefaultDetector.java
@@ -38,7 +38,6 @@ import org.apache.tika.utils.ServiceLoaderUtils;
* <ol>
* <li>MimeTypes (magic byte) detection</li>
* <li>Container and other detectors loaded via SPI</li>
- * <li>TextDetector as fallback for unknown types</li>
* <li>Returns the most specific type detected</li>
* </ol>
* <p>
@@ -61,14 +60,12 @@ public class DefaultDetector extends CompositeDetector {
private transient final ServiceLoader loader;
private final Collection<Class<? extends Detector>> excludedClasses;
private final MimeTypes mimeTypes;
- private final TextDetector textDetector;
public DefaultDetector(MimeTypes types, ServiceLoader loader,
Collection<Class<? extends Detector>>
excludeDetectors) {
super(types.getMediaTypeRegistry(), getDefaultDetectors(loader,
excludeDetectors));
this.loader = loader;
this.mimeTypes = types;
- this.textDetector = new TextDetector();
this.excludedClasses = excludeDetectors != null ?
Collections.unmodifiableCollection(new
ArrayList<>(excludeDetectors)) :
Collections.emptySet();
@@ -149,18 +146,12 @@ public class DefaultDetector extends CompositeDetector {
// Note: Container detectors that need random access handle their own
spooling
MediaType detectedType = super.detect(tis, metadata, parseContext);
- // 3. Text detection - only if still unknown
- MediaType textType = null;
- if (MediaType.OCTET_STREAM.equals(detectedType) &&
- MediaType.OCTET_STREAM.equals(magicType)) {
- textType = textDetector.detect(tis, metadata, parseContext);
- }
-
- // 4. Return most specific
- return mostSpecific(magicType, detectedType, textType);
+ // 3. Return most specific. No text re-sniff: MimeTypes judged text
over its full
+ // prefix, and a shorter window calls a text header on a binary body
text.
+ return mostSpecific(magicType, detectedType);
}
- private MediaType mostSpecific(MediaType magicType, MediaType
detectedType, MediaType textType) {
+ private MediaType mostSpecific(MediaType magicType, MediaType
detectedType) {
MediaTypeRegistry registry = mimeTypes.getMediaTypeRegistry();
// Collect non-null, non-octet-stream candidates
@@ -188,12 +179,6 @@ public class DefaultDetector extends CompositeDetector {
// else: best is already more specific than detectedType, keep best
}
- // Text detection as fallback only if still unknown
- if (MediaType.OCTET_STREAM.equals(best) && textType != null &&
- !MediaType.OCTET_STREAM.equals(textType)) {
- best = textType;
- }
-
return best;
}
diff --git a/tika-core/src/main/java/org/apache/tika/mime/MimeTypes.java
b/tika-core/src/main/java/org/apache/tika/mime/MimeTypes.java
index e16eba987b..70910abd66 100644
--- a/tika-core/src/main/java/org/apache/tika/mime/MimeTypes.java
+++ b/tika-core/src/main/java/org/apache/tika/mime/MimeTypes.java
@@ -25,6 +25,7 @@ import java.io.Serializable;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
@@ -37,6 +38,7 @@ import org.apache.tika.Tika;
import org.apache.tika.annotation.TikaComponent;
import org.apache.tika.detect.Detector;
import org.apache.tika.detect.TextDetector;
+import org.apache.tika.detect.TextStatistics;
import org.apache.tika.detect.XmlRootExtractor;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.HttpHeaders;
@@ -75,6 +77,18 @@ public final class MimeTypes implements Detector,
Serializable {
* Name of the {@link #xmlMimeType xml} type, application/xml.
*/
public static final String XML = "application/xml";
+ // below this share of control bytes, a text glob still wins over
binary-looking content
+ private static final int BINARY_CONTROL_PERCENT = 5;
+ private static final boolean[] SAFE_CONTROL = new boolean[0x20];
+ private static final int MIN_TEXT_BEFORE_NUL_TAIL = 128;
+
+ static {
+ SAFE_CONTROL['\t'] = true;
+ SAFE_CONTROL['\n'] = true;
+ SAFE_CONTROL[0x0C] = true;
+ SAFE_CONTROL['\r'] = true;
+ SAFE_CONTROL[0x1B] = true;
+ }
/**
* Serial version UID.
*/
@@ -236,19 +250,15 @@ public final class MimeTypes implements Detector,
Serializable {
return rootMimeTypeL;
}
- // Then, check for magic bytes
- List<MimeType> result = new ArrayList<>(1);
- int currentPriority = -1;
- for (Magic magic : magics) {
- if (currentPriority > 0 && currentPriority > magic.getPriority()) {
- break;
- }
- if (magic.eval(data)) {
- result.add(magic.getType());
- currentPriority = magic.getPriority();
+ // A UTF-8 BOM in front of binary magic (a server prepending one) is
not text
+ if (startsWithUtf8Bom(data)) {
+ List<MimeType> afterBom = binaryMagicAfterBom(data);
+ if (afterBom != null) {
+ return afterBom;
}
}
+ List<MimeType> result = matchMagics(data);
if (!result.isEmpty()) {
for (int i = 0; i < result.size(); i++) {
final MimeType matched = result.get(i);
@@ -299,7 +309,7 @@ public final class MimeTypes implements Detector,
Serializable {
// Finally, assume plain text if no control bytes are found
try {
TextDetector detector = new TextDetector(getMinLength());
- try (TikaInputStream tis = TikaInputStream.get(data)) {
+ try (TikaInputStream tis =
TikaInputStream.get(withoutNulTail(data))) {
MimeType type = forName(detector.detect(tis, new Metadata(),
new ParseContext()).toString());
return Collections.singletonList(type);
}
@@ -308,6 +318,140 @@ public final class MimeTypes implements Detector,
Serializable {
}
}
+ private List<MimeType> matchMagics(byte[] data) {
+ List<MimeType> result = new ArrayList<>(1);
+ int currentPriority = -1;
+ for (Magic magic : magics) {
+ if (currentPriority > 0 && currentPriority > magic.getPriority()) {
+ break;
+ }
+ if (magic.eval(data)) {
+ result.add(magic.getType());
+ currentPriority = magic.getPriority();
+ }
+ }
+ return result;
+ }
+
+ private static boolean startsWithUtf8Bom(byte[] data) {
+ return data.length > 3 && data[0] == (byte) 0xEF && data[1] == (byte)
0xBB
+ && data[2] == (byte) 0xBF;
+ }
+
+ /**
+ * Magic of a non-text type matched after the BOM, provided the rest does
not read as text,
+ * or null.
+ */
+ private List<MimeType> binaryMagicAfterBom(byte[] data) {
+ byte[] rest = Arrays.copyOfRange(data, 3, data.length);
+ List<MimeType> result = matchMagics(rest);
+ if (result.isEmpty()) {
+ return null;
+ }
+ for (MimeType type : result) {
+ if (isTextBased(type)) {
+ return null;
+ }
+ }
+ TextStatistics stats = new TextStatistics();
+ stats.addData(rest, 0, rest.length);
+ if (stats.isMostlyAscii() || stats.looksLikeUTF8()) {
+ return null;
+ }
+ return result;
+ }
+
+ private boolean isTextBased(MimeType type) {
+ return MediaType.TEXT_PLAIN.equals(type.getType())
+ || registry.isSpecializationOf(type.getType(),
MediaType.TEXT_PLAIN);
+ }
+
+ /**
+ * True when the prefix has too many control bytes to be text in any
single-byte encoding
+ * or UTF-8, and does not decode as UTF-16 or UTF-32 either. Only control
bytes count: text in a
+ * legacy 8-bit encoding is mostly high bytes and must not be called
binary.
+ */
+ private static boolean looksBinary(byte[] data) {
+ data = withoutNulTail(data);
+ if (data.length == 0) {
+ return false;
+ }
+ int control = 0;
+ for (byte b : data) {
+ int c = b & 0xFF;
+ if (c < 0x20 && !SAFE_CONTROL[c]) {
+ control++;
+ }
+ }
+ if ((long) control * 100 < (long) data.length *
BINARY_CONTROL_PERCENT) {
+ return false;
+ }
+ return !decodesAsUtf16(data, false) && !decodesAsUtf16(data, true)
+ && !decodesAsUtf32(data, false) && !decodesAsUtf32(data, true);
+ }
+
+ /**
+ * Drops zero fill after some lines of text (a preallocated or crashed
log). A short header
+ * padded with zeros is left alone: that is how binary formats look.
+ */
+ private static byte[] withoutNulTail(byte[] data) {
+ int end = data.length;
+ while (end > 0 && data[end - 1] == 0) {
+ end--;
+ }
+ if (end == data.length || end < MIN_TEXT_BEFORE_NUL_TAIL) {
+ return data;
+ }
+ for (int i = 0; i < end; i++) {
+ if (data[i] == '\n') {
+ return Arrays.copyOf(data, end);
+ }
+ }
+ return data;
+ }
+
+ // script-agnostic: needs no NULs, only that the code units are plausible
characters
+ private static boolean decodesAsUtf16(byte[] data, boolean bigEndian) {
+ int units = data.length / 2;
+ int bad = 0;
+ boolean pendingHigh = false;
+ for (int i = 0; i < units; i++) {
+ int a = data[2 * i] & 0xFF;
+ int b = data[2 * i + 1] & 0xFF;
+ char c = (char) (bigEndian ? (a << 8) | b : (b << 8) | a);
+ if (pendingHigh) {
+ pendingHigh = false;
+ if (Character.isLowSurrogate(c)) {
+ continue;
+ }
+ bad++;
+ }
+ if (Character.isHighSurrogate(c)) {
+ pendingHigh = true;
+ } else if (Character.isLowSurrogate(c) || c == 0xFFFE || c ==
0xFFFF
+ || (c < 0x20 && !SAFE_CONTROL[c]) || (c >= 0x7F && c <
0xA0)) {
+ bad++;
+ }
+ }
+ return units > 0 && (long) bad * 100 < (long) units * 2;
+ }
+
+ private static boolean decodesAsUtf32(byte[] data, boolean bigEndian) {
+ int units = data.length / 4;
+ int bad = 0;
+ for (int i = 0; i < units; i++) {
+ int o = 4 * i;
+ int cp = bigEndian
+ ? (data[o] & 0xFF) << 24 | (data[o + 1] & 0xFF) << 16 |
(data[o + 2] & 0xFF) << 8 | data[o + 3] & 0xFF
+ : (data[o + 3] & 0xFF) << 24 | (data[o + 2] & 0xFF) << 16
| (data[o + 1] & 0xFF) << 8 | data[o] & 0xFF;
+ if (cp < 0 || cp > Character.MAX_CODE_POINT || (cp >= 0xD800 && cp
<= 0xDFFF)
+ || (cp & 0xFFFE) == 0xFFFE || (cp < 0x20 &&
!SAFE_CONTROL[cp]) || (cp >= 0x7F && cp < 0xA0)) {
+ bad++;
+ }
+ }
+ return units > 0 && (long) bad * 100 < (long) units * 2;
+ }
+
/**
* Reads the first {@link #getMinLength()} bytes from the given stream.
* If the stream is shorter, then the entire content of the stream is
@@ -540,6 +684,8 @@ public final class MimeTypes implements Detector,
Serializable {
public MediaType detect(TikaInputStream tis, Metadata metadata,
ParseContext parseContext)
throws IOException {
List<MimeType> possibleTypes = null;
+ // content with no magic that is not text: a text type from a name or
header can't win
+ boolean binaryContent = false;
// Get type based on magic prefix
if (tis != null) {
@@ -556,6 +702,7 @@ public final class MimeTypes implements Detector,
Serializable {
try {
byte[] prefix = readMagicHeader(tis, toRead);
possibleTypes = getMimeType(prefix);
+ binaryContent = isRoot(possibleTypes) && looksBinary(prefix);
} finally {
tis.reset();
}
@@ -593,7 +740,7 @@ public final class MimeTypes implements Detector,
Serializable {
// For server-side scripting languages, we cannot rely on the
filename to detect
// the mime type
- if (!(isHttp && hint.isInterpreted())) {
+ if (!(isHttp && hint.isInterpreted()) && !(binaryContent &&
isTextBased(hint))) {
// If we have some types based on mime magic, try to
specialise
// and/or select the type based on that
// Otherwise, use the type identified from the name
@@ -607,7 +754,9 @@ public final class MimeTypes implements Detector,
Serializable {
if (typeName != null) {
try {
MimeType hint = forName(typeName);
- possibleTypes = applyHint(possibleTypes, hint);
+ if (!(binaryContent && isTextBased(hint))) {
+ possibleTypes = applyHint(possibleTypes, hint);
+ }
} catch (MimeTypeException e) {
// Malformed type name, ignore
}
@@ -627,6 +776,10 @@ public final class MimeTypes implements Detector,
Serializable {
* If the hint is a specialised form, use that instead
* If there are multiple possible types, use the hint to select one
*/
+ private boolean isRoot(List<MimeType> possibleTypes) {
+ return possibleTypes.size() == 1 &&
rootMimeType.equals(possibleTypes.get(0));
+ }
+
private List<MimeType> applyHint(List<MimeType> possibleTypes, MimeType
hint) {
if (possibleTypes == null || possibleTypes.isEmpty()) {
return Collections.singletonList(hint);
diff --git
a/tika-core/src/main/resources/org/apache/tika/mime/tika-mimetypes.xml
b/tika-core/src/main/resources/org/apache/tika/mime/tika-mimetypes.xml
index 4dafbda9ee..d3079faabe 100644
--- a/tika-core/src/main/resources/org/apache/tika/mime/tika-mimetypes.xml
+++ b/tika-core/src/main/resources/org/apache/tika/mime/tika-mimetypes.xml
@@ -790,7 +790,6 @@
<glob pattern="*.pkg"/>
<glob pattern="*.bpk"/>
<glob pattern="*.dump"/>
- <glob pattern="*.elc"/>
<glob pattern="*.deploy"/>
</mime-type>
@@ -1712,6 +1711,11 @@
</mime-type>
<mime-type type="application/vnd.font-fontforge-sfd"/>
<mime-type type="application/vnd.framemaker">
+ <_comment>FrameMaker binary document</_comment>
+ <!-- above the MIF "<Maker" prefix -->
+ <magic priority="60">
+ <match value="<MakerFile" type="string" offset="0"/>
+ </magic>
<glob pattern="*.fm"/>
<glob pattern="*.frame"/>
<glob pattern="*.maker"/>
@@ -2189,7 +2193,6 @@
<alias type="application/x-frame"/>
<sub-class-of type="text/plain"/>
<magic priority="50">
- <match value="<MakerFile" type="string" offset="0" />
<match value="<MIFFile" type="string" offset="0" />
<match value="<MakerDictionary" type="string" offset="0" />
<match value="<MakerScreenFont" type="string" offset="0" />
@@ -4497,8 +4500,8 @@
<magic priority="50">
<!-- Emacs 18 -->
<match value="\012(" type="string" offset="0" />
- <!-- Emacs 19 -->
- <match value=";ELC\023\000\000\000" type="string" offset="0" />
+ <!-- Emacs 19 and later: the fifth byte is the bytecode version -->
+ <match value=";ELC\000\000\000\000" mask="0xFFFFFFFFE0FFFFFF"
type="string" offset="0" />
</magic>
<glob pattern="*.elc"/>
</mime-type>
@@ -5097,6 +5100,14 @@
</match>
</magic>
</mime-type>
+ <mime-type type="application/x-lua-bytecode">
+ <_comment>Lua bytecode</_comment>
+ <magic priority="50">
+ <match value="\033Lua" type="string" offset="0"/>
+ </magic>
+ <glob pattern="*.luac"/>
+ </mime-type>
+
<mime-type type="application/x-lz4">
<_comment>First match LZ4 Frame</_comment>
<_comment>Second match Legacy Frame</_comment>
@@ -5664,6 +5675,14 @@
<match value="PAR1" type="string" offset="0" />
</magic>
</mime-type>
+ <mime-type type="application/x-pcp-archive">
+ <_comment>Performance Co-Pilot archive (metadata, index or
volume)</_comment>
+ <magic priority="50">
+ <!-- version 2 label record: length 132, then the label magic -->
+ <match value="0x0000008450052602" type="string" offset="0"/>
+ </magic>
+ </mime-type>
+
<mime-type type="application/x-pds">
<_comment>Planetary Data System Version 3 format</_comment>
<tika:link>http://fileformats.archiveteam.org/wiki/PDS</tika:link>
@@ -5778,6 +5797,15 @@
<glob pattern="*.qcow2"/>
</mime-type>
+ <mime-type type="application/x-qt-translation">
+ <_comment>Qt compiled translation</_comment>
+ <!-- the leading '<' would otherwise read as HTML -->
+ <magic priority="60">
+ <match value="0x3cb86418caef9c95cd211cbf60a1bddd" type="string"
offset="0"/>
+ </magic>
+ <glob pattern="*.qm"/>
+ </mime-type>
+
<mime-type type="application/x-quattro-pro">
<_comment>
Quattro Pro - Corel Spreadsheet (part of WordPerfect Office suite)
@@ -6007,6 +6035,13 @@
<sub-class-of type="application/zip"/>
</mime-type>
+ <mime-type type="application/x-selinux-policy-package">
+ <_comment>SELinux policy module package</_comment>
+ <magic priority="50">
+ <match value="0x8fff7cf9" type="string" offset="0"/>
+ </magic>
+ </mime-type>
+
<mime-type type="application/x-sh">
<_comment>UNIX/LINUX Shell Script</_comment>
<magic priority="50">
diff --git a/tika-core/src/test/java/org/apache/tika/TikaDetectionTest.java
b/tika-core/src/test/java/org/apache/tika/TikaDetectionTest.java
index 517f32fd26..c09000a0c7 100644
--- a/tika-core/src/test/java/org/apache/tika/TikaDetectionTest.java
+++ b/tika-core/src/test/java/org/apache/tika/TikaDetectionTest.java
@@ -88,7 +88,7 @@ public class TikaDetectionTest {
assertEquals("application/octet-stream", tika.detect("x.pkg"));
assertEquals("application/octet-stream", tika.detect("x.bpk"));
assertEquals("application/octet-stream", tika.detect("x.dump"));
- assertEquals("application/octet-stream", tika.detect("x.elc"));
+ assertEquals("application/x-elc", tika.detect("x.elc"));
assertEquals("application/octet-stream", tika.detect("x.deploy"));
assertEquals("application/oda", tika.detect("x.oda"));
assertEquals("application/oebps-package+xml", tika.detect("x.opf"));
diff --git
a/tika-core/src/test/java/org/apache/tika/mime/MimeDetectionTest.java
b/tika-core/src/test/java/org/apache/tika/mime/MimeDetectionTest.java
index 3c9d3a3e11..0800216d86 100644
--- a/tika-core/src/test/java/org/apache/tika/mime/MimeDetectionTest.java
+++ b/tika-core/src/test/java/org/apache/tika/mime/MimeDetectionTest.java
@@ -447,6 +447,134 @@ public class MimeDetectionTest {
}
}
+ @Test
+ public void testBinaryContentRefusesTextGlob() throws IOException {
+ // headers copied from real files that a text glob used to claim
+ String[][] cases = {
+ // Terminal Reality POD archive
+ {"Bulldog.pod",
"2f000000436f6d70696c65642062792042696e456469742056657273696f6e20"
+ +
"322e322e38207da62d2900000000000000000000000000000000000000000000"
+ +
"00000000000000000000000000000000000000004152545c4241545453352e41"},
+ // Mozilla startupCache XDR bytecode, named for its source
+ {"AccountManager.js",
+
"0000001c000000b9000000c5fcc6bf73b9000000000000000000000000000000"
+ +
"0081030000f0000000b900000031000000c1030000000000003b000000000000"
+ +
"0000000000000000000000000022000000000000000010000000010000016368"},
+ {"overlay.xul",
"42c073b9000000002306000096000000b9000000720000002b02000000000000"
+ +
"360000000000000000000000000000005d000000000000000430000001010000"
+ +
"016368726f6d653a2f2f676c6f62616c2f636f6e74656e742f42726f77736572"},
+ // CODESYS project .meta
+ {"1f876365.meta",
"02200928000000000000000000000000220000000f01bf0df704c00d0180060f"
+ + "070504f4010000c10df904000a00e70c236f3456d208"},
+ // BM-E record
+ {"10.T",
"424d2d4564312e332d574b0031ff0d1a35460bb1000600000048000600000051"
+ +
"0007006f00b800080075005b000b00080077000d006f00d7000e0075001f000e"
+ +
"000000270117007500080019007500c3001900750005001a0075004a001b0000"}};
+ for (String[] c : cases) {
+ byte[] bytes = hex(c[1]);
+ assertEquals(MediaType.OCTET_STREAM, detect(bytes, c[0]), c[0]);
+ Metadata metadata = new Metadata();
+ metadata.set(HttpHeaders.CONTENT_TYPE, "text/plain");
+ try (TikaInputStream tis = TikaInputStream.get(bytes)) {
+ assertEquals(MediaType.OCTET_STREAM,
+ MIME_TYPES.detect(tis, metadata, new ParseContext()),
c[0]);
+ }
+ }
+ }
+
+ @Test
+ public void testTextGlobStillWinsForText() throws IOException {
+ String latin = "The quick brown fox jumps over the lazy dog.\n";
+ // CJK whose code units have low bytes under 0x20: NULs and controls
in both columns
+ String cjk = "一丁七万三上下不与专 且"
+ + "世业东两 abc\n";
+ String russian = "Съешь же ещё "
+ + "этих мягких "
+ + "французских.\n";
+ MediaType log = MediaType.text("x-log");
+ assertEquals(MediaType.TEXT_PLAIN,
detect(latin.repeat(20).getBytes(UTF_16LE), "a.txt"));
+ assertEquals(MediaType.TEXT_PLAIN,
detect(latin.repeat(20).getBytes(UTF_16BE), "a.txt"));
+ assertEquals(MediaType.TEXT_PLAIN,
detect(cjk.repeat(20).getBytes(UTF_16LE), "a.txt"));
+ assertEquals(MediaType.TEXT_PLAIN,
detect(cjk.repeat(20).getBytes(UTF_16BE), "a.txt"));
+ for (String utf32 : new String[] {"UTF-32LE", "UTF-32BE"}) {
+ assertEquals(MediaType.TEXT_PLAIN,
detect(latin.repeat(20).getBytes(utf32), "a.txt"), utf32);
+ assertEquals(MediaType.TEXT_PLAIN,
detect(cjk.repeat(20).getBytes(utf32), "a.txt"), utf32);
+ Metadata metadata = new Metadata();
+ metadata.set(HttpHeaders.CONTENT_TYPE, "text/plain");
+ try (TikaInputStream tis =
TikaInputStream.get(russian.repeat(20).getBytes(utf32))) {
+ assertEquals(MediaType.TEXT_PLAIN, MIME_TYPES.detect(tis,
metadata, new ParseContext()), utf32);
+ }
+ }
+ assertEquals(MediaType.TEXT_PLAIN,
+ detect(russian.repeat(20).getBytes("windows-1251"), "a.txt"));
+ assertEquals(log, detect(("\u001b[32mINFO\u001b[0m
started\n").repeat(40)
+ .getBytes(US_ASCII), "server.log"));
+ // a stray control byte per line stays under the binary threshold
+ assertEquals(log, detect(("2026-11-07 12:00:00 ok\u0000\n").repeat(40)
+ .getBytes(US_ASCII), "server.log"));
+ // zero fill after the text, as a crashed or preallocated log leaves
+ byte[] text = "[Mon Oct 26 03:26:03 2015] [notice] SIGHUP
received.\n".repeat(18)
+ .getBytes(US_ASCII);
+ byte[] zeroFilled = java.util.Arrays.copyOf(text, 8192);
+ assertEquals(log, detect(zeroFilled, "error_log.log"));
+ assertEquals(MediaType.TEXT_PLAIN, detect(zeroFilled, null));
+ }
+
+ @Test
+ public void testUtf8BomBeforeBinaryMagic() throws IOException {
+ String bom = "efbbbf";
+ assertEquals(MediaType.image("jpeg"),
+ detect(hex(bom +
"ffd8ffe000104a46494600010101004800480000ffdb004300080606"), null));
+ assertEquals(MediaType.image("png"),
+ detect(hex(bom +
"89504e470d0a1a0a0000000d4948445200000710000004b008060000"), null));
+ assertEquals(MediaType.audio("mpeg"),
+ detect(hex(bom +
"49443303000000001f76434f4d4d0000001e000000646575007777"), null));
+ assertEquals(MediaType.image("gif"),
+ detect(hex(bom +
"4749463839610a000a00f70000000000800000008000808000000080"), null));
+ assertEquals(MediaType.application("zip"),
+ detect(hex(bom +
"504b03041400000008001c3f364d479244b25801000000a3d4b2"), null));
+ // text that merely starts with a magic string stays text
+ assertEquals(MediaType.TEXT_PLAIN,
+ detect("ID3 tags hold the title and
artist.\n".getBytes(UTF_8), null));
+ assertEquals(MediaType.TEXT_PLAIN,
+ detect("GIF89a is the animated variant.\n".getBytes(UTF_8),
"notes.txt"));
+ }
+
+ @Test
+ public void testTextHeaderOnBinaryBodyIsNotText() throws IOException {
+ // BSB/KAP chart: text header lines, then 0x1A 0x00 and raster bytes
+ StringBuilder header = new
StringBuilder("VER/2.0\r\nBSB/NA=Mallorca,NU=1\r\n");
+ for (int i = 0; header.length() < 1900; i++) {
+ header.append("RGB/").append(i).append(',').append(i * 7 %
256).append(',')
+ .append(i * 13 % 256).append(',').append(i * 3 %
256).append("\r\n");
+ }
+ byte[] text = header.toString().getBytes(US_ASCII);
+ byte[] raster = new byte[4000];
+ new java.util.Random(42).nextBytes(raster);
+ byte[] kap = new byte[text.length + 2 + raster.length];
+ System.arraycopy(text, 0, kap, 0, text.length);
+ kap[text.length] = 0x1A;
+ System.arraycopy(raster, 0, kap, text.length + 2, raster.length);
+ org.apache.tika.detect.Detector detector =
+ new org.apache.tika.detect.DefaultDetector(MIME_TYPES);
+ try (TikaInputStream tis = TikaInputStream.get(kap)) {
+ assertEquals(MediaType.OCTET_STREAM,
+ detector.detect(tis, new Metadata(), new ParseContext()));
+ }
+ try (TikaInputStream tis = TikaInputStream.get(text)) {
+ assertEquals(MediaType.TEXT_PLAIN,
+ detector.detect(tis, new Metadata(), new ParseContext()));
+ }
+ }
+
+ private static byte[] hex(String hex) {
+ byte[] bytes = new byte[hex.length() / 2];
+ for (int i = 0; i < bytes.length; i++) {
+ bytes[i] = (byte) Integer.parseInt(hex.substring(2 * i, 2 * i +
2), 16);
+ }
+ return bytes;
+ }
+
@Test
public void testXulByContent() throws IOException {
MediaType xul = MediaType.application("vnd.mozilla.xul+xml");
diff --git
a/tika-core/src/test/java/org/apache/tika/mime/NicheFormatDetectionTest.java
b/tika-core/src/test/java/org/apache/tika/mime/NicheFormatDetectionTest.java
index cac379a631..2c27de30fe 100644
--- a/tika-core/src/test/java/org/apache/tika/mime/NicheFormatDetectionTest.java
+++ b/tika-core/src/test/java/org/apache/tika/mime/NicheFormatDetectionTest.java
@@ -27,6 +27,7 @@ import org.junit.jupiter.api.Test;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.TikaCoreProperties;
import org.apache.tika.parser.ParseContext;
/**
@@ -340,6 +341,34 @@ public class NicheFormatDetectionTest {
}
}
+ @Test
+ public void testCompiledFormatsThatLookLikeText() throws Exception {
+ MediaType elc = MediaType.application("x-elc");
+ assertDetected(elc, ";ELC\u0013\u0000\u0000\u0000\n;;; Compiled by");
+ assertDetected(elc, ";ELC\u0017\u0000\u0000\u0000\n;;; Compiled",
"pymacs.elc");
+ assertDetected(MediaType.application("x-lua-bytecode"),
+
"\u001bLuaQ\u0000\u0001\u0004\u0004\u0004\b\u0000\u001a\u0000\u0000\u0000@light",
+ "gui.lua");
+ // the leading '<' used to read as HTML
+ assertDetected(MediaType.application("x-qt-translation"),
+
"<\u00b8d\u0018\u00ca\u00ef\u009c\u0095\u00cd!\u001c\u00bf`\u00a1\u00bd\u00dd"
+ + "B\u0000\u0000\u0000\u00a0\u0000\u0000\u00001",
"REDHAT-582476-0.tgz-167.html");
+ assertDetected(MediaType.application("x-selinux-policy-package"),
+
"\u008f\u00ff|\u00f9\u0001\u0000\u0000\u0000\u0001\u0000\u0000\u0000\u0010"
+ + "\u0000\u0000\u0000\u008d\u00ff|\u00f9\u000f",
"mypol-1.pp");
+ assertDetected(MediaType.application("x-pcp-archive"),
+
"\u0000\u0000\u0000\u0084P\u0005&\u0002\u0000\u0000\u0001\u0083VT\u0082\u00b6"
+ + "\u0000\u0000\u00fc%\u00ff\u00ff\u00ff\u00ff",
"pVZ3Sj.localhost.meta");
+ }
+
+ @Test
+ public void testFrameMakerBinaryIsNotMif() throws Exception {
+ MediaType frame = MediaType.application("vnd.framemaker");
+ assertDetected(frame, "<MakerFile 5.5Q>\r\n\u001a\u0000\u0000\b\t\n",
"TIKA-1898-0.mif");
+ assertDetected(frame, "<MakerFile 7.0H>\r\n\u001a\u0000\u0000\b\t\n");
+ assertDetected(MediaType.application("vnd.mif"), "<MIFFile 5.50> #
Generated by FrameMaker");
+ }
+
@Test
public void testSharedExtensionsAreNotClaimed() throws Exception {
// .atr Atari disks, .bsp Quake/GoldSrc maps, .lev other games,
.flipchart Wacom
@@ -354,11 +383,20 @@ public class NicheFormatDetectionTest {
}
private static void assertDetected(MediaType expected, String latin1)
throws IOException {
+ assertDetected(expected, latin1, null);
+ }
+
+ private static void assertDetected(MediaType expected, String latin1,
String name)
+ throws IOException {
byte[] bytes = new byte[64];
byte[] header = latin1.getBytes(ISO_8859_1);
System.arraycopy(header, 0, bytes, 0, header.length);
+ Metadata metadata = new Metadata();
+ if (name != null) {
+ metadata.set(TikaCoreProperties.RESOURCE_NAME_KEY, name);
+ }
try (TikaInputStream tis = TikaInputStream.get(bytes)) {
- assertEquals(expected, MIME_TYPES.detect(tis, new Metadata(), new
ParseContext()));
+ assertEquals(expected, MIME_TYPES.detect(tis, metadata, new
ParseContext()));
}
}
}
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-integration-tests/src/test/java/org/apache/tika/mime/TestMimeTypes.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-integration-tests/src/test/java/org/apache/tika/mime/TestMimeTypes.java
index e6ee585b45..f24d79b8d6 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-integration-tests/src/test/java/org/apache/tika/mime/TestMimeTypes.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-integration-tests/src/test/java/org/apache/tika/mime/TestMimeTypes.java
@@ -1280,8 +1280,11 @@ public class TestMimeTypes {
@Test
public void testMIF() throws Exception {
- assertType("application/vnd.mif", "testMIF.mif");
- assertTypeByData("application/vnd.mif", "testMIF.mif");
+ // a binary FrameMaker document (<MakerFile), not the MIF text format
+ assertType("application/vnd.framemaker", "testMIF.mif");
+ assertTypeByData("application/vnd.framemaker", "testMIF.mif");
+ assertType("application/vnd.mif", "testFramemakerMif.mif");
+ assertTypeByData("application/vnd.mif", "testFramemakerMif.mif");
}
@Test
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-integration-tests/src/test/java/org/apache/tika/parser/pkg/ZipParserTest.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-integration-tests/src/test/java/org/apache/tika/parser/pkg/ZipParserTest.java
index a3b7f56902..af6613128d 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-integration-tests/src/test/java/org/apache/tika/parser/pkg/ZipParserTest.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-integration-tests/src/test/java/org/apache/tika/parser/pkg/ZipParserTest.java
@@ -142,9 +142,8 @@ public class ZipParserTest extends AbstractPkgTest {
//mime is 0 bytes
assertContains("InputStream must have > 0 bytes",
results.get(1).get("tk:exception:embedded-exception"));
- //source.xml is binary, not xml
- assertContains("TikaException: XML parse error",
- results.get(2).get("tk:exception:embedded-exception"));
+ //source.xml is binary, not xml: the .xml name no longer wins
+ assertEquals("application/octet-stream",
results.get(2).get(HttpHeaders.CONTENT_TYPE));
//manifest.xml has malformed xml
assertContains("TikaException: XML parse error",
results.get(4).get("tk:exception:embedded-exception"));
