This is an automated email from the ASF dual-hosted git repository.
tballison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git
The following commit(s) were added to refs/heads/main by this push:
new 0f6a001857 TIKA-4937: detection follow-ups from the ICOParser review
(#3303)
0f6a001857 is described below
commit 0f6a0018571ccdb49c3af49ba33bff9eec78b531
Author: Tim Allison <[email protected]>
AuthorDate: Wed Oct 7 14:13:32 2026 -0400
TIKA-4937: detection follow-ups from the ICOParser review (#3303)
* TIKA-4937: detection follow-ups from the ICOParser review
* TIKA-4937: tighten TGA and cursor rules after review
---
CHANGES.txt | 8 +-
.../org/apache/tika/mime/tika-mimetypes.xml | 79 ++++++++++++++----
.../org/apache/tika/mime/MimeDetectionTest.java | 93 ++++++++++++++++++++++
.../org/apache/tika/parser/image/ICOParser.java | 13 +--
.../apache/tika/parser/image/ICOParserTest.java | 16 ++--
.../apache/tika/config/loader/InferenceLoader.java | 2 +-
.../tika/config/loader/InferenceLoaderTest.java | 3 +
7 files changed, 172 insertions(+), 42 deletions(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index b73770dcf8..ba87b8d77f 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -66,11 +66,9 @@ Release 4.2.0 - unreleased
SERVER/RESOURCE_TIMING) move from INFO to TRACE, and per-fork-start,
per-connection and plugin lifecycle lines move to DEBUG (TIKA-4949).
- * New ICOParser reads Windows icons and cursors: dimensions and colour
- depth of the largest image, icon:image-count, icon:images and, for
- cursors, icon:hotspot-x/-y. Cursors are detected as the new type
- image/x-win-bitmap (*.cur). Compat: image/vnd.microsoft.icon moves
- from ImageParser to ICOParser (TIKA-4937).
+ * New ICOParser for Windows icons and cursors (image/x-win-bitmap, *.cur).
+ Compat: image/vnd.microsoft.icon moves from ImageParser to ICOParser.
+ Tighter OS/2 bitmap array and TGA detection (TIKA-4937).
* ExecutableParser extracts the icon groups of PE files (EXE/DLL) as
embedded .ico documents, the first one typed THUMBNAIL. Compat: this is
diff --git
a/tika-core/src/main/resources/org/apache/tika/mime/tika-mimetypes.xml
b/tika-core/src/main/resources/org/apache/tika/mime/tika-mimetypes.xml
index f1399767c7..e39ed9036a 100644
--- a/tika-core/src/main/resources/org/apache/tika/mime/tika-mimetypes.xml
+++ b/tika-core/src/main/resources/org/apache/tika/mime/tika-mimetypes.xml
@@ -3391,20 +3391,71 @@
<mime-type type="image/x-tga">
<alias type="image/x-targa"/>
- <!-- trailer bytes: 54 52 55 45 56 49 53 49 4F 4E 2D 58 46 49 4C 45 2E 00
- trailer as string: TRUEVISION-XFILE\\x2E\\x00
- Some .tga files may be conflicting with application/x-123
recognition,
- therefore this mime-type must be set in front of application/x-123
-->
<_comment>Targa image data</_comment>
- <magic priority="90">
- <match value="0x01010000" type="big32" offset="1" >
- <match
value=".*[\\x54\\x52\\x55\\x45\\x56\\x49\\x53\\x49\\x4F\\x4E\\x2D\\x58\\x46\\x49\\x4C\\x45\\x2E\\x00]"
type="regex" offset="8" />
+ <!-- No signature: colour map type and image type at 1, colour map spec
at 3-7,
+ pixel depth at 16. Without a colour map the first index and length
(3-6) are
+ zero; the entry size at 7 is left free since some writers set it
anyway.
+ The optional TRUEVISION-XFILE footer is out of reach. -->
+ <magic priority="50">
+ <!-- true colour, greyscale, and their RLE forms: 2, 3, 10, 11 -->
+ <match value="\000\002\000\000\000\000" type="string" offset="1">
+ <match value="\010" type="string" offset="16"/>
+ <match value="\017" type="string" offset="16"/>
+ <match value="\020" type="string" offset="16"/>
+ <match value="\030" type="string" offset="16"/>
+ <match value="\040" type="string" offset="16"/>
</match>
- <match value="0x00020000" type="big32" offset="1" >
- <match
value=".*[\\x54\\x52\\x55\\x45\\x56\\x49\\x53\\x49\\x4F\\x4E\\x2D\\x58\\x46\\x49\\x4C\\x45\\x2E\\x00]"
type="regex" offset="8" />
+ <match value="\000\003\000\000\000\000" type="string" offset="1">
+ <match value="\010" type="string" offset="16"/>
+ <match value="\020" type="string" offset="16"/>
</match>
- <match value="0x00030000" type="big32" offset="1" >
- <match
value=".*[\\x54\\x52\\x55\\x45\\x56\\x49\\x53\\x49\\x4F\\x4E\\x2D\\x58\\x46\\x49\\x4C\\x45\\x2E\\x00]"
type="regex" offset="8" />
+ <match value="\000\012\000\000\000\000" type="string" offset="1">
+ <match value="\010" type="string" offset="16"/>
+ <match value="\017" type="string" offset="16"/>
+ <match value="\020" type="string" offset="16"/>
+ <match value="\030" type="string" offset="16"/>
+ <match value="\040" type="string" offset="16"/>
+ </match>
+ <match value="\000\013\000\000\000\000" type="string" offset="1">
+ <match value="\010" type="string" offset="16"/>
+ <match value="\020" type="string" offset="16"/>
+ </match>
+ <!-- colour mapped and RLE colour mapped: 1, 9; entry size at 7, index
depth at 16 -->
+ <match value="\001\001" type="string" offset="1">
+ <match value="\017" type="string" offset="7">
+ <match value="\010" type="string" offset="16"/>
+ <match value="\020" type="string" offset="16"/>
+ </match>
+ <match value="\020" type="string" offset="7">
+ <match value="\010" type="string" offset="16"/>
+ <match value="\020" type="string" offset="16"/>
+ </match>
+ <match value="\030" type="string" offset="7">
+ <match value="\010" type="string" offset="16"/>
+ <match value="\020" type="string" offset="16"/>
+ </match>
+ <match value="\040" type="string" offset="7">
+ <match value="\010" type="string" offset="16"/>
+ <match value="\020" type="string" offset="16"/>
+ </match>
+ </match>
+ <match value="\001\011" type="string" offset="1">
+ <match value="\017" type="string" offset="7">
+ <match value="\010" type="string" offset="16"/>
+ <match value="\020" type="string" offset="16"/>
+ </match>
+ <match value="\020" type="string" offset="7">
+ <match value="\010" type="string" offset="16"/>
+ <match value="\020" type="string" offset="16"/>
+ </match>
+ <match value="\030" type="string" offset="7">
+ <match value="\010" type="string" offset="16"/>
+ <match value="\020" type="string" offset="16"/>
+ </match>
+ <match value="\040" type="string" offset="7">
+ <match value="\010" type="string" offset="16"/>
+ <match value="\020" type="string" offset="16"/>
+ </match>
</match>
</magic>
<glob pattern="*.tga"/>
@@ -6893,8 +6944,6 @@
<tika:uti>com.microsoft.ico</tika:uti>
<alias type="image/x-icon" />
<magic priority="50">
- <match value="\102\101\050\000\000\000\056\000\000\000\000\000\000\000"
- type="string" offset="0"/>
<match value="\000\000\001\000" type="string" offset="0"/>
</magic>
<glob pattern="*.ico"/>
@@ -6905,9 +6954,9 @@
<_comment>Windows Cursor</_comment>
<tika:link>https://en.wikipedia.org/wiki/ICO_(file_format)</tika:link>
<magic priority="50">
- <!-- reserved 0, type 2, count below 256 -->
+ <!-- reserved 0, type 2, 1 to 255 images; a count of 0 would also match
a TGA header -->
<match value="\000\000\002\000" type="string" offset="0">
- <match value="\000" type="string" offset="5"/>
+ <match value="[\\x01-\\xff]\\x00" type="regex" offset="4"/>
</match>
</magic>
<glob pattern="*.cur"/>
diff --git
a/tika-core/src/test/java/org/apache/tika/mime/MimeDetectionTest.java
b/tika-core/src/test/java/org/apache/tika/mime/MimeDetectionTest.java
index a3794f3184..d9540190e4 100644
--- a/tika-core/src/test/java/org/apache/tika/mime/MimeDetectionTest.java
+++ b/tika-core/src/test/java/org/apache/tika/mime/MimeDetectionTest.java
@@ -93,6 +93,99 @@ public class MimeDetectionTest {
testFile("application/vnd.android.axml", "test-android-binary.xml");
}
+ /** An OS/2 bitmap array's first entry header (size 40, next at 46) is not
an icon. */
+ @Test
+ public void testOs2BitmapArrayIsNotAnIcon() throws Exception {
+ for (String tag : new String[]{"BM", "IC", "CI", "CP", "PT"}) {
+ byte[] bitmapArray = new byte[64];
+ bitmapArray[0] = 'B';
+ bitmapArray[1] = 'A';
+ bitmapArray[2] = 0x28;
+ bitmapArray[6] = 0x2e;
+ bitmapArray[14] = (byte) tag.charAt(0);
+ bitmapArray[15] = (byte) tag.charAt(1);
+ try (TikaInputStream tis = TikaInputStream.get(bitmapArray)) {
+ assertEquals(MediaType.parse("image/x-os2-graphics;
charset=binary"),
+ MIME_TYPES.detect(tis, new Metadata(), new
ParseContext()), tag);
+ }
+ }
+ }
+
+ /**
+ * TGA has no signature. The rules key on the colour map type and image
+ * type, the colour map spec and the pixel depth.
+ */
+ @Test
+ public void testTgaDetection() throws Exception {
+ MediaType tga = MediaType.image("x-tga");
+ MediaType cursor = MediaType.image("x-win-bitmap");
+ // true colour, no id, no colour map, 24 bpp
+ assertDetected(tga, tgaHeader(0, 0, 2, 0, 24));
+ // the same with an id field and 32 bpp, and the RLE form
+ assertDetected(tga, tgaHeader(5, 0, 2, 0, 32));
+ assertDetected(tga, tgaHeader(0, 0, 10, 0, 32));
+ // 15 bpp, with and without an id
+ assertDetected(tga, tgaHeader(0, 0, 2, 0, 15));
+ assertDetected(tga, tgaHeader(3, 0, 2, 0, 15));
+ // greyscale
+ assertDetected(tga, tgaHeader(0, 0, 3, 0, 8));
+ assertDetected(tga, tgaHeader(0, 0, 11, 0, 8));
+ // colour mapped, 24 bit entries, 8 bit indices, and the RLE form
+ assertDetected(tga, tgaHeader(0, 1, 1, 24, 8));
+ assertDetected(tga, tgaHeader(0, 1, 9, 24, 8));
+ // no colour map but an entry size in the spec, as some writers leave
it
+ byte[] strayEntrySize = tgaHeader(0, 0, 2, 0, 24);
+ strayEntrySize[7] = 24;
+ assertDetected(tga, strayEntrySize);
+
+ // a pixel depth no TGA has
+ assertDetected(MediaType.OCTET_STREAM, tgaHeader(0, 0, 2, 0, 7));
+ assertDetected(MediaType.OCTET_STREAM, tgaHeader(1, 0, 2, 0, 7));
+ // a colour map entry size and an index depth no TGA has
+ assertDetected(MediaType.OCTET_STREAM, tgaHeader(0, 1, 1, 7, 8));
+ assertDetected(MediaType.OCTET_STREAM, tgaHeader(0, 1, 1, 24, 99));
+ // a Windows cursor with one image starts like a true colour TGA
+ byte[] oneImage = tgaHeader(0, 0, 2, 0, 24);
+ oneImage[4] = 1;
+ assertDetected(cursor, oneImage);
+ // a cursor with no images is nothing; 256 images is a TGA with a
colour map length
+ byte[] noImages = tgaHeader(0, 0, 2, 0, 24);
+ noImages[16] = 0;
+ assertDetected(MediaType.OCTET_STREAM, noImages);
+ byte[] images256 = tgaHeader(0, 0, 2, 0, 24);
+ images256[5] = 1;
+ assertDetected(MediaType.OCTET_STREAM, images256);
+ // Lotus 1-2-3 shares the first four bytes
+ byte[] lotus = new byte[64];
+ lotus[2] = 2;
+ lotus[4] = 4;
+ lotus[5] = 4;
+
assertDetected(MediaType.parse("application/vnd.lotus-1-2-3;version=1"), lotus);
+ }
+
+ private static byte[] tgaHeader(int idLength, int colourMapType, int
imageType,
+ int colourMapEntrySize, int pixelDepth) {
+ byte[] header = new byte[64];
+ header[0] = (byte) idLength;
+ header[1] = (byte) colourMapType;
+ header[2] = (byte) imageType;
+ if (colourMapType == 1) {
+ // 256 entries
+ header[6] = 1;
+ header[7] = (byte) colourMapEntrySize;
+ }
+ header[12] = 32;
+ header[14] = 32;
+ header[16] = (byte) pixelDepth;
+ return header;
+ }
+
+ private static void assertDetected(MediaType expected, byte[] bytes)
throws IOException {
+ try (TikaInputStream tis = TikaInputStream.get(bytes)) {
+ assertEquals(expected, MIME_TYPES.detect(tis, new Metadata(), new
ParseContext()));
+ }
+ }
+
@Test
public void testDetectionWithoutContent() throws IOException {
testUrlWithoutContent("text/html", "test.html");
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/ICOParser.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/ICOParser.java
index 3aabf1d73a..b144affd8c 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/ICOParser.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/ICOParser.java
@@ -50,9 +50,6 @@ import org.apache.tika.sax.XHTMLContentHandler;
* from each image's PNG IHDR or BITMAPINFOHEADER and the directory is only
* the fallback. Colour depth is reported the TIFF way, bits per sample and
* samples per pixel; the per-image list carries the total bits per pixel.
- * <p>
- * OS/2 bitmap arrays are detected as the same type. They pass through
- * without metadata.
*/
@TikaComponent
public class ICOParser implements Parser {
@@ -93,21 +90,13 @@ public class ICOParser implements Parser {
public void parse(TikaInputStream tis, ContentHandler handler, Metadata
metadata,
ParseContext context) throws IOException, SAXException,
TikaException {
byte[] file = IOUtils.toByteArray(new
BoundedInputStream(MAX_FILE_SIZE, tis));
- if (isOs2BitmapArray(file)) {
- metadata.set(HttpHeaders.CONTENT_TYPE, ICO_TYPE.toString());
- } else {
- extractMetadata(file, metadata, context);
- }
+ extractMetadata(file, metadata, context);
XHTMLContentHandler xhtml = new XHTMLContentHandler(handler, metadata,
context);
xhtml.startDocument();
xhtml.endDocument();
}
- private static boolean isOs2BitmapArray(byte[] file) {
- return file.length >= 2 && file[0] == 'B' && file[1] == 'A';
- }
-
private static void extractMetadata(byte[] file, Metadata metadata,
ParseContext context)
throws TikaException {
if (file.length < COUNT_OFFSET || EndianUtils.getUShortLE(file, 0) !=
0) {
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/test/java/org/apache/tika/parser/image/ICOParserTest.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/test/java/org/apache/tika/parser/image/ICOParserTest.java
index d9d8646674..cc92b79413 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/test/java/org/apache/tika/parser/image/ICOParserTest.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/test/java/org/apache/tika/parser/image/ICOParserTest.java
@@ -207,8 +207,8 @@ public class ICOParserTest extends TikaTest {
}
/**
- * OS/2 bitmap arrays share the icon type's magic. They are not read, but
- * they are not a failure either.
+ * An OS/2 bitmap array is not an icon: it is detected as its own type and
+ * never reaches this parser, which rejects it when called directly.
*/
@Test
public void testOs2BitmapArray() throws Exception {
@@ -217,19 +217,17 @@ public class ICOParserTest extends TikaTest {
bitmapArray[1] = 'A';
bitmapArray[2] = 0x28;
bitmapArray[6] = 0x2e;
+ bitmapArray[14] = 'I';
+ bitmapArray[15] = 'C';
- Metadata metadata = parse(bitmapArray);
- assertEquals("image/vnd.microsoft.icon",
metadata.get(HttpHeaders.CONTENT_TYPE));
- assertNull(metadata.get(Icon.IMAGE_COUNT));
- assertNull(metadata.get(TIFF.IMAGE_WIDTH));
+ assertThrows(TikaException.class, () -> parse(bitmapArray));
Metadata detected = new Metadata();
try (TikaInputStream tis = TikaInputStream.get(bitmapArray)) {
getXML(tis, AUTO_DETECT_PARSER, detected);
}
- assertEquals("image/vnd.microsoft.icon",
detected.get(HttpHeaders.CONTENT_TYPE));
- assertContains(ICOParser.class.getName(),
-
Arrays.asList(detected.getValues(TikaCoreProperties.TIKA_PARSED_BY)));
+ assertEquals("image/x-os2-graphics; charset=binary",
+ detected.get(HttpHeaders.CONTENT_TYPE));
assertNull(detected.get(TikaCoreProperties.CONTAINER_EXCEPTION));
}
diff --git
a/tika-serialization/src/main/java/org/apache/tika/config/loader/InferenceLoader.java
b/tika-serialization/src/main/java/org/apache/tika/config/loader/InferenceLoader.java
index cdf0870d1d..621a9984cc 100644
---
a/tika-serialization/src/main/java/org/apache/tika/config/loader/InferenceLoader.java
+++
b/tika-serialization/src/main/java/org/apache/tika/config/loader/InferenceLoader.java
@@ -59,7 +59,7 @@ class InferenceLoader implements
ComponentLoader<InferenceDispatcher> {
private static final List<String> NON_RASTER = List.of("image/svg+xml",
"image/vnd.dwg",
"image/vnd.dxf", "image/x-emf", "image/x-wmf", "image/wmf",
"image/emf",
"image/vnd.adobe.photoshop", "image/x-photoshop",
"image/vnd.microsoft.icon",
- "image/x-icon", "image/x-win-bitmap");
+ "image/x-icon", "image/x-win-bitmap", "image/x-os2-graphics");
@Override
public InferenceDispatcher load(TikaJsonConfig config, LoaderContext
context)
diff --git
a/tika-serialization/src/test/java/org/apache/tika/config/loader/InferenceLoaderTest.java
b/tika-serialization/src/test/java/org/apache/tika/config/loader/InferenceLoaderTest.java
index 82a9dc0cf9..76dd8aa60d 100644
---
a/tika-serialization/src/test/java/org/apache/tika/config/loader/InferenceLoaderTest.java
+++
b/tika-serialization/src/test/java/org/apache/tika/config/loader/InferenceLoaderTest.java
@@ -94,6 +94,9 @@ public class InferenceLoaderTest {
"without an include list, non-raster image types are excluded
by default");
assertFalse(second.binding().accepts(InputKind.IMAGES,
MediaType.image("x-win-bitmap")),
"a cursor holds images, it is not one");
+ assertFalse(second.binding().accepts(InputKind.IMAGES,
+ MediaType.parse("image/x-os2-graphics;
charset=binary")),
+ "an OS/2 bitmap array holds images, it is not one");
AutoDetectParser parser = (AutoDetectParser)
loader.loadAutoDetectParser();
assertEquals(List.of(dispatcher), parser.getParseHooks().getHooks(),
