This is an automated email from the ASF dual-hosted git repository.
tballison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git
The following commit(s) were added to refs/heads/main by this push:
new 6470d7d1c5 TIKA-5029: detect RPM headers, pdf.js binary CMaps...
(#3599)
6470d7d1c5 is described below
commit 6470d7d1c5b2cb244064ca1705e8edcc64b930db
Author: Tim Allison <[email protected]>
AuthorDate: Thu Nov 12 14:03:09 2026 -0500
TIKA-5029: detect RPM headers, pdf.js binary CMaps... (#3599)
Across Lite crosswords, C64 tape images, OpenOffice.org registry caches and
LibreOffice profile packs by content. LibreOffice pack files with two or three
entries are no longer detected as git packs.
Co-authored-by: Claude Opus 5.5 <[email protected]>
---
CHANGES.txt | 6 ++
.../org/apache/tika/mime/tika-mimetypes.xml | 76 ++++++++++++++++++++++
.../apache/tika/mime/NicheFormatDetectionTest.java | 49 +++++++++++++-
3 files changed, 130 insertions(+), 1 deletion(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index 4db7145361..0e94fe707e 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -113,6 +113,12 @@ Release 4.3.0 - ???
* ASX playlists (application/x-ms-asx) are parsed by JSoupParser; XMLParser
failed on
many real ones (mixed-case or unclosed tags, raw '&', non-UTF-8 without a
declaration).
Texinfo files are parsed as text; they produced no content (TIKA-5029).
+
+ * Bare RPM headers (yum *.hdr files), pdf.js binary CMaps, Across Lite
crosswords
+ (application/x-crossword), Commodore 64 tape images
(application/x-commodore-tape),
+ OpenOffice.org 2.x/3.x registry cache files and LibreOffice user-profile
backup
+ (user/pack) files are detected by content; they were
application/octet-stream, or
+ application/x-git-pack for a pack file with two or three entries
(TIKA-5029).
* ISO-2022-JP is decoded as x-windows-50221, so NEC/IBM extension
characters (circled
numbers, Roman numerals) no longer become U+FFFD. ISO-8859-1, ISO-8859-9,
TIS-620 and
diff --git
a/tika-core/src/main/resources/org/apache/tika/mime/tika-mimetypes.xml
b/tika-core/src/main/resources/org/apache/tika/mime/tika-mimetypes.xml
index f2f793daab..655da13968 100644
--- a/tika-core/src/main/resources/org/apache/tika/mime/tika-mimetypes.xml
+++ b/tika-core/src/main/resources/org/apache/tika/mime/tika-mimetypes.xml
@@ -4398,6 +4398,34 @@
<sub-class-of type="application/x-tika-msoffice"/>
</mime-type>
+ <mime-type type="application/x-commodore-tape">
+ <_comment>Commodore 64 tape image</_comment>
+ <magic priority="50">
+ <match value="C64-TAPE-RAW" type="string" offset="0"/>
+ <!-- T64: then the version, 0x0100, 0x0101 or 0x0200 little-endian -->
+ <match value="C64 tape image file" type="string" offset="0">
+ <match value="0x0001" type="string" offset="32"/>
+ <match value="0x0101" type="string" offset="32"/>
+ <match value="0x0002" type="string" offset="32"/>
+ </match>
+ <match value="C64S tape" type="string" offset="0">
+ <match value="0x0001" type="string" offset="32"/>
+ <match value="0x0101" type="string" offset="32"/>
+ <match value="0x0002" type="string" offset="32"/>
+ </match>
+ </magic>
+ <glob pattern="*.t64"/>
+ </mime-type>
+
+ <mime-type type="application/x-crossword">
+ <_comment>Across Lite crossword puzzle</_comment>
+ <magic priority="50">
+ <!-- after a 2-byte checksum -->
+ <match value="ACROSS&DOWN\x00" type="string" offset="2"/>
+ </magic>
+ <glob pattern="*.puz"/>
+ </mime-type>
+
<mime-type type="application/x-cpio">
<_comment>UNIX CPIO Archive</_comment>
<magic priority="50">
@@ -4591,6 +4619,15 @@
</match>
</magic>
</mime-type>
+ <mime-type type="application/x-openoffice-registry-cache">
+ <_comment>OpenOffice.org 2.x/3.x configuration registry binary cache
(user/registry/cache)</_comment>
+ <magic priority="50">
+ <!-- then the length-prefixed build id, as in 320m12(Build:9483) -->
+ <match value="\x27\x11\x00\x03\x80\x00\x00" type="string" offset="0">
+ <match value="(Build:" type="string" offset="8:40"/>
+ </match>
+ </magic>
+ </mime-type>
<mime-type type="application/x-openscad">
<tika:link>https://openscad.org/index.html</tika:link>
<glob pattern="*.scad"/>
@@ -5076,6 +5113,29 @@
</magic>
</mime-type>
+ <mime-type type="application/x-libreoffice-pack">
+ <_comment>LibreOffice user profile backup (user/pack)</_comment>
+ <magic priority="50">
+ <!-- entry count, then a full size, CRC-32 and packed size per entry;
git packs have version 2 or 3 here -->
+ <match value="PACK\x00\x00\x00\x01" type="string" offset="0"/>
+ </magic>
+ <magic priority="60">
+ <!-- 2 or 3 entries: the first entry's zlib stream follows the entry
table -->
+ <match value="PACK\x00\x00\x00\x02" type="string" offset="0">
+ <match value="0x7801" type="string" offset="32"/>
+ <match value="0x785e" type="string" offset="32"/>
+ <match value="0x789c" type="string" offset="32"/>
+ <match value="0x78da" type="string" offset="32"/>
+ </match>
+ <match value="PACK\x00\x00\x00\x03" type="string" offset="0">
+ <match value="0x7801" type="string" offset="44"/>
+ <match value="0x785e" type="string" offset="44"/>
+ <match value="0x789c" type="string" offset="44"/>
+ <match value="0x78da" type="string" offset="44"/>
+ </match>
+ </magic>
+ </mime-type>
+
<mime-type type="application/x-ligo-frame">
<_comment>LIGO/Virgo gravitational-wave frame file</_comment>
<magic priority="50">
@@ -5683,6 +5743,15 @@
</magic>
</mime-type>
+ <mime-type type="application/x-pdfjs-bcmap">
+ <_comment>pdf.js binary CMap</_comment>
+ <magic priority="50">
+ <!-- after the flags byte: a comment record and the Adobe cmap-resources
copyright -->
+ <match value="\xe0RCopyright 1990-2009 Adobe Systems" type="string"
offset="1"/>
+ </magic>
+ <glob pattern="*.bcmap"/>
+ </mime-type>
+
<mime-type type="application/x-pds">
<_comment>Planetary Data System Version 3 format</_comment>
<tika:link>http://fileformats.archiveteam.org/wiki/PDS</tika:link>
@@ -5915,6 +5984,13 @@
<match value="\xed\xab\xee\xdb" type="string" offset="0"/>
</magic>
</mime-type>
+ <mime-type type="application/x-rpm-header">
+ <_comment>RPM header without the lead (yum headers/*.hdr, rpmdb
entries)</_comment>
+ <magic priority="50">
+ <!-- header magic, version 1, then 4 reserved zero bytes -->
+ <match value="\x8e\xad\xe8\x01\x00\x00\x00\x00" type="string"
offset="0"/>
+ </magic>
+ </mime-type>
<mime-type type="application/x-sas">
<_comment>SAS Program</_comment>
diff --git
a/tika-core/src/test/java/org/apache/tika/mime/NicheFormatDetectionTest.java
b/tika-core/src/test/java/org/apache/tika/mime/NicheFormatDetectionTest.java
index 2c27de30fe..5cb63082ba 100644
--- a/tika-core/src/test/java/org/apache/tika/mime/NicheFormatDetectionTest.java
+++ b/tika-core/src/test/java/org/apache/tika/mime/NicheFormatDetectionTest.java
@@ -269,7 +269,54 @@ public class NicheFormatDetectionTest {
assertDetected(MediaType.application("x-python-pickle"),
"\u0080\u0004\u0095\u0010\u0000\u0000\u0000");
assertDetected(MediaType.application("x-lucene-index"),
"?×l\u0017\u0013Lucene62SegmentInfo");
assertDetected(MediaType.application("x-git-pack"),
"PACK\u0000\u0000\u0000\u0002\u0000\u0000\u0000\u0003");
- assertDetected(MediaType.OCTET_STREAM,
"PACK\u0000\u0000\u0000\u0001\u0000\u0000\u0002-");
+ assertDetected(MediaType.application("x-libreoffice-pack"),
+
"PACK\u0000\u0000\u0000\u0001\u0000\u0000\u0002!\u000e\u009dEb\u0000\u0000\u0001\u0017xÚ");
+ // two entries: the zlib stream after the 24-byte entry table tells it
from a git pack
+ assertDetected(MediaType.application("x-libreoffice-pack"),
+
"PACK\u0000\u0000\u0000\u0002\u0000\u0000(\u0092\u008aSø¸\u0000\u0000\u0005\u0003"
+ +
"\u0000\u0000\b{\u0088\u009eaÈ\u0000\u0000\u0001ÑxÚÅ\u009a");
+ assertDetected(MediaType.application("x-git-pack"),
+
"PACK\u0000\u0000\u0000\u0002\u0000\u0000\u0000:\u009b\u000fx\u009c\u009dËKj");
+ assertDetected(MediaType.OCTET_STREAM,
"PACK\u0000\u0000\u0000\u0000\u0000\u0000\u0002!");
+ }
+
+ @Test
+ public void testOfficeProfileCaches() throws Exception {
+ MediaType cache = MediaType.application("x-openoffice-registry-cache");
+ assertDetected(cache,
"'\u0011\u0000\u0003\u0080\u0000\u0000\u0012320m12(Build:9483)\u0000\u0000\u0000\u0001");
+ assertDetected(cache,
"'\u0011\u0000\u0003\u0080\u0000\u0000\u0011300m15(Build:9379)\u0000\u0000\u0000\u0001");
+ assertDetected(MediaType.OCTET_STREAM,
"'\u0011\u0000\u0003\u0080\u0000\u0000\u0012320m12 Build 9483");
+ }
+
+ @Test
+ public void testRpmHeader() throws Exception {
+ assertDetected(MediaType.application("x-rpm-header"),
+
"\u008eè\u0001\u0000\u0000\u0000\u0000\u0000\u0000\u00009\u0000\u0000T\u0080");
+ assertDetected(MediaType.OCTET_STREAM,
"\u008eè\u0001\u0001\u0000\u0000\u0000\u0000\u0000\u00009");
+ assertDetected(MediaType.application("x-rpm"),
"í«îÛ\u0003\u0000\u0000\u0000");
+ }
+
+ @Test
+ public void testPdfJsBinaryCMap() throws Exception {
+ MediaType bcmap = MediaType.application("x-pdfjs-bcmap");
+ String comment = "àRCopyright 1990-2009 Adobe Systems
Incorporated.\nAll rights";
+ // flags byte: bit 0 vertical, bits 1-2 the CMap type
+ assertDetected(bcmap, "\u0002" + comment);
+ assertDetected(bcmap, "\u0003" + comment);
+ assertDetected(bcmap, "\u0004" + comment);
+ }
+
+ @Test
+ public void testHobbyFormats() throws Exception {
+ MediaType puz = MediaType.application("x-crossword");
+ assertDetected(puz,
"\u000e\u0000ACROSS&DOWN\u0000\u0002ºKÙá\u0083û\\æE1.2\u0000\u0000");
+ assertDetected(puz,
"=EACROSS&DOWN\u0000\u0002ªKÂ3\u0083ë\u0001ÇQ1.3\u0000\u0000");
+ assertDetected(MediaType.OCTET_STREAM, "ACROSS&DOWN\u0000\u0002ºKÙá");
+ MediaType tape = MediaType.application("x-commodore-tape");
+ assertDetected(tape, "C64 tape image file" + "\u0000".repeat(13) +
"\u0001\u0001\u001e\u0000\u0001\u0000");
+ assertDetected(tape, "C64S tape file\r\nDemo
tape\u001a......\u0000\u0001\u001e\u0000\u0001\u0000");
+ assertDetected(tape, "C64-TAPE-RAW\u0001\u0000\u0000\u0000");
+ assertDetected(MediaType.OCTET_STREAM, "C64 tape image file" +
"\u0000".repeat(13) + "\t\t");
}
@Test