This is an automated email from the ASF dual-hosted git repository. tallison pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/tika.git
commit cace0ca11330588c21e67476820d4f920034d99a Author: tallison <[email protected]> AuthorDate: Fri Dec 3 18:24:32 2021 -0500 TIKA-3526 -- improve extraction of attachments from MSOffice files created by non-MS tools --- CHANGES.txt | 3 ++ .../detect/microsoft/ooxml/OPCPackageDetector.java | 12 +++++- .../parser/microsoft/AbstractPOIFSExtractor.java | 8 ++-- .../tika/parser/microsoft/HSLFExtractor.java | 44 ++++++++++++++++++- .../microsoft/ooxml/AbstractOOXMLExtractor.java | 44 +++++++++++++------ .../microsoft/POIContainerExtractionTest.java | 48 +++++++++++++++++++-- .../resources/test-documents/testWPSAttachment.doc | Bin 0 -> 222720 bytes .../test-documents/testWPSAttachment.docx | Bin 0 -> 130189 bytes .../resources/test-documents/testWPSAttachment.ppt | Bin 0 -> 269312 bytes .../test-documents/testWPSAttachment.pptx | Bin 0 -> 151705 bytes .../resources/test-documents/testWPSAttachment.xls | Bin 0 -> 229888 bytes .../test-documents/testWPSAttachment.xlsx | Bin 0 -> 131498 bytes .../microsoft/POIContainerExtractionTest.java | 1 - 13 files changed, 135 insertions(+), 25 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index f3bcdf2..7348acc 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,5 +1,8 @@ Release 2.2.0 - ??? + * Improve extraction of embedded files from MSOffice files created + by non-Microsoft tools (TIKA-3526). + * Added back ability to ignore load errors in TikaConfig (TIKA-3575). * Make SecureContentHandler and other parameters configurable in diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/detect/microsoft/ooxml/OPCPackageDetector.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/detect/microsoft/ooxml/OPCPackageDetector.java index 64a39e2..dd58d10 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/detect/microsoft/ooxml/OPCPackageDetector.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/detect/microsoft/ooxml/OPCPackageDetector.java @@ -282,15 +282,23 @@ public class OPCPackageDetector implements ZipContainerDetector { detectContext.set(OOXMLHintCounter.class, cnt); } cnt.increment(); - if (cnt.getCount() > 2) { + //we used to short-circuit, but ooxml files that are + //not created by actual microsoft products do not tend to + //put [Content_Types.xml] as the first physical entry in the zip file + //so we need to read through the full file + /*if (cnt.getCount() > 2) { return TIKA_OOXML; - } + }*/ } return null; } @Override public MediaType streamingDetectFinal(StreamingDetectContext detectContext) { + OOXMLHintCounter cnt = detectContext.get(OOXMLHintCounter.class); + if (cnt != null && cnt.getCount() > 2) { + return TIKA_OOXML; + } return null; } diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/AbstractPOIFSExtractor.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/AbstractPOIFSExtractor.java index 19cc622..f4ee3e8 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/AbstractPOIFSExtractor.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/AbstractPOIFSExtractor.java @@ -157,12 +157,14 @@ abstract class AbstractPOIFSExtractor { XHTMLContentHandler xhtml) throws IOException, SAXException, TikaException { + // Is it an embedded OLE2 document, or an embedded OOXML document? + //first try for ooxml + Entry ooxml = dir.hasEntry("Package") ? dir.getEntry("Package") : + (dir.hasEntry("package") ? dir.getEntry("package") : null); - if (dir.hasEntry("Package")) { + if (ooxml != null) { // It's OOXML (has a ZipFile): - Entry ooxml = dir.getEntry("Package"); - try (TikaInputStream stream = TikaInputStream .get(new DocumentInputStream((DocumentEntry) ooxml))) { Detector detector = new DefaultZipContainerDetector(); diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/HSLFExtractor.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/HSLFExtractor.java index 2b20089..38de8f2 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/HSLFExtractor.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/HSLFExtractor.java @@ -19,7 +19,9 @@ package org.apache.tika.parser.microsoft; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; +import java.util.Set; import org.apache.commons.io.input.CloseShieldInputStream; import org.apache.poi.common.usermodel.Hyperlink; @@ -44,6 +46,7 @@ import org.apache.poi.hslf.usermodel.HSLFTextParagraph; import org.apache.poi.hslf.usermodel.HSLFTextRun; import org.apache.poi.hslf.usermodel.HSLFTextShape; import org.apache.poi.poifs.filesystem.DirectoryNode; +import org.apache.poi.poifs.filesystem.FileMagic; import org.apache.poi.poifs.filesystem.POIFSFileSystem; import org.apache.poi.sl.usermodel.Comment; import org.apache.poi.sl.usermodel.ShapeContainer; @@ -65,6 +68,7 @@ public class HSLFExtractor extends AbstractPOIFSExtractor { public HSLFExtractor(ParseContext context, Metadata metadata) { super(context, metadata); } + private final Set<Integer> processedEmbeddedObjects = new HashSet<>(); // remove trailing paragraph break private static String removePBreak(String fragment) { @@ -150,6 +154,8 @@ public class HSLFExtractor extends AbstractPOIFSExtractor { } handleSlideEmbeddedPictures(ss, xhtml); + handleShowEmbeddedResources(ss, xhtml); + if (officeParserConfig.isExtractMacros()) { extractMacros(ss, xhtml); } @@ -160,6 +166,39 @@ public class HSLFExtractor extends AbstractPOIFSExtractor { xhtml.endElement("div"); } + /** + * This is the catch-all for embedded objects. If we didn't come across + * them in the shapes in the slides, headers/footers, etc, try to + * extract them here. + **/ + private void handleShowEmbeddedResources(HSLFSlideShow ss, XHTMLContentHandler xhtml) + throws SAXException { + + HSLFObjectData[] objectData = ss.getEmbeddedObjects(); + + for (HSLFObjectData d : objectData) { + if (processedEmbeddedObjects.contains(d.getExOleObjStg().getPersistId())) { + continue; + } + try (TikaInputStream tis = TikaInputStream.get(d.getInputStream())) { + if (FileMagic.valueOf(tis) == FileMagic.OLE2) { + try (POIFSFileSystem pfs = new POIFSFileSystem(tis)) { + //coz ppts can have empty pfs...shrug... + //we may need to add more stringent requirements + if (pfs.getRoot().getEntryNames().size() < 1) { + return; + } + handleEmbeddedOfficeDoc(pfs.getRoot(), d.getFileName(), xhtml); + } + } else { + handleEmbeddedResource(tis, d.getFileName(), null, null, xhtml, true); + } + } catch (IOException | TikaException e) { + EmbeddedDocumentUtil.recordException(e, parentMetadata); + } + } + } + private void handleComments(HSLFSlide slide, XHTMLContentHandler xhtml) throws SAXException { if (slide.getComments() == null || slide.getComments().size() == 0) { return; @@ -469,7 +508,7 @@ public class HSLFExtractor extends AbstractPOIFSExtractor { private void handleSlideEmbeddedResources(ShapeContainer shapeContainer, XHTMLContentHandler xhtml) - throws TikaException, SAXException, IOException { + throws TikaException, SAXException { List<HSLFShape> shapes = getShapes(shapeContainer); if (shapes == null) { return; @@ -478,6 +517,7 @@ public class HSLFExtractor extends AbstractPOIFSExtractor { //handle ActiveXShape, movie shape? if (shape instanceof HSLFObjectShape) { HSLFObjectShape oleShape = (HSLFObjectShape) shape; + HSLFObjectData data = null; try { data = oleShape.getObjectData(); @@ -489,7 +529,7 @@ public class HSLFExtractor extends AbstractPOIFSExtractor { if (data != null) { String objID = Integer.toString(oleShape.getObjectID()); - + processedEmbeddedObjects.add(data.getExOleObjStg().getPersistId()); // Embedded Object: add a <div // class="embedded" id="X"/> so consumer can see where // in the main text each embedded document diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/AbstractOOXMLExtractor.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/AbstractOOXMLExtractor.java index f23ae12..e1a6598 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/AbstractOOXMLExtractor.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/AbstractOOXMLExtractor.java @@ -242,6 +242,7 @@ public abstract class AbstractOOXMLExtractor implements OOXMLExtractor { return; } } + URI sourceURI = rel.getSourceURI(); String sourceDesc; if (sourceURI != null) { @@ -269,16 +270,22 @@ public abstract class AbstractOOXMLExtractor implements OOXMLExtractor { if (POIXMLDocument.OLE_OBJECT_REL_TYPE.equals(type) && TYPE_OLE_OBJECT.equals(target.getContentType())) { handleEmbeddedOLE(target, handler, sourceDesc + rel.getId(), parentMetadata); - handledTarget.add(targetURI.toString()); + if (targetURI != null) { + handledTarget.add(targetURI.toString()); + } } else if (RELATION_MEDIA.equals(type) || RELATION_VIDEO.equals(type) || RELATION_AUDIO.equals(type) || PackageRelationshipTypes.IMAGE_PART.equals(type) || POIXMLDocument.PACK_OBJECT_REL_TYPE.equals(type) || POIXMLDocument.OLE_OBJECT_REL_TYPE.equals(type)) { handleEmbeddedFile(target, handler, sourceDesc + rel.getId()); - handledTarget.add(targetURI.toString()); + if (targetURI != null) { + handledTarget.add(targetURI.toString()); + } } else if (XSSFRelation.VBA_MACROS.getRelation().equals(type)) { handleMacros(target, handler); - handledTarget.add(targetURI.toString()); + if (targetURI != null) { + handledTarget.add(targetURI.toString()); + } } } @@ -311,21 +318,13 @@ public abstract class AbstractOOXMLExtractor implements OOXMLExtractor { DirectoryNode root = fs.getRoot(); POIFSDocumentType type = POIFSDocumentType.detectType(root); - if (root.hasEntry("\u0001Ole") && root.hasEntry("\u0001CompObj") && - (root.hasEntry("CONTENTS") || root.hasEntry("Package"))) { + String packageEntryName = getPackageEntryName(root); + if (packageEntryName != null) { // TIKA-704: OLE 2.0 embedded non-Office document? //TODO: figure out if the equivalent of OLE 1.0's //getCommand() and getFileName() exist for OLE 2.0 to populate //TikaCoreProperties.ORIGINAL_RESOURCE_NAME - if (root.hasEntry("CONTENTS")) { - stream = TikaInputStream.get(fs.createDocumentInputStream("CONTENTS")); - } else if (root.hasEntry("Package")) { - //TIKA-2588 - stream = TikaInputStream.get(fs.createDocumentInputStream("Package")); - } else { - throw new IllegalStateException( - "Shouldn't ever arrive here; please open a ticket on our jira"); - } + stream = TikaInputStream.get(fs.createDocumentInputStream(packageEntryName)); if (embeddedExtractor.shouldParseEmbedded(metadata)) { embeddedExtractor .parseEmbedded(stream, new EmbeddedContentHandler(handler), metadata, @@ -372,6 +371,23 @@ public abstract class AbstractOOXMLExtractor implements OOXMLExtractor { } } + private String getPackageEntryName(DirectoryNode root) { + if (root.hasEntry("\u0001Ole")) { + //we used to require this too: root.hasEntry("\u0001CompObj") before TIKA-3526 + if (root.hasEntry("Package")) { + return "Package"; + } else if (root.hasEntry("CONTENTS")) { + return "CONTENTS"; + } else if (root.hasEntry("package")) { + return "package"; + } + } + if (root.hasEntry("package")) { + return "package"; + } + return null; + } + /** * Handles an embedded file in the document */ diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/POIContainerExtractionTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/POIContainerExtractionTest.java index 55ec414..203bdac 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/POIContainerExtractionTest.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/POIContainerExtractionTest.java @@ -19,8 +19,11 @@ package org.apache.tika.parser.microsoft; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; +import java.util.HashSet; import java.util.List; +import java.util.Set; import org.junit.jupiter.api.Test; @@ -43,7 +46,7 @@ public class POIContainerExtractionTest extends AbstractPOIContainerExtractionTe @Test public void testWithoutEmbedded() throws Exception { ContainerExtractor extractor = new ParserContainerExtractor(); - + debug(getRecursiveMetadata("testPPT.ppt")); String[] files = new String[]{"testEXCEL.xls", "testWORD.doc", "testPPT.ppt", "testVISIO.vsd", "test-outlook.msg"}; @@ -52,8 +55,8 @@ public class POIContainerExtractionTest extends AbstractPOIContainerExtractionTe TrackingHandler handler = process(file, extractor, false); // Won't have fired - assertEquals(0, handler.filenames.size()); - assertEquals(0, handler.mediaTypes.size()); + assertEquals(0, handler.filenames.size(), file); + assertEquals(0, handler.mediaTypes.size(), file); // Ditto with recursing handler = process(file, extractor, true); @@ -168,4 +171,43 @@ public class POIContainerExtractionTest extends AbstractPOIContainerExtractionTe assertEquals(3, metadataList.size()); assertEquals("application/vnd.ms-equation", metadataList.get(2).get(Metadata.CONTENT_TYPE)); } + + @Test + public void testWPSVariantAttachments() throws Exception { + //test that files created by WPS have embedded files extracted + //TIKA-3526 + Set<String> expected = new HashSet<>(); + expected.add("application/msword"); + expected.add("application/vnd.ms-excel"); + expected.add("application/vnd.ms-powerpoint"); + expected.add("application/vnd.openxmlformats-officedocument.wordprocessingml.document"); + expected.add("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"); + expected.add("application/vnd.openxmlformats-officedocument.presentationml.presentation"); + expected.add("application/pdf"); + expected.add("application/xml"); + expected.add("text/plain; charset=ISO-8859-1"); + //test that we're correctly handling attachment variants for + // files created by WPS 表格 (https://www.wps.cn/) + for (String suffix : new String[]{ + "ppt", "doc", "xls", "docx", "pptx", "xlsx" + }) { + List<Metadata> metadataList = getRecursiveMetadata("testWPSAttachment." + suffix); + Set<String> found = new HashSet<>(); + int i = 0; + for (Metadata m : metadataList) { + if (i++ > 0) { + found.add(m.get(Metadata.CONTENT_TYPE)); + } + } + Set<String> notFound = new HashSet<>(); + for (String ex : expected) { + if (! found.contains(ex)) { + notFound.add(ex); + } + } + if (notFound.size() > 0) { + fail("Couldn't find: " + notFound); + } + } + } } diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/resources/test-documents/testWPSAttachment.doc b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/resources/test-documents/testWPSAttachment.doc new file mode 100644 index 0000000..6958362 Binary files /dev/null and b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/resources/test-documents/testWPSAttachment.doc differ diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/resources/test-documents/testWPSAttachment.docx b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/resources/test-documents/testWPSAttachment.docx new file mode 100644 index 0000000..115bfa4 Binary files /dev/null and b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/resources/test-documents/testWPSAttachment.docx differ diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/resources/test-documents/testWPSAttachment.ppt b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/resources/test-documents/testWPSAttachment.ppt new file mode 100644 index 0000000..e75b17d Binary files /dev/null and b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/resources/test-documents/testWPSAttachment.ppt differ diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/resources/test-documents/testWPSAttachment.pptx b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/resources/test-documents/testWPSAttachment.pptx new file mode 100644 index 0000000..2a5c9c8 Binary files /dev/null and b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/resources/test-documents/testWPSAttachment.pptx differ diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/resources/test-documents/testWPSAttachment.xls b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/resources/test-documents/testWPSAttachment.xls new file mode 100644 index 0000000..27a451b Binary files /dev/null and b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/resources/test-documents/testWPSAttachment.xls differ diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/resources/test-documents/testWPSAttachment.xlsx b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/resources/test-documents/testWPSAttachment.xlsx new file mode 100644 index 0000000..38f1c28 Binary files /dev/null and b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/resources/test-documents/testWPSAttachment.xlsx differ diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/src/test/java/org/apache/tika/parser/microsoft/POIContainerExtractionTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/src/test/java/org/apache/tika/parser/microsoft/POIContainerExtractionTest.java index 6fb166e..783554f 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/src/test/java/org/apache/tika/parser/microsoft/POIContainerExtractionTest.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-package/src/test/java/org/apache/tika/parser/microsoft/POIContainerExtractionTest.java @@ -45,7 +45,6 @@ public class POIContainerExtractionTest extends AbstractPOIContainerExtractionTe ContainerExtractor extractor = new ParserContainerExtractor(); TikaTest.TrackingHandler handler; - // Excel with a word doc and a powerpoint doc, both of which have images in them // Without recursion, should see both documents + the images handler = process("testEXCEL_embeded.xls", extractor, false);
