This is an automated email from the ASF dual-hosted git repository.
tballison pushed a commit to branch branch_3x
in repository https://gitbox.apache.org/repos/asf/tika.git
The following commit(s) were added to refs/heads/branch_3x by this push:
new c30499824b 3x improve sax ooxml (#2952)
c30499824b is described below
commit c30499824b90727fc80c999e557b34793e4208f9
Author: Tim Allison <[email protected]>
AuthorDate: Mon Jul 13 13:07:18 2026 -0400
3x improve sax ooxml (#2952)
---
.../tika/parser/microsoft/AbstractListManager.java | 6 ++
.../microsoft/ooxml/AbstractOOXMLExtractor.java | 42 +++++++++-
.../parser/microsoft/ooxml/MetadataExtractor.java | 32 ++++---
.../microsoft/ooxml/SAXBasedMetadataExtractor.java | 12 ++-
.../ooxml/SXSLFPowerPointExtractorDecorator.java | 62 +++++++++++---
.../ooxml/SXWPFWordExtractorDecorator.java | 69 ++++++++++++---
.../microsoft/ooxml/VSDXExtractorDecorator.java | 4 +-
.../ooxml/XSSFExcelExtractorDecorator.java | 8 +-
.../microsoft/ooxml/XSSFSharedStringsShim.java | 7 ++
.../parser/microsoft/ooxml/XWPFListManager.java | 7 ++
.../ooxml/xwpf/XWPFEventBasedWordExtractor.java | 8 +-
.../microsoft/ooxml/xwpf/XWPFFeatureExtractor.java | 98 ++++++++++++++++++++++
.../parser/microsoft/ooxml/SXSLFExtractorTest.java | 4 +-
13 files changed, 299 insertions(+), 60 deletions(-)
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/AbstractListManager.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/AbstractListManager.java
index 2e5f8f21c4..06d3e98b1c 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/AbstractListManager.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/AbstractListManager.java
@@ -60,6 +60,12 @@ public abstract class AbstractListManager {
*/
public String incrementLevel(int levelNumber, LevelTuple[]
overrideLevelTuples) {
+ //a level beyond the defined levels is non-standard; return ""
rather than
+ //iterating up to it
+ if (levelNumber < 0 || levelNumber >= levelTuples.length) {
+ return "";
+ }
+
for (int i = lastLevel + 1; i < levelNumber; i++) {
if (i >= counts.size()) {
int val = getStart(i, overrideLevelTuples);
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/AbstractOOXMLExtractor.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/AbstractOOXMLExtractor.java
index d19aaae4a1..769775a492 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/AbstractOOXMLExtractor.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/AbstractOOXMLExtractor.java
@@ -65,6 +65,7 @@ import
org.apache.tika.parser.microsoft.OfficeParser.POIFSDocumentType;
import org.apache.tika.parser.microsoft.OfficeParserConfig;
import org.apache.tika.parser.microsoft.SummaryExtractor;
import org.apache.tika.sax.EmbeddedContentHandler;
+import org.apache.tika.sax.XHTMLBalancingHandler;
import org.apache.tika.sax.XHTMLContentHandler;
import org.apache.tika.utils.ExceptionUtils;
import org.apache.tika.utils.StringUtils;
@@ -512,8 +513,8 @@ public abstract class AbstractOOXMLExtractor implements
OOXMLExtractor {
}
/**
- * Emits an external reference as an anchor element with appropriate class.
- * Used for detecting external resources that could be security risks.
+ * Emits an external reference as an anchor element with appropriate class,
+ * surfacing external resources for downstream analysis.
*/
private void emitExternalRef(XHTMLContentHandler xhtml, String refType,
String url)
throws SAXException {
@@ -651,12 +652,24 @@ public abstract class AbstractOOXMLExtractor implements
OOXMLExtractor {
relatedPartPRC.getRelationship(i);
try {
PackagePart relatedPartPart =
-
parentPart.getRelatedPart(relatedPartPackageRelationship);
+ safeGetRelatedPart(parentPart,
relatedPartPackageRelationship);
+ if (relatedPartPart == null) {
+ continue;
+ }
+ //balance tags a failed sub-parse left open, else the
</div> below is misnested
+ XHTMLBalancingHandler balancer =
+ new XHTMLBalancingHandler(contentHandler);
try (InputStream stream =
relatedPartPart.getInputStream()) {
XMLReaderUtils.parseSAX(stream,
- new EmbeddedContentHandler(contentHandler),
context);
+ new EmbeddedContentHandler(balancer), context);
} catch (IOException | TikaException e) {
+ balancer.drainOpenElements();
+
parentMetadata.add(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING,
+ ExceptionUtils.getStackTrace(e));
+ } catch (SAXException e) {
+ balancer.drainOpenElements();
+ WriteLimitReachedException.throwIfWriteLimitReached(e);
parentMetadata.add(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING,
ExceptionUtils.getStackTrace(e));
}
@@ -670,4 +683,25 @@ public abstract class AbstractOOXMLExtractor implements
OOXMLExtractor {
}
+ /**
+ * Like {@code getRelatedPart} but returns null instead of throwing an
unchecked
+ * IllegalArgumentException when the target part is missing (common in
truncated files).
+ */
+ public static PackagePart safeGetRelatedPart(PackagePart source,
+ PackageRelationship
relationship)
+ throws InvalidFormatException {
+ if (source == null || relationship == null) {
+ return null;
+ }
+ if (!source.isRelationshipExists(relationship)) {
+ return null;
+ }
+ try {
+ return source.getRelatedPart(relationship);
+ } catch (IllegalArgumentException e) {
+ // Relationship exists but target part is missing from the package
+ return null;
+ }
+ }
+
}
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/MetadataExtractor.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/MetadataExtractor.java
index 8298e84095..9fcc518627 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/MetadataExtractor.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/MetadataExtractor.java
@@ -65,11 +65,9 @@ public class MetadataExtractor {
/**
* Hard cap on the accumulated text-content of a single property element
* inside docProps/custom.xml. Real OOXML property values are at most a few
- * hundred bytes; anything beyond this is either corruption or an attacker
- * trying to drive memory or CPU pressure (cf. the {@code <vt:decimal>}
- * BigDecimal DoS where a 1M-digit literal compresses ~1000:1 in deflate).
- * 64 KB leaves headroom for any legitimate value while bounding the
- * slow-path inputs decisively.
+ * hundred bytes; anything beyond this is malformed (e.g. a {@code
<vt:decimal>}
+ * with a 1M-digit literal). 64 KB leaves headroom for any legitimate value
+ * while bounding the slow-path inputs.
*/
static final int MAX_TEXT_BUFFER_LENGTH = 64 * 1024;
@@ -106,10 +104,9 @@ public class MetadataExtractor {
extractMetadata(extractor.getExtendedProperties(), metadata);
// Custom properties are read via SAX directly from the OPC part
// rather than through POI/XMLBeans. The XMLBeans path materializes
- // an attacker-controlled <vt:decimal> through BigDecimal(String),
- // which is O(n²) on JDK 17 -- a 3 KB crafted carrier with a
- // 1,000,000-digit literal burns ~25 s of CPU before this method
- // even returns. See ooxml-bigdecimal-dos.
+ // a <vt:decimal> through BigDecimal(String), which is O(n²) on
+ // JDK 17 -- a 3 KB carrier with a 1,000,000-digit literal burns
+ // ~25 s of CPU before this method even returns.
extractCustomPropertiesViaSAX(extractor.getPackage(), metadata);
}
}
@@ -204,12 +201,12 @@ public class MetadataExtractor {
/**
* Parse {@code docProps/custom.xml} directly via SAX, bypassing
- * POI/XMLBeans. The XMLBeans path materializes an attacker-controlled
- * {@code <vt:decimal>} through {@link BigDecimal#BigDecimal(String)}
- * during XML deserialization, which is O(n²) in the digit count on
- * JDK 17. By reading the part ourselves we can cap both the buffered
- * text content ({@link #MAX_TEXT_BUFFER_LENGTH}) and the decimal
- * literal length ({@link #MAX_DECIMAL_LENGTH}) before any slow parse
+ * POI/XMLBeans. The XMLBeans path materializes a {@code <vt:decimal>}
+ * through {@link BigDecimal#BigDecimal(String)} during XML
+ * deserialization, which is O(n²) in the digit count on JDK 17. By
+ * reading the part ourselves we can cap both the buffered text content
+ * ({@link #MAX_TEXT_BUFFER_LENGTH}) and the decimal literal length
+ * ({@link #MAX_DECIMAL_LENGTH}) before any slow parse
* runs.
*/
private void extractCustomPropertiesViaSAX(OPCPackage opcPackage, Metadata
metadata) {
@@ -349,9 +346,8 @@ public class MetadataExtractor {
break;
case "decimal":
// BigDecimal(String) is O(n²) on JDK 17; cap the input
- // length to keep an attacker-controlled <vt:decimal>
- // from burning CPU. Real values are < 50 chars; 256 is
- // generous. See ooxml-bigdecimal-dos.
+ // length so a large <vt:decimal> can't burn CPU. Real
+ // values are < 50 chars; 256 is generous.
if (trimmed.length() > MAX_DECIMAL_LENGTH) {
break;
}
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/SAXBasedMetadataExtractor.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/SAXBasedMetadataExtractor.java
index 941ad12906..4b7189a9fd 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/SAXBasedMetadataExtractor.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/SAXBasedMetadataExtractor.java
@@ -59,10 +59,9 @@ class SAXBasedMetadataExtractor extends MetadataExtractor {
/**
* Hard cap on the accumulated text-content of a single property element.
* Real OOXML property values are at most a few hundred bytes; anything
beyond
- * this is either corruption or an attacker trying to drive memory or CPU
- * pressure (cf. the {@code <vt:decimal>} BigDecimal DoS where a 1M-digit
- * literal compresses ~1000:1 in deflate). 64 KB leaves headroom for any
- * legitimate value while bounding the slow-path inputs decisively.
+ * this is malformed (e.g. a {@code <vt:decimal>} with a 1M-digit literal).
+ * 64 KB leaves headroom for any legitimate value while bounding the
+ * slow-path inputs.
*/
static final int MAX_TEXT_BUFFER_LENGTH = 64 * 1024;
@@ -482,9 +481,8 @@ class SAXBasedMetadataExtractor extends MetadataExtractor {
break;
case "decimal":
// BigDecimal(String) is O(n²) on JDK 17; cap the input
- // length to keep an attacker-controlled <vt:decimal>
- // from burning CPU. Real values are < 50 chars; 256 is
- // generous. See ooxml-bigdecimal-dos.
+ // length so a large <vt:decimal> can't burn CPU. Real
+ // values are < 50 chars; 256 is generous.
if (trimmed.length() > MAX_DECIMAL_LENGTH) {
break;
}
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/SXSLFPowerPointExtractorDecorator.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/SXSLFPowerPointExtractorDecorator.java
index ee5d7050c0..3c5d3f05fc 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/SXSLFPowerPointExtractorDecorator.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/SXSLFPowerPointExtractorDecorator.java
@@ -40,6 +40,7 @@ import org.xml.sax.SAXException;
import org.xml.sax.helpers.DefaultHandler;
import org.apache.tika.exception.TikaException;
+import org.apache.tika.exception.WriteLimitReachedException;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.metadata.Office;
import org.apache.tika.metadata.TikaCoreProperties;
@@ -48,6 +49,7 @@ import
org.apache.tika.parser.microsoft.ooxml.xslf.XSLFEventBasedPowerPointExtra
import org.apache.tika.sax.EmbeddedContentHandler;
import org.apache.tika.sax.XHTMLContentHandler;
import org.apache.tika.utils.ExceptionUtils;
+import org.apache.tika.utils.StringUtils;
import org.apache.tika.utils.XMLReaderUtils;
/**
@@ -100,6 +102,7 @@ public class SXSLFPowerPointExtractorDecorator extends
AbstractOOXMLExtractor {
protected void buildXHTML(XHTMLContentHandler xhtml) throws SAXException,
IOException {
loadCommentAuthors();
+ addCommentAuthorMetadata();
PackageRelationshipCollection slidesPRC = null;
@@ -110,17 +113,25 @@ public class SXSLFPowerPointExtractorDecorator extends
AbstractOOXMLExtractor {
ExceptionUtils.getStackTrace(e));
}
+ int hiddenSlideCount = 0;
if (slidesPRC != null && slidesPRC.size() > 0) {
for (int i = 0; i < slidesPRC.size(); i++) {
try {
-
handleSlidePart(mainDocument.getRelatedPart(slidesPRC.getRelationship(i)),
- xhtml);
- } catch (InvalidFormatException | ZipException e) {
+ PackagePart slidePart =
+ safeGetRelatedPart(mainDocument,
slidesPRC.getRelationship(i));
+ if (slidePart == null) {
+ continue;
+ }
+ hiddenSlideCount += handleSlidePart(slidePart, xhtml);
+ } catch (InvalidFormatException | ZipException |
IllegalArgumentException e) {
metadata.add(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING,
ExceptionUtils.getStackTrace(e));
}
}
}
+ if (hiddenSlideCount > 0) {
+ metadata.set(Office.NUM_HIDDEN_SLIDES, hiddenSlideCount);
+ }
if (config.isIncludeSlideMasterContent()) {
handleGeneralTextContainingPart(XSLFRelation.SLIDE_MASTER.getRelation(),
"slide-master",
@@ -152,7 +163,7 @@ public class SXSLFPowerPointExtractorDecorator extends
AbstractOOXMLExtractor {
PackagePart commentAuthorsPart = null;
try {
commentAuthorsPart =
mainDocument.getRelatedPart(prc.getRelationship(i));
- } catch (InvalidFormatException e) {
+ } catch (InvalidFormatException | IllegalArgumentException e) {
metadata.add(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING,
ExceptionUtils.getStackTrace(e));
}
@@ -171,27 +182,50 @@ public class SXSLFPowerPointExtractorDecorator extends
AbstractOOXMLExtractor {
}
- private void handleSlidePart(PackagePart slidePart, XHTMLContentHandler
xhtml)
+ private void addCommentAuthorMetadata() {
+ for (String name : commentAuthors.nameMap.values()) {
+ if (!StringUtils.isBlank(name)) {
+ metadata.add(Office.COMMENT_PERSONS, name);
+ }
+ }
+ }
+
+ /**
+ * @return 1 if the slide is hidden, 0 otherwise
+ */
+ private int handleSlidePart(PackagePart slidePart, XHTMLContentHandler
xhtml)
throws IOException, SAXException {
Map<String, String> linkedRelationships =
loadLinkedRelationships(slidePart, false, metadata);
// Map<String, String> hyperlinks =
loadHyperlinkRelationships(packagePart);
+ int hidden = 0;
xhtml.startElement("div", "class", "slide-content");
//pass metadata so the body handler can emit HAS_* signals for the
slide, and keep a
//reference so we can flag hidden slides after the parse
- OOXMLWordAndPowerPointTextHandler wordAndPPTHandler =
- new OOXMLWordAndPowerPointTextHandler(
- new OOXMLTikaBodyPartHandler(xhtml, metadata),
linkedRelationships);
+ OOXMLTikaBodyPartHandler bodyHandler = new
OOXMLTikaBodyPartHandler(xhtml, metadata);
try (InputStream stream = slidePart.getInputStream()) {
+ OOXMLWordAndPowerPointTextHandler wordAndPPTHandler =
+ new OOXMLWordAndPowerPointTextHandler(bodyHandler,
linkedRelationships);
XMLReaderUtils.parseSAX(CloseShieldInputStream.wrap(stream),
new EmbeddedContentHandler(wordAndPPTHandler), context);
if (wordAndPPTHandler.isHiddenSlide()) {
metadata.set(Office.HAS_HIDDEN_SLIDES, true);
+ hidden = 1;
+ }
+ if (wordAndPPTHandler.hasAnimations()) {
+ metadata.set(Office.HAS_ANIMATIONS, true);
}
+ } catch (SAXException e) {
+ //don't let one bad slide abort the rest; close any open tags first
+ WriteLimitReachedException.throwIfWriteLimitReached(e);
+ metadata.add(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING,
+ ExceptionUtils.getStackTrace(e));
+ bodyHandler.closeAnyPending();
} catch (TikaException | IOException e) {
metadata.add(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING,
ExceptionUtils.getStackTrace(e));
+ bodyHandler.closeAnyPending();
}
xhtml.endElement("div");
@@ -226,6 +260,7 @@ public class SXSLFPowerPointExtractorDecorator extends
AbstractOOXMLExtractor {
handleGeneralTextContainingPart(XSLFRelation.CHART.getRelation(),
"chart", slidePart,
metadata, new OOXMLWordAndPowerPointTextHandler(new
OOXMLTikaBodyPartHandler(xhtml),
linkedRelationships));
+ return hidden;
}
/**
@@ -250,11 +285,13 @@ public class SXSLFPowerPointExtractorDecorator extends
AbstractOOXMLExtractor {
PackagePart slidePart = null;
try {
slidePart =
mainDocument.getRelatedPart(slidePRC.getRelationship(i));
- } catch (InvalidFormatException e) {
+ } catch (InvalidFormatException | IllegalArgumentException e) {
metadata.add(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING,
ExceptionUtils.getStackTrace(e));
}
- addSlideParts(slidePart, parts);
+ if (slidePart != null) {
+ addSlideParts(slidePart, parts);
+ }
}
}
@@ -272,7 +309,7 @@ public class SXSLFPowerPointExtractorDecorator extends
AbstractOOXMLExtractor {
for (int i = 0; i < prc.size(); i++) {
PackagePart pp = null;
try {
- pp =
mainDocument.getRelatedPart(prc.getRelationship(i));
+ pp = safeGetRelatedPart(mainDocument,
prc.getRelationship(i));
} catch (InvalidFormatException e) {
metadata.add(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING,
ExceptionUtils.getStackTrace(e));
@@ -291,7 +328,8 @@ public class SXSLFPowerPointExtractorDecorator extends
AbstractOOXMLExtractor {
for (String relation : new
String[]{XSLFRelation.VML_DRAWING.getRelation(),
XSLFRelation.SLIDE_LAYOUT.getRelation(),
XSLFRelation.NOTES_MASTER.getRelation(),
- XSLFRelation.NOTES.getRelation()}) {
+ XSLFRelation.NOTES.getRelation(),
XSLFRelation.CHART.getRelation(),
+ XSLFRelation.DIAGRAM_DRAWING.getRelation()}) {
PackageRelationshipCollection prc = null;
try {
prc = slidePart.getRelationshipsByType(relation);
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/SXWPFWordExtractorDecorator.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/SXWPFWordExtractorDecorator.java
index 86cdb6e03e..08948aca21 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/SXWPFWordExtractorDecorator.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/SXWPFWordExtractorDecorator.java
@@ -21,6 +21,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -48,12 +49,16 @@ import org.apache.tika.metadata.Metadata;
import org.apache.tika.metadata.Office;
import org.apache.tika.metadata.TikaCoreProperties;
import org.apache.tika.parser.ParseContext;
+import org.apache.tika.parser.microsoft.EMFParser;
import org.apache.tika.parser.microsoft.ooxml.xwpf.XWPFEventBasedWordExtractor;
+import org.apache.tika.parser.microsoft.ooxml.xwpf.XWPFFeatureExtractor;
import org.apache.tika.parser.microsoft.ooxml.xwpf.XWPFNumberingShim;
import org.apache.tika.parser.microsoft.ooxml.xwpf.XWPFStylesShim;
import org.apache.tika.sax.EmbeddedContentHandler;
+import org.apache.tika.sax.ToTextContentHandler;
import org.apache.tika.sax.XHTMLContentHandler;
import org.apache.tika.utils.ExceptionUtils;
+import org.apache.tika.utils.StringUtils;
import org.apache.tika.utils.XMLReaderUtils;
/**
@@ -75,7 +80,8 @@ public class SXWPFWordExtractorDecorator extends
AbstractOOXMLExtractor {
new String[]{XWPFRelation.HEADER.getRelation(),
XWPFRelation.FOOTER.getRelation(),
XWPFRelation.FOOTNOTE.getRelation(),
"http://schemas.openxmlformats.org/officeDocument/2006/relationships/endnotes";,
-
"http://schemas.openxmlformats.org/officeDocument/2006/relationships/comments"};
+
"http://schemas.openxmlformats.org/officeDocument/2006/relationships/comments";,
+ AbstractOOXMLExtractor.RELATION_DIAGRAM_DATA};
// Relationship types for Word settings
private static final String SETTINGS_RELATION =
@@ -97,6 +103,9 @@ public class SXWPFWordExtractorDecorator extends
AbstractOOXMLExtractor {
};
private final OPCPackage opcPackage;
+
+ //accumulated across all body parts as they're parsed; consumed by
handleEmbeddedParts()
+ private final Map<String, EmbeddedPartMetadata> embeddedPartMetadataMap =
new HashMap<>();
private final ParseContext context;
private final Metadata metadata;
@@ -157,6 +166,9 @@ public class SXWPFWordExtractorDecorator extends
AbstractOOXMLExtractor {
* subdocuments, and framesets.
*/
private void detectSecurityFeatures(PackagePart documentPart,
XHTMLContentHandler xhtml) {
+ //hidden text, track changes, comments, comment persons
+ new XWPFFeatureExtractor().process(documentPart, metadata, context);
+
// Check for attached template (external template reference)
try {
PackageRelationshipCollection templateRels =
@@ -357,9 +369,55 @@ public class SXWPFWordExtractorDecorator extends
AbstractOOXMLExtractor {
ExceptionUtils.getStackTrace(e));
bodyHandler.closeAnyPending();
}
+ //the body handler collects progId + the emf relationship id for
embedded OLE refs as
+ //it parses. Resolve the display name from the emf, then hand the map
to
+ //handleEmbeddedParts() via getEmbeddedPartMetadataMap() below.
+ Map<String, EmbeddedPartMetadata> partMap =
bodyHandler.getEmbeddedPartMetadataMap();
+ for (EmbeddedPartMetadata embeddedPartMetadata : partMap.values()) {
+ tryToParseEmbeddedName(packagePart, embeddedPartMetadata);
+ }
+ embeddedPartMetadataMap.putAll(partMap);
return bodyHandler;
}
+ @Override
+ protected Map<String, EmbeddedPartMetadata> getEmbeddedPartMetadataMap() {
+ return embeddedPartMetadataMap;
+ }
+
+ /**
+ * The OLE object's display name (e.g. "Acrobat Document") is stored in a
comment field in
+ * the emf that renders it. This mirrors
XWPFWordExtractorDecorator#tryToParseEmbeddedName.
+ */
+ private void tryToParseEmbeddedName(PackagePart sourcePart,
+ EmbeddedPartMetadata
embeddedPartMetadata) {
+ String rid = embeddedPartMetadata.getEmfRelationshipId();
+ if (StringUtils.isBlank(rid)) {
+ return;
+ }
+ try {
+ PackageRelationship rel = sourcePart.getRelationship(rid);
+ if (rel == null) {
+ return;
+ }
+ PackagePart emfPart = safeGetRelatedPart(sourcePart, rel);
+ if (emfPart == null ||
!"image/x-emf".equals(emfPart.getContentType())) {
+ return;
+ }
+ try (InputStream is = emfPart.getInputStream()) {
+ Metadata m = new Metadata();
+ ToTextContentHandler toTextContentHandler = new
ToTextContentHandler();
+ new EMFParser().parse(is, toTextContentHandler, m, new
ParseContext());
+
embeddedPartMetadata.setRenderedName(toTextContentHandler.toString().trim());
+
embeddedPartMetadata.setFullName(m.get(EMFParser.EMF_ICON_STRING));
+ }
+ } catch (SecurityException e) {
+ throw e;
+ } catch (Exception e) {
+ //we tried
+ }
+ }
+
/**
* Dumps comments that were not inlined at a {@code commentReference} in
the main document
* (matching the DOM parser, which appends orphaned comments at the end).
@@ -445,15 +503,6 @@ public class SXWPFWordExtractorDecorator extends
AbstractOOXMLExtractor {
}
}
- private PackagePart safeGetRelatedPart(PackagePart parent,
PackageRelationship rel) {
- try {
- return parent.getRelatedPart(rel);
- } catch (Exception e) {
- return null;
- }
- }
-
-
private XWPFStylesShim loadStyles(PackagePart packagePart)
throws InvalidFormatException, TikaException, IOException,
SAXException {
PackageRelationshipCollection stylesParts =
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/VSDXExtractorDecorator.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/VSDXExtractorDecorator.java
index 197ccf6eda..0447c8fe0f 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/VSDXExtractorDecorator.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/VSDXExtractorDecorator.java
@@ -96,7 +96,7 @@ public class VSDXExtractorDecorator extends
AbstractOOXMLExtractor {
PackageRelationshipCollection pageRels =
pagesPart.getRelationshipsByType(VISIO_PAGE_REL);
for (PackageRelationship rel : pageRels) {
- PackagePart pagePart = pagesPart.getRelatedPart(rel);
+ PackagePart pagePart = safeGetRelatedPart(pagesPart, rel);
if (pagePart != null) {
pageParts.add(pagePart);
}
@@ -119,7 +119,7 @@ public class VSDXExtractorDecorator extends
AbstractOOXMLExtractor {
if (rels.isEmpty()) {
return null;
}
- return part.getRelatedPart(rels.getRelationship(0));
+ return safeGetRelatedPart(part, rels.getRelationship(0));
}
@Override
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/XSSFExcelExtractorDecorator.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/XSSFExcelExtractorDecorator.java
index 194d206a0e..26ad974d46 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/XSSFExcelExtractorDecorator.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/XSSFExcelExtractorDecorator.java
@@ -283,7 +283,7 @@ public class XSSFExcelExtractorDecorator extends
AbstractOOXMLExtractor {
//swallow
}
- // Extract external data sources (HIGH security risk - can hide
malicious URLs)
+ // Extract external data sources (surface external URLs for downstream
analysis)
try {
extractExternalDataSources(container, xhtml);
} catch (InvalidFormatException | TikaException | IOException |
SAXException e) {
@@ -526,7 +526,7 @@ public class XSSFExcelExtractorDecorator extends
AbstractOOXMLExtractor {
emitExternalRef(xhtml, "oleLink", "relationship:" + rId);
}
}
- // DDE links - security risk: can execute commands
+ // DDE links
if ("ddeLink".equals(localName)) {
foundDdeLink = true;
String ddeService = atts.getValue("ddeService");
@@ -676,7 +676,7 @@ public class XSSFExcelExtractorDecorator extends
AbstractOOXMLExtractor {
return;
}
for (PackageRelationship rel : coll) {
- PackagePart threadedCommentPart = sheetPart.getRelatedPart(rel);
+ PackagePart threadedCommentPart = safeGetRelatedPart(sheetPart,
rel);
if (threadedCommentPart == null) {
continue;
}
@@ -703,7 +703,7 @@ public class XSSFExcelExtractorDecorator extends
AbstractOOXMLExtractor {
return;
}
for (PackageRelationship rel : coll) {
- PackagePart personsPart = workbookPart.getRelatedPart(rel);
+ PackagePart personsPart = safeGetRelatedPart(workbookPart, rel);
if (personsPart == null) {
continue;
}
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/XSSFSharedStringsShim.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/XSSFSharedStringsShim.java
index 8556d0fbb3..a5b2b518c6 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/XSSFSharedStringsShim.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/XSSFSharedStringsShim.java
@@ -60,6 +60,13 @@ class XSSFSharedStringsShim {
}
String getItemAt(int idx) {
+ //a workbook whose sharedStrings part or relationship is missing still
has cells
+ //carrying t="s" indices. The string table is then empty and every
index is out of
+ //range. Return null (a supported outcome for the caller) so the
numeric cells and
+ //the rest of the workbook survive.
+ if (idx < 0 || idx >= strings.size()) {
+ return null;
+ }
return strings.get(idx);
}
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/XWPFListManager.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/XWPFListManager.java
index 383c31f0f7..c7e2ec6da6 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/XWPFListManager.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/XWPFListManager.java
@@ -91,6 +91,10 @@ public class XWPFListManager extends AbstractListManager {
if (lc == null) {
lc = loadLevelTuples(abNum);
}
+ if (lc == null) {
+ //the numbering.xml references an abstractNumId that isn't there
+ return "";
+ }
if (overrideTuples == null) {
overrideTuples = loadOverrideTuples(ctNum, lc.getNumberOfLevels());
}
@@ -133,6 +137,9 @@ public class XWPFListManager extends AbstractListManager {
//to get the abstract num information for the edge case where
//someone skips a level and the format is not context-free, e.g.
"1.B.i".
XWPFAbstractNum abstractNum = numbering.getAbstractNum(abNum.getVal());
+ if (abstractNum == null) {
+ return null;
+ }
CTAbstractNum ctAbstractNum = abstractNum.getCTAbstractNum();
LevelTuple[] levels = new LevelTuple[ctAbstractNum.sizeOfLvlArray()];
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/xwpf/XWPFEventBasedWordExtractor.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/xwpf/XWPFEventBasedWordExtractor.java
index f509d3be7c..6a72e9b5ee 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/xwpf/XWPFEventBasedWordExtractor.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/xwpf/XWPFEventBasedWordExtractor.java
@@ -175,7 +175,9 @@ public class XWPFEventBasedWordExtractor implements
POIXMLTextExtractor {
handlePart(header, xwpfListManager, sb);
}
}
- } catch (InvalidFormatException e) {
+ } catch (InvalidFormatException | IllegalArgumentException e) {
+ //getRelatedPart throws an unchecked IllegalArgumentException when
the target
+ //part is missing -- common in truncated/damaged packages
LOG.warn("Invalid format", e);
}
@@ -229,7 +231,9 @@ public class XWPFEventBasedWordExtractor implements
POIXMLTextExtractor {
hyperlinks.put(id, url);
}
}
- } catch (InvalidFormatException e) {
+ } catch (InvalidFormatException | IllegalArgumentException e) {
+ //getRelatedPart throws an unchecked IllegalArgumentException when
the target
+ //part is missing -- common in truncated/damaged packages
LOG.warn("Invalid format", e);
}
return hyperlinks;
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/xwpf/XWPFFeatureExtractor.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/xwpf/XWPFFeatureExtractor.java
new file mode 100644
index 0000000000..3cb1935cc6
--- /dev/null
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/xwpf/XWPFFeatureExtractor.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.microsoft.ooxml.xwpf;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.poi.openxml4j.opc.PackagePart;
+import org.apache.poi.xwpf.usermodel.XWPFDocument;
+import org.xml.sax.Attributes;
+import org.xml.sax.SAXException;
+import org.xml.sax.helpers.DefaultHandler;
+
+import org.apache.tika.exception.TikaException;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.Office;
+import org.apache.tika.parser.ParseContext;
+import org.apache.tika.utils.StringUtils;
+import org.apache.tika.utils.XMLReaderUtils;
+
+/**
+ * This is designed to extract features that are useful for forensics,
e-discovery and digital preservation.
+ * Specifically, the presence of: tracked changes, hidden text, comments and
comment authors. Because several of these
+ * features can be placed on run properties, which can be in lots of places,
we're scraping
+ * the document xml
+ */
+public class XWPFFeatureExtractor {
+
+ public void process(XWPFDocument xwpfDocument, Metadata metadata,
ParseContext parseContext) {
+ process(xwpfDocument.getPackagePart(), metadata, parseContext);
+ }
+
+ public void process(PackagePart packagePart, Metadata metadata,
ParseContext parseContext) {
+ try (InputStream is = packagePart.getInputStream()) {
+ FeatureHandler featureHandler = new FeatureHandler();
+ XMLReaderUtils.parseSAX(is, featureHandler, parseContext);
+ if (featureHandler.hasComments) {
+ metadata.set(Office.HAS_COMMENTS, true);
+ }
+ if (featureHandler.hasHidden) {
+ metadata.set(Office.HAS_HIDDEN_TEXT, true);
+ }
+ if (featureHandler.hasTrackChanges) {
+ metadata.set(Office.HAS_TRACK_CHANGES, true);
+ }
+ if (! featureHandler.authors.isEmpty()) {
+ for (String author : featureHandler.authors) {
+ metadata.add(Office.COMMENT_PERSONS, author);
+ }
+ }
+ } catch (IOException | TikaException | SAXException e) {
+ //swallow
+ }
+ }
+
+ private static class FeatureHandler extends DefaultHandler {
+ //see:
https://www.ericwhite.com/blog/using-xml-dom-to-detect-tracked-revisions-in-an-open-xml-wordprocessingml-document/
+ private static final Set<String> TRACK_CHANGES = Set.of("ins", "del",
"moveFrom", "moveTo");
+ private final Set<String> authors = new HashSet<>();
+ private boolean hasHidden = false;
+ private boolean hasTrackChanges = false;
+ private boolean hasComments = false;
+
+ @Override
+ public void startElement(String uri, String localName, String qName,
Attributes atts)
+ throws SAXException {
+ //we could check to ensure that the vanish element actually
surrounds text
+ //the current check could lead to false positives where
<w:vanish/> is around a space or no text.
+ if ("vanish".equals(localName)) {
+ hasHidden = true;
+ } else if (TRACK_CHANGES.contains(localName)) {
+ String trackChangesAuthor =
XMLReaderUtils.getAttrValue("author", atts);
+ if (!StringUtils.isBlank(trackChangesAuthor)) {
+ authors.add(trackChangesAuthor);
+ }
+ hasTrackChanges = true;
+ } else if ("commentReference".equals(localName) ||
"commentRangeStart".equals(localName)) {
+ hasComments = true;
+ }
+ }
+ }
+}
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/ooxml/SXSLFExtractorTest.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/ooxml/SXSLFExtractorTest.java
index a33139ad38..48de3013ee 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/ooxml/SXSLFExtractorTest.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/ooxml/SXSLFExtractorTest.java
@@ -62,7 +62,9 @@ public class SXSLFExtractorTest extends TikaTest {
List<Metadata> metadataList =
getRecursiveMetadata("testPPT_various2.pptx", parseContext);
- assertEquals(14, metadataList.size(), "right number of attachments");
+ //15, not 14 -- the chart-backed workbook
(Microsoft_Excel_Worksheet.xlsx) is reachable
+ //only via the CHART relation; this matches the DOM extractor's
attachment set exactly
+ assertEquals(15, metadataList.size(), "right number of attachments");
String mainContent =
metadataList.get(0).get(TikaCoreProperties.TIKA_CONTENT);
