This is an automated email from the ASF dual-hosted git repository. tballison pushed a commit to branch TIKA-4814-followup-1 in repository https://gitbox.apache.org/repos/asf/tika.git
commit 34b881ed37e1c2a99a7fffcc519370895586d19d Author: tallison <[email protected]> AuthorDate: Fri Aug 21 16:25:42 2026 -0400 TIKA-4814 -- OneNote follow-ups: fall back to legacy string dump when parsing fails or yields no content, record dual-role authors, cap file-derived allocations in fsshttpb deserializers, cache flattened property actions --- CHANGES.txt | 21 +-- .../apache/tika/parser/microsoft/onenote/GUID.java | 18 ++- .../parser/microsoft/onenote/OneNoteDocument.java | 3 + .../parser/microsoft/onenote/OneNoteParser.java | 137 ++++++++++++------- .../onenote/fsshttpb/MSOneStorePackage.java | 150 +++++++++++++-------- .../property/PrtArrayOfPropertyValues.java | 6 + .../onenote/fsshttpb/streamobj/ObjectDataBLOB.java | 19 --- .../streamobj/ObjectDataBLOBDataElementData.java | 15 --- .../streamobj/RevisionStoreObjectGroup.java | 12 +- .../space/ObjectSpaceObjectStreamOfContextIDs.java | 8 +- .../space/ObjectSpaceObjectStreamOfOIDs.java | 8 +- .../space/ObjectSpaceObjectStreamOfOSIDs.java | 6 + .../tika/parser/microsoft/onenote/GUIDTest.java | 19 +++ .../microsoft/onenote/OneNoteParserTest.java | 55 ++++++-- .../onenote/fsshttpb/MSOneStoreBlobTest.java | 17 ++- .../onenote/fsshttpb/MSOneStorePackageTest.java | 87 ++++++++++++ 16 files changed, 422 insertions(+), 159 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 5db77dc8be..390f3ff3c9 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,5 +1,13 @@ Release 4.1.0 - ??? + * OneNote extraction now follows document order, omits superseded page + revisions, sorts author metadata, extracts embedded object BLOBs, and + bounds malformed-input recursion and file-derived allocations. Parse + warnings and embedded relationship IDs are exposed in metadata. Malformed + or truncated files that cannot be fully parsed, and files whose walk + yields no content, now fall back to the legacy string dump instead of + failing or returning empty output (TIKA-4814). + * RawTiffParser extracts the camera-generated JPEG previews embedded in TIFF-based raw images (Nikon NEF/NRW, Sony ARW/SRF/SR2, Pentax PEF/PTX, Adobe DNG and Canon CR2, including BigTIFF DNG containers) as thumbnail @@ -481,15 +489,6 @@ Release 4.0.0 - 8/18/2026 OTHER CHANGES - * OneNote extraction now follows document order, omits superseded page - revisions, sorts author metadata, extracts embedded object BLOBs, and - bounds malformed-input recursion and per-file-node-list property allocation. - Malformed truncated property arrays now fail rather than return partial data; - parse warnings and embedded relationship IDs are exposed in metadata (TIKA-4814). - - * Dependency upgrades since 4.0.0-beta-1, including Jetty 12.1.12, CXF - 4.2.3 and SolrJ 10.0.0 (TIKA-4327). - * Release artifacts are now channel-specific. Maven Central gets slim per-module jars (plus pom, sources and javadoc); the Apache dist area gets runnable zip distributions (tika-app, tika-server-standard, @@ -559,6 +558,10 @@ Release 4.0.0 - 8/18/2026 * Fix concurrency bug in TikaToXMP (TIKA-4393). + * Dependency upgrades throughout the 4.0.0 line, including Jetty 11 -> + 12.1.12, CXF 4.0 -> 4.2.3 and SolrJ 8.11.4 -> 10.0.0, plus routine + library updates (TIKA-4327). + Release 4.0.0-beta-1 - 6/29/2026 diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/GUID.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/GUID.java index fe63cc9bf4..55e788bb4a 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/GUID.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/GUID.java @@ -56,8 +56,8 @@ public class GUID implements Comparable<GUID> { throw new TikaException("Invalid GUID string"); } for (int i = 0; i < hex.length(); i += 2) { - int high = Character.digit(hex.charAt(i), 16); - int low = Character.digit(hex.charAt(i + 1), 16); + int high = asciiHexDigit(hex.charAt(i)); + int low = asciiHexDigit(hex.charAt(i + 1)); if (high < 0 || low < 0) { throw new TikaException("Invalid GUID string"); } @@ -66,6 +66,20 @@ public class GUID implements Comparable<GUID> { return new GUID(intGuid); } + // Character.digit accepts non-ASCII Unicode digits; GUIDs are ASCII hex only + private static int asciiHexDigit(char c) { + if (c >= '0' && c <= '9') { + return c - '0'; + } + if (c >= 'a' && c <= 'f') { + return c - 'a' + 10; + } + if (c >= 'A' && c <= 'F') { + return c - 'A' + 10; + } + return -1; + } + public static int memcmp(int[] b1, int[] b2, int sz) { for (int i = 0; i < sz; i++) { if (b1[i] != b2[i]) { diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/OneNoteDocument.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/OneNoteDocument.java index 936228f25c..cd76a58a55 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/OneNoteDocument.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/OneNoteDocument.java @@ -34,6 +34,9 @@ class OneNoteDocument { Map<ExtendedGUID, Pair<Long, ExtendedGUID>> revisionRoleMap = new HashMap<>(); ExtendedGUID currentRevision = ExtendedGUID.nil(); FileNodeList root = new FileNodeList(); + // set when the root file node list could not be fully parsed (e.g. a truncated file); + // the header and any structure parsed so far remain usable + Exception structureParseException; public OneNoteDocument() { diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/OneNoteParser.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/OneNoteParser.java index d3c4ec2515..85dc2f2e78 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/OneNoteParser.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/OneNoteParser.java @@ -128,39 +128,56 @@ public class OneNoteParser implements Parser { metadata.set(OneNote.RGB_PLACEHOLDER, "0x" + Long.toHexString(oneNoteDocument.header.rgbPlaceholder)); - Pair<Long, ExtendedGUID> roleAndContext = Pair.of(1L, ExtendedGUID.nil()); - OneNoteTreeWalker oneNoteTreeWalker = - new OneNoteTreeWalker(options, oneNoteDocument, oneNoteDirectFileResource, - xhtml, metadata, context, roleAndContext); + Exception structureFailure = oneNoteDocument.structureParseException; + boolean walked = false; + if (structureFailure == null) { + try { + Pair<Long, ExtendedGUID> roleAndContext = Pair.of(1L, ExtendedGUID.nil()); + OneNoteTreeWalker oneNoteTreeWalker = + new OneNoteTreeWalker(options, oneNoteDocument, + oneNoteDirectFileResource, xhtml, metadata, context, + roleAndContext); - oneNoteTreeWalker.walkTree(); + oneNoteTreeWalker.walkTree(); - if (!oneNoteTreeWalker.getAuthors().isEmpty()) { - metadata.set(TikaCoreProperties.CREATOR, - sortedValues(oneNoteTreeWalker.getAuthors())); - } - if (!oneNoteTreeWalker.getMostRecentAuthors().isEmpty()) { - metadata.set(OneNote.MOST_RECENT_AUTHORS, - sortedValues(oneNoteTreeWalker.getMostRecentAuthors())); - } - if (!oneNoteTreeWalker.getOriginalAuthors().isEmpty()) { - metadata.set(OneNote.ORIGINAL_AUTHORS, - sortedValues(oneNoteTreeWalker.getOriginalAuthors())); - } - if (!Instant.MAX.equals( - Instant.ofEpochMilli(oneNoteTreeWalker.getCreationTimestamp()))) { - metadata.set(OneNote.CREATION_TIMESTAMP, - String.valueOf(oneNoteTreeWalker.getCreationTimestamp())); - } - if (!Instant.MIN.equals(oneNoteTreeWalker.getLastModifiedTimestamp())) { - metadata.set(OneNote.LAST_MODIFIED_TIMESTAMP, String.valueOf( - oneNoteTreeWalker.getLastModifiedTimestamp().toEpochMilli())); + if (!oneNoteTreeWalker.getAuthors().isEmpty()) { + metadata.set(TikaCoreProperties.CREATOR, + sortedValues(oneNoteTreeWalker.getAuthors())); + } + if (!oneNoteTreeWalker.getMostRecentAuthors().isEmpty()) { + metadata.set(OneNote.MOST_RECENT_AUTHORS, + sortedValues(oneNoteTreeWalker.getMostRecentAuthors())); + } + if (!oneNoteTreeWalker.getOriginalAuthors().isEmpty()) { + metadata.set(OneNote.ORIGINAL_AUTHORS, + sortedValues(oneNoteTreeWalker.getOriginalAuthors())); + } + if (!Instant.MAX.equals( + Instant.ofEpochMilli(oneNoteTreeWalker.getCreationTimestamp()))) { + metadata.set(OneNote.CREATION_TIMESTAMP, + String.valueOf(oneNoteTreeWalker.getCreationTimestamp())); + } + if (!Instant.MIN.equals(oneNoteTreeWalker.getLastModifiedTimestamp())) { + metadata.set(OneNote.LAST_MODIFIED_TIMESTAMP, String.valueOf( + oneNoteTreeWalker.getLastModifiedTimestamp().toEpochMilli())); + } + if (oneNoteTreeWalker.getLastModified() > Long.MIN_VALUE) { + metadata.set(TikaCoreProperties.MODIFIED, + String.valueOf(oneNoteTreeWalker.getLastModified())); + } + walked = true; + } catch (Exception e) { + rethrowIfLimitReached(e); + structureFailure = e; + } } - if (oneNoteTreeWalker.getLastModified() > Long.MIN_VALUE) { - metadata.set(TikaCoreProperties.MODIFIED, - String.valueOf(oneNoteTreeWalker.getLastModified())); + if (!walked) { + legacyFallbackDump("OneNote parse failed; falling back to legacy text dump: " + + failureMessage(structureFailure), structureFailure, metadata, + xhtml, oneNoteDirectFileResource); } } else if (header.isLegacyOrAlternativePackaging()) { + MSOneStorePackage pkg = null; try { AlternativePackaging alternatePackageOneStoreFile = new AlternativePackaging(); byte[] bytes = Files.readAllBytes(tis.getPath()); @@ -168,26 +185,23 @@ public class OneNoteParser implements Parser { alternatePackageOneStoreFile.doDeserializeFromByteArray(bytes, 0); MSOneStoreParser onenoteParser = new MSOneStoreParser(); - MSOneStorePackage pkg = - onenoteParser.parse(alternatePackageOneStoreFile.dataElementPackage); + pkg = onenoteParser.parse(alternatePackageOneStoreFile.dataElementPackage); pkg.walkTree(options, metadata, xhtml, context); } catch (Exception e) { - WriteLimitReachedException.throwIfWriteLimitReached(e); - if (e instanceof EmbeddedLimitReachedException) { - throw (EmbeddedLimitReachedException) e; - } - String failure = e.getMessage() == null ? e.getClass().getSimpleName() : - e.getMessage(); - LOG.warn("OneNote FSSHTTPB parse failed; falling back to legacy text dump: {}", - failure); - LOG.debug("OneNote FSSHTTPB parse failure", e); - metadata.add(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING, + rethrowIfLimitReached(e); + legacyFallbackDump( "OneNote FSSHTTPB parse failed; falling back to legacy text dump: " + - failure); - OneNoteLegacyDumpStrings dumpStrings = - new OneNoteLegacyDumpStrings(oneNoteDirectFileResource, xhtml); - dumpStrings.dump(); + failureMessage(e), e, metadata, xhtml, + oneNoteDirectFileResource); + pkg = null; + } + if (pkg != null && !pkg.hasEmittedContent()) { + // the walk completed but every page dangled - without this a degraded + // file would yield empty output where the dump still finds its text + legacyFallbackDump("OneNote FSSHTTPB parse produced no content; " + + "falling back to legacy text dump", null, metadata, xhtml, + oneNoteDirectFileResource); } } else { throw new TikaException("Invalid OneStore document - could not parse headers"); @@ -204,6 +218,29 @@ public class OneNoteParser implements Parser { return sorted; } + private static void rethrowIfLimitReached(Exception e) throws TikaException, SAXException { + WriteLimitReachedException.throwIfWriteLimitReached(e); + if (e instanceof EmbeddedLimitReachedException) { + throw (EmbeddedLimitReachedException) e; + } + } + + private static String failureMessage(Exception e) { + return e.getMessage() == null ? e.getClass().getSimpleName() : e.getMessage(); + } + + private static void legacyFallbackDump(String warning, Exception cause, Metadata metadata, + XHTMLContentHandler xhtml, + OneNoteDirectFileResource oneNoteDirectFileResource) + throws TikaException, SAXException { + LOG.warn(warning); + if (cause != null) { + LOG.debug("OneNote parse failure", cause); + } + metadata.add(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING, warning); + new OneNoteLegacyDumpStrings(oneNoteDirectFileResource, xhtml).dump(); + } + /** * Create a OneNoteDocument object. * <p> @@ -249,9 +286,15 @@ public class OneNoteParser implements Parser { if (oneNoteDocument.header.isMsOneStoreFormat()) { // Now that we parsed the header, the "root file node list" - oneNotePtr.reposition(oneNoteDocument.header.fcrFileNodeListRoot); - FileNodePtr curPath = new FileNodePtr(); - oneNotePtr.deserializeFileNodeList(oneNoteDocument.root, curPath); + try { + oneNotePtr.reposition(oneNoteDocument.header.fcrFileNodeListRoot); + FileNodePtr curPath = new FileNodePtr(); + oneNotePtr.deserializeFileNodeList(oneNoteDocument.root, curPath); + } catch (TikaException | IOException | RuntimeException e) { + // a truncated or malformed root list is recorded, not thrown, so the + // caller can fall back to the legacy string dump + oneNoteDocument.structureParseException = e; + } } return oneNoteDocument; } diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStorePackage.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStorePackage.java index b8c9bd0b31..cc015d5239 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStorePackage.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStorePackage.java @@ -28,6 +28,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.IdentityHashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -142,6 +143,11 @@ public class MSOneStorePackage { private final Set<String> recordedParseWarningKeys = new HashSet<>(); private boolean parseWarningsSuppressed; private boolean storageMappingsIndexed; + private boolean contentEmitted; + // flattened property actions per object; objects can be re-flattened many times during + // picture/resource-name resolution, which is quadratic without this cache + private final Map<RevisionStoreObject, List<PropertyAction>> objectActionsCache = + new IdentityHashMap<>(); private final Map<CellID, StorageIndexCellMapping> storageIndexCellMappingsById = new HashMap<>(); private final Map<ExGuid, StorageIndexRevisionMapping> storageIndexRevisionMappingsById = @@ -305,8 +311,7 @@ public class MSOneStorePackage { } } for (RevisionStoreCell cell : remainingCells.values()) { - if (cell.cellID == null || cell.cellID.extendGUID2 == null || - !coveredObjectSpaces.contains(cell.cellID.extendGUID2)) { + if (!coveredObjectSpaces.contains(cell.cellID.extendGUID2)) { // not an older version of one of the pages - keep it so no content is lost otherCells.add(cell); } @@ -350,7 +355,7 @@ public class MSOneStorePackage { if (object.objectID != null && !visited.add(object.objectID)) { return; } - List<PropertyAction> actions = collectObjectActions(object, depth); + List<PropertyAction> actions = collectObjectActions(object); for (PropertyAction action : actions) { if (action.spaceReference != null) { out.add(action.spaceReference); @@ -373,19 +378,26 @@ public class MSOneStorePackage { // Only objects reachable from the root objects of the current revision are part of // the current content. The object groups may also contain older, superseded versions // of objects (under a different object ID); those are intentionally not walked. - boolean resolvedRoot = false; + // A blob-only root (no property set) cannot reach the page body, so it does not + // count as a resolved content root: if the content root declare dangles next to it, + // the walk-everything fallback must still fire or the page body is silently lost. + boolean resolvedContentRoot = false; + boolean unresolvedRoot = false; for (RevisionManifestRootDeclare rootDeclare : cell.rootDeclares) { RevisionStoreObject rootObject = objectsById.get(rootDeclare.objectExGuid); if (rootObject == null) { recordParseWarning("OneNote cell root object " + rootDeclare.objectExGuid + " could not be resolved"); + unresolvedRoot = true; } else { - resolvedRoot = true; + if (rootObject.propertySet != null) { + resolvedContentRoot = true; + } walkObject(rootObject, objectsById, visited, AuthorRole.NONE, options, metadata, xhtml, 0); } } - if (!resolvedRoot) { + if (cell.rootDeclares.isEmpty() || (unresolvedRoot && !resolvedContentRoot)) { if (cell.rootDeclares.isEmpty()) { recordParseWarning("OneNote cell has no declared root objects; walking all objects"); } else { @@ -481,11 +493,15 @@ public class MSOneStorePackage { return; } if (object.objectID != null && !visited.add(object.objectID)) { + // one object may be referenced under several author roles (e.g. the same author + // as both AuthorOriginal and AuthorMostRecent) - record the role even though + // the subtree is not walked again + recordAuthors(object, authorRole); return; } List<PropertyAction> actions = object.propertySet != null && object.propertySet.objectSpaceObjectPropSet != null ? - collectObjectActions(object, depth) : Collections.emptyList(); + collectObjectActions(object) : Collections.emptyList(); EmbeddedResourceInfo resourceInfo = embeddedResourceInfo(actions); if (resourceInfo == null) { resourceInfo = inheritedResourceInfo; @@ -552,7 +568,7 @@ public class MSOneStorePackage { if (object.propertySet == null || object.propertySet.objectSpaceObjectPropSet == null) { return false; } - for (PropertyAction action : collectObjectActions(object, depth)) { + for (PropertyAction action : collectObjectActions(object)) { if (action.isChildReference && action.childReference != null && hasUsablePicture(action.childReference, objectsById, visited, depth + 1)) { return true; @@ -572,7 +588,7 @@ public class MSOneStorePackage { candidate.propertySet.objectSpaceObjectPropSet == null) { continue; } - List<PropertyAction> candidateActions = collectObjectActions(candidate, depth); + List<PropertyAction> candidateActions = collectObjectActions(candidate); for (PropertyAction action : candidateActions) { if (action.isChildReference && target.objectID.equals(action.childReference) && (action.oneNotePropertyEnum == OneNotePropertyEnum.PictureContainer || @@ -643,22 +659,24 @@ public class MSOneStorePackage { } /** - * Flattens the properties of an object, in order, into a list of actions. + * Flattens the properties of an object, in order, into a list of actions. The result is + * cached per object; callers must not mutate the returned list. */ private List<PropertyAction> collectObjectActions(RevisionStoreObject object) { - return collectObjectActions(object, 0); - } - - private List<PropertyAction> collectObjectActions(RevisionStoreObject object, int depth) { + List<PropertyAction> actions = objectActionsCache.get(object); + if (actions != null) { + return actions; + } List<ExGuid> referencedObjects = object.referencedObjectID == null || object.referencedObjectID.content == null ? Collections.emptyList() : object.referencedObjectID.content; List<CellID> referencedSpaces = object.referencedObjectSpacesID == null || object.referencedObjectSpacesID.content == null ? Collections.emptyList() : object.referencedObjectSpacesID.content; - List<PropertyAction> actions = new ArrayList<>(); + actions = new ArrayList<>(); collectActions(object.propertySet.objectSpaceObjectPropSet.body, referencedObjects, - new int[]{0}, referencedSpaces, new int[]{0}, actions, depth); + new int[]{0}, referencedSpaces, new int[]{0}, actions, 0); + objectActionsCache.put(object, actions); return actions; } @@ -862,17 +880,8 @@ public class MSOneStorePackage { } metadata.set(TikaCoreProperties.MODIFIED, String.valueOf(lastModified)); } else if (oneNotePropertyEnum == OneNotePropertyEnum.Author) { - String author = decodeOneNoteText( - ((PrtFourBytesOfLengthFollowedByData) property).data); - if (authorRole == AuthorRole.MOST_RECENT) { - mostRecentAuthors.add(author); - } else if (authorRole == AuthorRole.ORIGINAL) { - originalAuthors.add(author); - // the original authors are the creators of the content - authors.add(author); - } else { - authors.add(author); - } + recordAuthor(decodeOneNoteText( + ((PrtFourBytesOfLengthFollowedByData) property).data), authorRole); } else if (propertyType == PropertyType.FourBytesOfLengthFollowedByData) { boolean isBinary = propertyIsBinary(oneNotePropertyEnum); PrtFourBytesOfLengthFollowedByData dataProperty = @@ -880,28 +889,15 @@ public class MSOneStorePackage { if ((dataProperty.data.length & 1) == 0 && oneNotePropertyEnum != OneNotePropertyEnum.TextExtendedAscii && !isBinary) { if (options.getUtf16PropertiesToPrint().contains(oneNotePropertyEnum)) { - xhtml.startElement(P); - try { - xhtml.characters(new String(dataProperty.data, StandardCharsets.UTF_16LE)); - } finally { - xhtml.endElement(P); - } + emitParagraph(xhtml, new String(dataProperty.data, + StandardCharsets.UTF_16LE)); } } else if (oneNotePropertyEnum == OneNotePropertyEnum.TextExtendedAscii) { - xhtml.startElement(P); - try { - xhtml.characters(new String(dataProperty.data, StandardCharsets.US_ASCII)); - } finally { - xhtml.endElement(P); - } + emitParagraph(xhtml, new String(dataProperty.data, StandardCharsets.US_ASCII)); } else if (!isBinary) { if (options.getUtf16PropertiesToPrint().contains(oneNotePropertyEnum)) { - xhtml.startElement(P); - try { - xhtml.characters(new String(dataProperty.data, StandardCharsets.UTF_16LE)); - } finally { - xhtml.endElement(P); - } + emitParagraph(xhtml, new String(dataProperty.data, + StandardCharsets.UTF_16LE)); } } else { if (oneNotePropertyEnum == OneNotePropertyEnum.RichEditTextUnicode) { @@ -920,6 +916,35 @@ public class MSOneStorePackage { return new String(bytes, StandardCharsets.UTF_16LE).replace("\u0000", ""); } + private void recordAuthor(String author, AuthorRole role) { + if (role == AuthorRole.MOST_RECENT) { + mostRecentAuthors.add(author); + } else if (role == AuthorRole.ORIGINAL) { + originalAuthors.add(author); + // the original authors are the creators of the content + authors.add(author); + } else { + authors.add(author); + } + } + + /** + * Records the Author properties of an already-visited object under the given role. + */ + private void recordAuthors(RevisionStoreObject object, AuthorRole role) { + if (role == AuthorRole.NONE || object.propertySet == null || + object.propertySet.objectSpaceObjectPropSet == null) { + return; + } + for (PropertyAction action : collectObjectActions(object)) { + if (action.oneNotePropertyEnum == OneNotePropertyEnum.Author && + action.property instanceof PrtFourBytesOfLengthFollowedByData) { + recordAuthor(decodeOneNoteText( + ((PrtFourBytesOfLengthFollowedByData) action.property).data), role); + } + } + } + private EmbeddedResourceInfo embeddedResourceInfo(List<PropertyAction> actions) { for (PropertyAction action : actions) { if (action.property instanceof PrtFourBytesOfLengthFollowedByData && @@ -939,10 +964,11 @@ public class MSOneStorePackage { return null; } - private String sanitizeResourceName(String name) { + static String sanitizeResourceName(String name) { name = name.replace('\\', '/'); + // drop a drive-letter prefix (C:...) so the basename below survives if (name.length() > 1 && name.charAt(1) == ':') { - return ""; + name = name.substring(2); } int slash = name.lastIndexOf('/'); name = slash >= 0 ? name.substring(slash + 1) : name; @@ -959,6 +985,7 @@ public class MSOneStorePackage { if (data == null || data.length == 0 || embeddedDocumentExtractor == null) { return; } + contentEmitted = true; Metadata embeddedMetadata = Metadata.newInstance(this.parseContext); embeddedMetadata.set(TikaCoreProperties.EMBEDDED_RESOURCE_TYPE, resourceInfo == null ? TikaCoreProperties.EmbeddedResourceType.ATTACHMENT.toString() : @@ -1011,16 +1038,33 @@ public class MSOneStorePackage { } finally { xhtml.endElement("a"); } + contentEmitted = true; } else { - xhtml.startElement(P); - try { - xhtml.characters(txt); - } finally { - xhtml.endElement(P); - } + emitParagraph(xhtml, txt); } } + private void emitParagraph(XHTMLContentHandler xhtml, String text) throws SAXException { + xhtml.startElement(P); + try { + xhtml.characters(text); + } finally { + xhtml.endElement(P); + } + if (!text.isEmpty()) { + contentEmitted = true; + } + } + + /** + * Whether the tree walk emitted any content - text or an embedded object. When it did + * not, the caller can fall back to the legacy string dump so a degraded file still + * yields its text. + */ + public boolean hasEmittedContent() { + return contentEmitted; + } + private long getScalar(IProperty property) throws TikaException, IOException { if (property instanceof FourBytesOfData) { FourBytesOfData fourBytesOfDataProp = (FourBytesOfData) property; diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/property/PrtArrayOfPropertyValues.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/property/PrtArrayOfPropertyValues.java index 68b1dab169..d39506b553 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/property/PrtArrayOfPropertyValues.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/property/PrtArrayOfPropertyValues.java @@ -47,6 +47,12 @@ public class PrtArrayOfPropertyValues implements IProperty { this.propertyID = new PropertyID(); int len = this.propertyID.doDeserializeFromByteArray(byteArray, index); index += len; + // each PropertySet consumes at least its 2-byte count, so a valid count cannot + // exceed the remaining bytes / 2; bounds the allocation on malformed counts + if (this.cProperties < 0 || this.cProperties > (byteArray.length - index) / 2) { + throw new IOException("prtArrayOfPropertyValues count " + this.cProperties + + " exceeds remaining data " + (byteArray.length - index)); + } this.data = new PropertySet[this.cProperties]; for (int i = 0; i < this.cProperties; i++) { this.data[i] = new PropertySet(); diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/ObjectDataBLOB.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/ObjectDataBLOB.java index e92d04f6e9..2873e1d013 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/ObjectDataBLOB.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/ObjectDataBLOB.java @@ -30,14 +30,8 @@ import org.apache.tika.parser.microsoft.onenote.fsshttpb.util.ByteUtil; * e.g. an embedded image or file. See MS-FSSHTTPB section 2.2.1.12.8. */ public class ObjectDataBLOB extends StreamObject { - /** - * A binary item that holds the opaque binary data. - */ public BinaryItem data; - /** - * Initializes a new instance of the ObjectDataBLOB class. - */ public ObjectDataBLOB() { super(StreamObjectTypeHeaderStart.ObjectDataBLOB); this.data = new BinaryItem(); @@ -53,13 +47,6 @@ public class ObjectDataBLOB extends StreamObject { return ByteUtil.toByteArray(this.data.content); } - /** - * Used to de-serialize the element. - * - * @param byteArray A Byte array - * @param currentIndex Start position - * @param lengthOfItems The length of the items - */ @Override protected void deserializeItemsFromByteArray(byte[] byteArray, AtomicInteger currentIndex, int lengthOfItems) @@ -75,12 +62,6 @@ public class ObjectDataBLOB extends StreamObject { currentIndex.set(index.get()); } - /** - * Used to convert the element into a byte List - * - * @param byteList A Byte list - * @return The number of elements actually contained in the list - */ @Override protected int serializeItemsToByteList(List<Byte> byteList) throws IOException { int startPoint = byteList.size(); diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/ObjectDataBLOBDataElementData.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/ObjectDataBLOBDataElementData.java index 17828cf0d7..ff28e9d096 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/ObjectDataBLOBDataElementData.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/ObjectDataBLOBDataElementData.java @@ -29,20 +29,10 @@ import org.apache.tika.exception.TikaException; public class ObjectDataBLOBDataElementData extends DataElementData { public ObjectDataBLOB objectDataBLOB; - /** - * Initializes a new instance of the ObjectDataBLOBDataElementData class. - */ public ObjectDataBLOBDataElementData() { this.objectDataBLOB = new ObjectDataBLOB(); } - /** - * Deserializes this element from the supplied byte array. - * - * @param byteArray A byte array - * @param startIndex Start position - * @return The number of bytes consumed - */ @Override public int deserializeDataElementDataFromByteArray(byte[] byteArray, int startIndex) throws TikaException, IOException { @@ -51,11 +41,6 @@ public class ObjectDataBLOBDataElementData extends DataElementData { return index.get() - startIndex; } - /** - * Used to convert the element into a byte List. - * - * @return The Byte list - */ @Override public List<Byte> serializeToByteList() throws TikaException, IOException { return this.objectDataBLOB.serializeToByteList(); diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/RevisionStoreObjectGroup.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/RevisionStoreObjectGroup.java index 1381fdbdbe..fb16a8402a 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/RevisionStoreObjectGroup.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/RevisionStoreObjectGroup.java @@ -18,7 +18,7 @@ package org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj; import java.io.IOException; import java.util.ArrayList; -import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -42,11 +42,16 @@ public class RevisionStoreObjectGroup { Map<ExGuid, DataElement> blobElements) throws IOException { RevisionStoreObjectGroup objectGroup = new RevisionStoreObjectGroup(objectGroupId); - Map<ExGuid, RevisionStoreObject> objectDict = new HashMap<>(); + // insertion-ordered so the object list (and any degraded-file fallback walk) is + // deterministic + Map<ExGuid, RevisionStoreObject> objectDict = new LinkedHashMap<>(); if (!isEncryption) { RevisionStoreObject revisionObject = null; for (int i = 0; i < dataObject.objectGroupDeclarations.objectDeclarationList.size(); i++) { + if (i >= dataObject.objectGroupData.objectGroupObjectDataList.size()) { + throw new IOException("Missing object data for object declaration " + i); + } ObjectGroupObjectDeclare objectDeclaration = dataObject.objectGroupDeclarations.objectDeclarationList.get(i); ObjectGroupObjectData objectData = @@ -108,6 +113,9 @@ public class RevisionStoreObjectGroup { } else { for (int i = 0; i < dataObject.objectGroupDeclarations.objectDeclarationList.size(); i++) { + if (i >= dataObject.objectGroupData.objectGroupObjectDataList.size()) { + throw new IOException("Missing object data for object declaration " + i); + } ObjectGroupObjectDeclare objectDeclaration = dataObject.objectGroupDeclarations.objectDeclarationList.get(i); ObjectGroupObjectData objectData = diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/space/ObjectSpaceObjectStreamOfContextIDs.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/space/ObjectSpaceObjectStreamOfContextIDs.java index fe6d288330..7e1e348e7d 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/space/ObjectSpaceObjectStreamOfContextIDs.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/space/ObjectSpaceObjectStreamOfContextIDs.java @@ -57,10 +57,16 @@ public class ObjectSpaceObjectStreamOfContextIDs { int headerCount = this.header.doDeserializeFromByteArray(byteArray, index); index += headerCount; + // each CompactID consumes 4 bytes, so a valid count cannot exceed the remaining + // bytes / 4; bounds the allocation on malformed counts + if (this.header.count > (byteArray.length - index) / 4L) { + throw new IOException("ObjectSpaceObjectStreamOfContextIDs count " + + this.header.count + " exceeds remaining data " + (byteArray.length - index)); + } this.body = new CompactID[(int) this.header.count]; for (int i = 0; i < this.header.count; i++) { CompactID compactID = new CompactID(); - int count = compactID.doDeserializeFromByteArray(byteArray, startIndex); + int count = compactID.doDeserializeFromByteArray(byteArray, index); this.body[i] = compactID; index += count; } diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/space/ObjectSpaceObjectStreamOfOIDs.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/space/ObjectSpaceObjectStreamOfOIDs.java index 7901dd015e..53398663c3 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/space/ObjectSpaceObjectStreamOfOIDs.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/space/ObjectSpaceObjectStreamOfOIDs.java @@ -58,10 +58,16 @@ public class ObjectSpaceObjectStreamOfOIDs { int headerCount = this.header.doDeserializeFromByteArray(byteArray, index); index += headerCount; + // each CompactID consumes 4 bytes, so a valid count cannot exceed the remaining + // bytes / 4; bounds the allocation on malformed counts + if (this.header.count > (byteArray.length - index) / 4L) { + throw new IOException("ObjectSpaceObjectStreamOfOIDs count " + this.header.count + + " exceeds remaining data " + (byteArray.length - index)); + } this.body = new CompactID[(int) this.header.count]; for (int i = 0; i < this.header.count; i++) { CompactID compactID = new CompactID(); - int count = compactID.doDeserializeFromByteArray(byteArray, startIndex); + int count = compactID.doDeserializeFromByteArray(byteArray, index); this.body[i] = compactID; index += count; } diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/space/ObjectSpaceObjectStreamOfOSIDs.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/space/ObjectSpaceObjectStreamOfOSIDs.java index 61f7deda90..f37de26f76 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/space/ObjectSpaceObjectStreamOfOSIDs.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/space/ObjectSpaceObjectStreamOfOSIDs.java @@ -57,6 +57,12 @@ public class ObjectSpaceObjectStreamOfOSIDs { int headerCount = this.header.doDeserializeFromByteArray(byteArray, index); index += headerCount; + // each CompactID consumes 4 bytes, so a valid count cannot exceed the remaining + // bytes / 4; bounds the allocation on malformed counts + if (this.header.count > (byteArray.length - index) / 4L) { + throw new IOException("ObjectSpaceObjectStreamOfOSIDs count " + this.header.count + + " exceeds remaining data " + (byteArray.length - index)); + } this.body = new CompactID[(int) this.header.count]; for (int i = 0; i < this.header.count; i++) { CompactID compactID = new CompactID(); diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/GUIDTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/GUIDTest.java index 6e7f3b70d1..0f21f861db 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/GUIDTest.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/GUIDTest.java @@ -16,6 +16,7 @@ */ package org.apache.tika.parser.microsoft.onenote; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import java.nio.charset.StandardCharsets; @@ -26,6 +27,24 @@ import org.apache.tika.exception.TikaException; public class GUIDTest { + @Test + public void testParsesValidGuidRoundTrip() throws Exception { + byte[] valid = "{638DE92F-a6d4-4BC1-9A36-4AFC2511A5B7}" + .getBytes(StandardCharsets.UTF_16LE); + + assertEquals("{638DE92F-A6D4-4BC1-9A36-4AFC2511A5B7}", + GUID.fromCurlyBraceUTF16Bytes(valid).toString()); + } + + @Test + public void testRejectsNonAsciiUnicodeDigits() { + // fullwidth '6' is a Unicode digit that Character.digit(c, 16) would accept + byte[] malformed = "{６38DE92F-A6D4-4BC1-9A36-4AFC2511A5B7}" + .getBytes(StandardCharsets.UTF_16LE); + + assertThrows(TikaException.class, () -> GUID.fromCurlyBraceUTF16Bytes(malformed)); + } + @Test public void testRejectsMalformedCurlyBraceGuid() { byte[] malformed = "{638DE92F-A6D4-4BC1-9IJ36-4AFC2511A5B7}" diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/OneNoteParserTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/OneNoteParserTest.java index 90e1ada59e..891757d9f0 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/OneNoteParserTest.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/OneNoteParserTest.java @@ -42,7 +42,6 @@ import org.junit.jupiter.api.io.TempDir; import org.xml.sax.ContentHandler; import org.apache.tika.TikaTest; -import org.apache.tika.exception.TikaException; import org.apache.tika.exception.TikaMemoryLimitException; import org.apache.tika.extractor.EmbeddedDocumentExtractor; import org.apache.tika.io.TikaInputStream; @@ -57,7 +56,9 @@ public class OneNoteParserTest extends TikaTest { //test recursive parser wrapper for image files @Test - public void testFuzzerRegressionInputsFailWithTikaExceptions() throws Exception { + public void testFuzzerRegressionInputsFallBackToLegacyDump() throws Exception { + // structural failures no longer abort the parse - the legacy string dump runs and + // the original failure is pinned in the parse-warning metadata String[][] resources = { {"testOneNote-fuzz1.one", "Missing dependent revision"}, {"testOneNote-fuzz2.one", "unified property count"}, @@ -68,15 +69,40 @@ public class OneNoteParserTest extends TikaTest { assertNotNull(input, resource[0]); try (InputStream stream = input; TikaInputStream tis = TikaInputStream.get(stream)) { - TikaException exception = assertThrows(TikaException.class, - () -> new OneNoteParser().parse(tis, new ToTextContentHandler(), - new Metadata(), new ParseContext()), resource[0]); - assertTrue(exception.getMessage().contains(resource[1]), - exception.getMessage()); + Metadata metadata = new Metadata(); + new OneNoteParser().parse(tis, new ToTextContentHandler(), metadata, + new ParseContext()); + assertTrue(Arrays.stream( + metadata.getValues(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING)) + .anyMatch(warning -> warning.contains(resource[1]) + && warning.contains("falling back to legacy text dump")), + () -> resource[0] + ": " + Arrays.toString(metadata.getValues( + TikaCoreProperties.TIKA_META_EXCEPTION_WARNING))); } } } + @Test + public void testTruncatedFileFallsBackToLegacyDump(@TempDir Path tempDir) throws Exception { + byte[] full; + try (InputStream is = getClass() + .getResourceAsStream("/test-documents/testOneNote1.one")) { + full = is.readAllBytes(); + } + // keep the 1024-byte header plus a sliver of content so the root file node list + // is unreachable + Path truncated = tempDir.resolve("truncated.one"); + Files.write(truncated, Arrays.copyOf(full, 2048)); + + Metadata metadata = new Metadata(); + try (TikaInputStream tis = TikaInputStream.get(truncated)) { + new OneNoteParser().parse(tis, new ToTextContentHandler(), metadata, + new ParseContext()); + } + assertTrue(Arrays.stream(metadata.getValues(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING)) + .anyMatch(warning -> warning.contains("falling back to legacy text dump"))); + } + /** * This is the sample document that is automatically created from onenote 2013. */ @@ -293,6 +319,16 @@ public class OneNoteParserTest extends TikaTest { assertTrue(embeddedTypes.contains("INLINE")); } + @Test + public void testOneNoteEmbeddedImageRecursiveMetadata() throws Exception { + List<Metadata> metadataList = getRecursiveMetadata("testOneNoteEmbeddedImage.one"); + + assertEquals(2, metadataList.size()); + Metadata embedded = metadataList.get(1); + assertEquals("INLINE", embedded.get(TikaCoreProperties.EMBEDDED_RESOURCE_TYPE)); + assertNotNull(embedded.get(TikaCoreProperties.EMBEDDED_RELATIONSHIP_ID)); + } + @Test public void testPropertyValueBudgetIsSharedAcrossCopiesAndResetsPerList( @TempDir Path tempDir) throws Exception { @@ -351,7 +387,10 @@ public class OneNoteParserTest extends TikaTest { // older page version snapshots are not reported assertEquals(Arrays.asList("Chang Du", "Du Chang"), Arrays.asList(metadata.getValues(TikaCoreProperties.CREATOR))); - assertEquals(1, metadata.getValues(ONE_NOTE_PREFIX + "mostRecentAuthors").length); + // both authors are referenced as AuthorMostRecent by current content; one of them + // is first visited under another role, so its most-recent role must still register + assertEquals(Arrays.asList("Chang Du", "Du Chang"), + Arrays.asList(metadata.getValues(ONE_NOTE_PREFIX + "mostRecentAuthors"))); assertEquals(Instant.ofEpochSecond(1636621406), Instant.ofEpochSecond(Long.parseLong(metadata.get(ONE_NOTE_PREFIX + "creationTimestamp")))); diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStoreBlobTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStoreBlobTest.java index 38b5fc9b1b..b7d7508f8b 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStoreBlobTest.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStoreBlobTest.java @@ -18,6 +18,7 @@ package org.apache.tika.parser.microsoft.onenote.fsshttpb; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -213,8 +214,7 @@ public class MSOneStoreBlobTest { assertNull(blob.getData()); blob.data = new BinaryItem(); blob.data.content.add((byte) 1); - // Exercise serialization of a non-null BLOB; the round-trip assertion is above. - blob.serializeToByteList(); + assertFalse(blob.serializeToByteList().isEmpty()); ObjectDataBLOB validBlob = new ObjectDataBLOB(); validBlob.data.content.add((byte) 1); @@ -263,6 +263,19 @@ public class MSOneStoreBlobTest { }); } + @Test + public void testDeclarationWithoutObjectDataFailsCleanly() { + ObjectGroupDataElementData mismatched = new ObjectGroupDataElementData(); + ObjectGroupObjectDeclare declaration = new ObjectGroupObjectDeclare(); + declaration.objectPartitionID.setDecodedValue(1); + mismatched.objectGroupDeclarations.objectDeclarationList.add(declaration); + + IOException e = assertThrows(IOException.class, + () -> RevisionStoreObjectGroup.createInstance(new ExGuid(9, UUID.randomUUID()), + mismatched, false, Collections.emptyMap())); + assertTrue(e.getMessage().contains("Missing object data")); + } + @Test public void testEncryptedObjectGroupAndMissingBlobReference() throws Exception { ObjectGroupDataElementData encryptedData = new ObjectGroupDataElementData(); diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStorePackageTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStorePackageTest.java index 4de6e92c12..720fed1ca4 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStorePackageTest.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStorePackageTest.java @@ -285,6 +285,91 @@ public class MSOneStorePackageTest { assertEquals("Иван Петров", metadata.get(OneNote.ORIGINAL_AUTHORS)); } + @Test + public void testDualRoleAuthorRecordedForBothRoles() throws Exception { + ExGuid authorId = id(710); + // the same author object referenced as both AuthorOriginal and AuthorMostRecent + RevisionStoreObject root = object(id(711), propertySet( + new PropertySpec(PropertyType.ObjectID, 0x20001D78, new NoData()), + new PropertySpec(PropertyType.ObjectID, 0x20001D79, new NoData())), + Arrays.asList(authorId, authorId), Collections.emptyList()); + RevisionStoreObject author = object(authorId, propertySet( + new PropertySpec(PropertyType.FourBytesOfLengthFollowedByData, + 0x1C001D75, utf16Text("Single Author"))), + Collections.emptyList(), Collections.emptyList()); + RevisionStoreCell cell = new RevisionStoreCell(); + cell.objectGroups.add(group(root, author)); + RevisionManifestRootDeclare rootDeclare = new RevisionManifestRootDeclare(); + rootDeclare.objectExGuid = root.objectID; + cell.rootDeclares.add(rootDeclare); + + MSOneStorePackage pkg = new MSOneStorePackage(); + pkg.cells.add(cell); + Metadata metadata = new Metadata(); + walk(pkg, metadata); + + assertEquals("Single Author", metadata.get(TikaCoreProperties.CREATOR)); + assertEquals("Single Author", metadata.get(OneNote.ORIGINAL_AUTHORS)); + assertEquals("Single Author", metadata.get(OneNote.MOST_RECENT_AUTHORS)); + } + + @Test + public void testBlobOnlyRootWithDanglingContentRootWalksAllObjects() throws Exception { + RevisionStoreObject blobRoot = object(id(620), propertySet(), + Collections.emptyList(), Collections.emptyList()); + blobRoot.propertySet = null; + blobRoot.fileDataObject = fileData("blob root"); + RevisionStoreObject textObject = object(id(621), propertySet( + new PropertySpec(PropertyType.FourBytesOfLengthFollowedByData, + 0x1C003498, text("page body text"))), + Collections.emptyList(), Collections.emptyList()); + RevisionStoreCell cell = new RevisionStoreCell(); + cell.objectGroups.add(group(blobRoot, textObject)); + RevisionManifestRootDeclare blobDeclare = new RevisionManifestRootDeclare(); + blobDeclare.objectExGuid = blobRoot.objectID; + cell.rootDeclares.add(blobDeclare); + RevisionManifestRootDeclare danglingContentRoot = new RevisionManifestRootDeclare(); + danglingContentRoot.objectExGuid = id(622); + cell.rootDeclares.add(danglingContentRoot); + MSOneStorePackage pkg = new MSOneStorePackage(); + pkg.cells.add(cell); + Metadata metadata = new Metadata(); + + // the blob root alone cannot reach the page body; the dangling content root must + // trigger the walk-everything fallback so the body is not lost + assertTrue(walk(pkg, metadata).contains("page body text")); + assertTrue(Arrays.stream(metadata.getValues(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING)) + .anyMatch(warning -> warning.contains("walking all objects"))); + } + + @Test + public void testHasEmittedContentTracksTextAndEmptyWalks() throws Exception { + MSOneStorePackage withText = new MSOneStorePackage(); + withText.cells.add(cellWithText(cell(70, 71), "some text")); + walk(withText); + assertTrue(withText.hasEmittedContent()); + + MSOneStorePackage empty = new MSOneStorePackage(); + RevisionStoreCell danglingCell = new RevisionStoreCell(); + RevisionManifestRootDeclare missingRoot = new RevisionManifestRootDeclare(); + missingRoot.objectExGuid = id(72); + danglingCell.rootDeclares.add(missingRoot); + empty.cells.add(danglingCell); + walk(empty); + assertFalse(empty.hasEmittedContent()); + } + + @Test + public void testSanitizeResourceNameKeepsBasenameOfPathShapedNames() { + assertEquals("pic 1.png", + MSOneStorePackage.sanitizeResourceName("D:\\images\\pic 1.png")); + assertEquals("pic.png", MSOneStorePackage.sanitizeResourceName("C:pic.png")); + assertEquals("a.png", MSOneStorePackage.sanitizeResourceName("/tmp/a.png")); + assertEquals("plain.png", MSOneStorePackage.sanitizeResourceName("plain.png")); + assertEquals("", MSOneStorePackage.sanitizeResourceName("..")); + assertEquals("", MSOneStorePackage.sanitizeResourceName("D:\\images\\..")); + } + @Test public void testBlobOnlyRootDoesNotFallBackToOtherObjects() throws Exception { RevisionStoreObject blobRoot = object(id(610), propertySet(), @@ -476,6 +561,8 @@ public class MSOneStorePackageTest { new java.util.HashMap<>(), new java.util.HashSet<>(), null, new OneNoteTreeWalkerOptions(), metadata, xhtml, 1000); xhtml.endDocument(); + // at the depth limit the walk must stop before emitting anything + assertTrue(writer.toString().isBlank()); } @Test
