This is an automated email from the ASF dual-hosted git repository.
tballison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git
The following commit(s) were added to refs/heads/main by this push:
new 2c946ccabd [TIKA-4944] Extract GUIDs from onenote files (#3278)
2c946ccabd is described below
commit 2c946ccabd13df305560a260fd2d098409fd8ae5
Author: Henry Lindeman <[email protected]>
AuthorDate: Thu Oct 8 13:08:29 2026 -0700
[TIKA-4944] Extract GUIDs from onenote files (#3278)
* Extract GUIDs from onenote files
Generated-by: Glean Tau <various models>
Signed-off-by: Henry Lindeman <[email protected]>
* AI Review fixes and jacoco
Generated-by: Glean Tau <various models>
Signed-off-by: Henry Lindeman <[email protected]>
* Move section guid from bag to scalar metadata, add pages metadata key
Generated-by: Glean Tay <various models>
Signed-off-by: Henry Lindeman <[email protected]>
* More PR Comments (drop entityGuids entirely)
Generated-by: Glean Tau <various models>
Signed-off-by: Henry Lindeman <[email protected]>
* More AI reviews
Generated-by: Glean Tau <various models>
Signed-off-by: Henry Lindeman <[email protected]>
* copilot comments
Signed-off-by: Henry Lindeman <[email protected]>
---------
Signed-off-by: Henry Lindeman <[email protected]>
---
CHANGES.txt | 9 +
.../java/org/apache/tika/metadata/OneNote.java | 29 +-
.../apache/tika/metadata/metadata-key-fields.json | 5 +
.../org/apache/tika/metadata/metadata-keys.json | 5 +
.../apache/tika/parser/microsoft/onenote/GUID.java | 24 +
.../microsoft/onenote/OneNoteGuidCollector.java | 141 +++
.../tika/parser/microsoft/onenote/OneNoteJcid.java | 30 +
.../parser/microsoft/onenote/OneNoteParser.java | 8 +
.../tika/parser/microsoft/onenote/OneNotePtr.java | 33 +-
.../microsoft/onenote/OneNoteTreeWalker.java | 83 +-
.../onenote/fsshttpb/MSOneStorePackage.java | 374 ++++++--
.../tika/parser/microsoft/onenote/GUIDTest.java | 20 +
.../onenote/OneNoteGuidCollectorTest.java | 106 +++
.../microsoft/onenote/OneNoteParserTest.java | 94 ++
.../onenote/OneNoteTreeWalkerGuidTest.java | 218 +++++
.../onenote/fsshttpb/MSOneStorePackageTest.java | 998 ++++++++++++++++++++-
16 files changed, 2083 insertions(+), 94 deletions(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index 2fcd701d48..7f7b5de69f 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -68,6 +68,15 @@ Release 4.2.0 - unreleased
Every resolver Tika installs answers with an empty stream, never null
(TIKA-4941).
* Deprecate XMLReaderUtils.getXMLInputFactory() and use SAX for XFA
(TIKA-4938).
+ * OneNote extracts a section GUID and page, page-series, page-node, and
conflict-page
+ GUID metadata from classic and FSSHTTPB files; the GUID bags are
deduplicated and
+ sorted, not page-ordered. Tree-collected GUID metadata is capped at
100,000 values
+ with a warning if the limit is reached. FSSHTTPB page divs now carry
+ id="{page GUID}" when a page GUID was recovered (fallback content can be
unwrapped).
+ FSSHTTPB root resolution is bounded at 10,000 roots per cell; when the
bound
+ truncates the declared roots before a content root, the walk falls back
to every
+ object in the cell so page content is not silently lost (TIKA-4944).
+
* Raise the default maxFilesProcessedPerProcess from 10,000 to 100,000;
forked pipes JVMs restart a tenth as often (TIKA-4950).
diff --git a/tika-core/src/main/java/org/apache/tika/metadata/OneNote.java
b/tika-core/src/main/java/org/apache/tika/metadata/OneNote.java
index 0077753c8b..e635825aca 100644
--- a/tika-core/src/main/java/org/apache/tika/metadata/OneNote.java
+++ b/tika-core/src/main/java/org/apache/tika/metadata/OneNote.java
@@ -18,10 +18,20 @@ package org.apache.tika.metadata;
/**
* OneNote file-header and revision metadata. Key suffixes are the
[MS-ONESTORE]/[MS-FSSHTTPB]
- * field names verbatim (external-standard spellings), under the {@code
onenote:} namespace.
+ * field names verbatim (external-standard spellings), under the {@code
onenote:} namespace,
+ * except the GUID keys below, which are projected from the source object that
carries the
+ * value (the section file identity, or a NotebookManagementEntityGuid keyed
by the carrying
+ * object's JCID).
* Previously these were minted per parse via the registering Property
factories
* (TIKA-4816 round-3 review: per-record global-lock interning); a bounded,
spec-defined
* vocabulary belongs in curated constants.
+ *
+ * <p>The GUID bags (pageGuids, pageSeriesGuids, pageNodeGuids,
conflictPageGuids) are
+ * unordered, deduplicated, lexically sorted inventories: do not join them
positionally
+ * to each other or to page content, and do not treat values as unique across
files.
+ * Page order and page-to-content association are available only through the
FSSHTTPB
+ * XHTML output, where each recognized page is wrapped in a div whose id is
the page
+ * GUID when one was recovered (fallback content can be unwrapped).
*/
public interface OneNote {
@@ -58,6 +68,23 @@ public interface OneNote {
Property MOST_RECENT_AUTHORS = Property.externalTextBag(PREFIX +
"mostRecentAuthors");
Property ORIGINAL_AUTHORS = Property.externalTextBag(PREFIX +
"originalAuthors");
+
+ /**
+ * GUID of the section file: guidFile from the classic header, or the first
+ * FileIdentityGuid property found in a newer-format package.
+ */
+ Property SECTION_GUID = Property.externalText(PREFIX + "sectionGuid");
+ /**
+ * GUIDs of jcidPageMetadata (0x30) objects. Classic parsing records only
pages referenced
+ * via MetaDataObjectsAboveGraphSpace; FSSHTTPB fallback walks may still
include older copies.
+ */
+ Property PAGE_GUIDS = Property.externalTextBag(PREFIX + "pageGuids");
+ /** GUIDs of jcidPageSeriesNode (0x08) objects. */
+ Property PAGE_SERIES_GUIDS = Property.externalTextBag(PREFIX +
"pageSeriesGuids");
+ /** GUIDs of jcidPageNode (0x0B) objects; source-object identifiers, not
cross-file page keys. */
+ Property PAGE_NODE_GUIDS = Property.externalTextBag(PREFIX +
"pageNodeGuids");
+ /** GUIDs of jcidConflictPageMetadata (0x38) objects. */
+ Property CONFLICT_PAGE_GUIDS = Property.externalTextBag(PREFIX +
"conflictPageGuids");
Property CREATION_TIMESTAMP = Property.externalText(PREFIX +
"creationTimestamp");
Property LAST_MODIFIED_TIMESTAMP = Property.externalText(PREFIX +
"lastModifiedTimestamp");
}
diff --git
a/tika-metadata-schema/src/main/resources/org/apache/tika/metadata/metadata-key-fields.json
b/tika-metadata-schema/src/main/resources/org/apache/tika/metadata/metadata-key-fields.json
index f884866b7c..2926590d46 100644
---
a/tika-metadata-schema/src/main/resources/org/apache/tika/metadata/metadata-key-fields.json
+++
b/tika-metadata-schema/src/main/resources/org/apache/tika/metadata/metadata-key-fields.json
@@ -377,6 +377,7 @@
{"class":"org.apache.tika.metadata.OneNote","field":"CB_FREE_SPACE_IN_FREE_CHUNK_LIST","key":"onenote:cbFreeSpaceInFreeChunkList"},
{"class":"org.apache.tika.metadata.OneNote","field":"CB_LEGACY_EXPECTED_FILE_LENGTH","key":"onenote:cbLegacyExpectedFileLength"},
{"class":"org.apache.tika.metadata.OneNote","field":"CB_LEGACY_FREE_SPACE_IN_FREE_CHUNK_LIST","key":"onenote:cbLegacyFreeSpaceInFreeChunkList"},
+
{"class":"org.apache.tika.metadata.OneNote","field":"CONFLICT_PAGE_GUIDS","key":"onenote:conflictPageGuids"},
{"class":"org.apache.tika.metadata.OneNote","field":"CRC_NAME","key":"onenote:crcName"},
{"class":"org.apache.tika.metadata.OneNote","field":"CREATION_TIMESTAMP","key":"onenote:creationTimestamp"},
{"class":"org.apache.tika.metadata.OneNote","field":"C_TRANSACTIONS_IN_LOG","key":"onenote:cTransactionsInLog"},
@@ -389,7 +390,11 @@
{"class":"org.apache.tika.metadata.OneNote","field":"MOST_RECENT_AUTHORS","key":"onenote:mostRecentAuthors"},
{"class":"org.apache.tika.metadata.OneNote","field":"N_FILE_VERSION_GENERATION","key":"onenote:nFileVersionGeneration"},
{"class":"org.apache.tika.metadata.OneNote","field":"ORIGINAL_AUTHORS","key":"onenote:originalAuthors"},
+
{"class":"org.apache.tika.metadata.OneNote","field":"PAGE_GUIDS","key":"onenote:pageGuids"},
+
{"class":"org.apache.tika.metadata.OneNote","field":"PAGE_NODE_GUIDS","key":"onenote:pageNodeGuids"},
+
{"class":"org.apache.tika.metadata.OneNote","field":"PAGE_SERIES_GUIDS","key":"onenote:pageSeriesGuids"},
{"class":"org.apache.tika.metadata.OneNote","field":"RGB_PLACEHOLDER","key":"onenote:rgbPlaceholder"},
+
{"class":"org.apache.tika.metadata.OneNote","field":"SECTION_GUID","key":"onenote:sectionGuid"},
{"class":"org.apache.tika.metadata.PDF","field":"ACTION_TRIGGER","key":"pdf:action-trigger"},
{"class":"org.apache.tika.metadata.PDF","field":"ACTION_TRIGGERS","key":"pdf:action-triggers"},
{"class":"org.apache.tika.metadata.PDF","field":"ACTION_TYPES","key":"pdf:action-types"},
diff --git
a/tika-metadata-schema/src/main/resources/org/apache/tika/metadata/metadata-keys.json
b/tika-metadata-schema/src/main/resources/org/apache/tika/metadata/metadata-keys.json
index 95d4221cb7..3f57a2f533 100644
---
a/tika-metadata-schema/src/main/resources/org/apache/tika/metadata/metadata-keys.json
+++
b/tika-metadata-schema/src/main/resources/org/apache/tika/metadata/metadata-keys.json
@@ -401,6 +401,7 @@
{"key":"onenote:cbFreeSpaceInFreeChunkList","namespace":"onenote","valueType":"TEXT","cardinality":"SIMPLE","module":"tika-core"},
{"key":"onenote:cbLegacyExpectedFileLength","namespace":"onenote","valueType":"TEXT","cardinality":"SIMPLE","module":"tika-core"},
{"key":"onenote:cbLegacyFreeSpaceInFreeChunkList","namespace":"onenote","valueType":"TEXT","cardinality":"SIMPLE","module":"tika-core"},
+
{"key":"onenote:conflictPageGuids","namespace":"onenote","valueType":"TEXT","cardinality":"BAG","module":"tika-core"},
{"key":"onenote:crcName","namespace":"onenote","valueType":"TEXT","cardinality":"SIMPLE","module":"tika-core"},
{"key":"onenote:creationTimestamp","namespace":"onenote","valueType":"TEXT","cardinality":"SIMPLE","module":"tika-core"},
{"key":"onenote:ffvLastCodeThatWroteToThisFile","namespace":"onenote","valueType":"TEXT","cardinality":"SIMPLE","module":"tika-core"},
@@ -412,7 +413,11 @@
{"key":"onenote:mostRecentAuthors","namespace":"onenote","valueType":"TEXT","cardinality":"BAG","module":"tika-core"},
{"key":"onenote:nFileVersionGeneration","namespace":"onenote","valueType":"TEXT","cardinality":"SIMPLE","module":"tika-core"},
{"key":"onenote:originalAuthors","namespace":"onenote","valueType":"TEXT","cardinality":"BAG","module":"tika-core"},
+
{"key":"onenote:pageGuids","namespace":"onenote","valueType":"TEXT","cardinality":"BAG","module":"tika-core"},
+
{"key":"onenote:pageNodeGuids","namespace":"onenote","valueType":"TEXT","cardinality":"BAG","module":"tika-core"},
+
{"key":"onenote:pageSeriesGuids","namespace":"onenote","valueType":"TEXT","cardinality":"BAG","module":"tika-core"},
{"key":"onenote:rgbPlaceholder","namespace":"onenote","valueType":"TEXT","cardinality":"SIMPLE","module":"tika-core"},
+
{"key":"onenote:sectionGuid","namespace":"onenote","valueType":"TEXT","cardinality":"SIMPLE","module":"tika-core"},
{"key":"pdf:action-trigger","namespace":"pdf","valueType":"TEXT","cardinality":"SIMPLE","module":"tika-core"},
{"key":"pdf:action-triggers","namespace":"pdf","valueType":"TEXT","cardinality":"BAG","module":"tika-core"},
{"key":"pdf:action-types","namespace":"pdf","valueType":"TEXT","cardinality":"BAG","module":"tika-core"},
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/GUID.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/GUID.java
index 55e788bb4a..04377e7e70 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/GUID.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/GUID.java
@@ -34,6 +34,30 @@ public class GUID implements Comparable<GUID> {
this.guid = guid;
}
+ /**
+ * Decodes a 16-byte GUID stored in the mixed-endian layout used by
MS-DTYP.
+ *
+ * @return the GUID, or {@code null} if {@code data} is not exactly 16
bytes
+ */
+ public static GUID fromMicrosoftBytes(byte[] data) {
+ if (data == null || data.length != 16) {
+ return null;
+ }
+ int[] guid = new int[16];
+ guid[0] = data[3] & 0xff;
+ guid[1] = data[2] & 0xff;
+ guid[2] = data[1] & 0xff;
+ guid[3] = data[0] & 0xff;
+ guid[4] = data[5] & 0xff;
+ guid[5] = data[4] & 0xff;
+ guid[6] = data[7] & 0xff;
+ guid[7] = data[6] & 0xff;
+ for (int i = 8; i < 16; i++) {
+ guid[i] = data[i] & 0xff;
+ }
+ return new GUID(guid);
+ }
+
/**
* Converts a GUID of format: {AAAAAAAA-BBBB-CCCC-DDDD-EEEEEEEEEEEE} (in
bytes) to a GUID
* object.
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/OneNoteGuidCollector.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/OneNoteGuidCollector.java
new file mode 100644
index 0000000000..e99bec5d3f
--- /dev/null
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/OneNoteGuidCollector.java
@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.microsoft.onenote;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.EnumMap;
+import java.util.LinkedHashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Consumer;
+
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.OneNote;
+import org.apache.tika.metadata.Property;
+
+/**
+ * Collects the GUID metadata of one OneNote section for the classic and
FSSHTTPB parsers.
+ * Values are deduplicated per key and capped at {@link #MAX_GUID_COUNT} in
total.
+ */
+public final class OneNoteGuidCollector {
+
+ public static final int MAX_GUID_COUNT = 100_000;
+
+ public enum Category {
+ SECTION(OneNote.SECTION_GUID),
+ PAGE(OneNote.PAGE_GUIDS),
+ PAGE_SERIES(OneNote.PAGE_SERIES_GUIDS),
+ PAGE_NODE(OneNote.PAGE_NODE_GUIDS),
+ CONFLICT_PAGE(OneNote.CONFLICT_PAGE_GUIDS);
+
+ private final Property property;
+
+ Category(Property property) {
+ this.property = property;
+ }
+ }
+
+ private final Map<Category, LinkedHashSet<String>> values = new
EnumMap<>(Category.class);
+ private final Consumer<String> warningHandler;
+ private int count;
+ private boolean limitWarningReported;
+
+ public OneNoteGuidCollector(Consumer<String> warningHandler) {
+ this.warningHandler = warningHandler;
+ for (Category category : Category.values()) {
+ values.put(category, new LinkedHashSet<>());
+ }
+ }
+
+ public void add(Category category, String guid) {
+ if (guid == null) {
+ return;
+ }
+ Set<String> categoryValues = values.get(category);
+ if (categoryValues.contains(guid)
+ || (category == Category.SECTION &&
!categoryValues.isEmpty())) {
+ return;
+ }
+ if (count >= MAX_GUID_COUNT) {
+ reportLimitWarning();
+ return;
+ }
+ categoryValues.add(guid);
+ count++;
+ if (count == MAX_GUID_COUNT) {
+ reportLimitWarning();
+ }
+ }
+
+ /**
+ * Records a NotebookManagementEntityGuid under the key for the JCID of
the object that
+ * carries it. Section-node objects and unrecognized JCIDs are skipped.
+ */
+ public void addForObjectType(int jcid, String guid) {
+ Category category = categoryFor(jcid);
+ if (category != null) {
+ add(category, guid);
+ }
+ }
+
+ static Category categoryFor(int jcid) {
+ switch (jcid) {
+ case OneNoteJcid.PAGE_METADATA:
+ return Category.PAGE;
+ case OneNoteJcid.PAGE_SERIES_NODE:
+ return Category.PAGE_SERIES;
+ case OneNoteJcid.PAGE_NODE:
+ return Category.PAGE_NODE;
+ case OneNoteJcid.CONFLICT_PAGE_METADATA:
+ return Category.CONFLICT_PAGE;
+ default:
+ return null;
+ }
+ }
+
+ private void reportLimitWarning() {
+ if (!limitWarningReported) {
+ limitWarningReported = true;
+ warningHandler.accept("Capping OneNote GUID metadata at " +
MAX_GUID_COUNT +
+ " distinct values");
+ }
+ }
+
+ public Set<String> values(Category category) {
+ return Collections.unmodifiableSet(values.get(category));
+ }
+
+ public boolean isFull() {
+ return count >= MAX_GUID_COUNT;
+ }
+
+ /** Sets each non-empty key on {@code metadata}, with values sorted. */
+ public void publish(Metadata metadata) {
+ for (Map.Entry<Category, LinkedHashSet<String>> entry :
values.entrySet()) {
+ if (!entry.getValue().isEmpty()) {
+ if (entry.getKey() == Category.SECTION) {
+ metadata.set(entry.getKey().property,
entry.getValue().iterator().next());
+ } else {
+ String[] sorted = entry.getValue().toArray(new String[0]);
+ Arrays.sort(sorted);
+ metadata.set(entry.getKey().property, sorted);
+ }
+ }
+ }
+ }
+}
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/OneNoteJcid.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/OneNoteJcid.java
new file mode 100644
index 0000000000..d4f87508e0
--- /dev/null
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/OneNoteJcid.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.microsoft.onenote;
+
+/** JCID index values from MS-ONE 2.1.13. */
+public final class OneNoteJcid {
+
+ public static final int SECTION_NODE = 0x07;
+ public static final int PAGE_SERIES_NODE = 0x08;
+ public static final int PAGE_NODE = 0x0B;
+ public static final int PAGE_METADATA = 0x30;
+ public static final int CONFLICT_PAGE_METADATA = 0x38;
+
+ private OneNoteJcid() {
+ }
+}
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/OneNoteParser.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/OneNoteParser.java
index 2e4935016b..4a4399c41b 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/OneNoteParser.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/OneNoteParser.java
@@ -129,6 +129,7 @@ public class OneNoteParser implements Parser {
"0x" +
Long.toHexString(oneNoteDocument.header.nFileVersionGeneration));
metadata.set(OneNote.RGB_PLACEHOLDER,
"0x" +
Long.toHexString(oneNoteDocument.header.rgbPlaceholder));
+ emitSectionFileGuid(header, metadata);
Exception structureFailure =
oneNoteDocument.structureParseException;
boolean walked = false;
@@ -211,6 +212,13 @@ public class OneNoteParser implements Parser {
}
+ static void emitSectionFileGuid(OneNoteHeader header, Metadata metadata) {
+ GUID guidFile = header.getGuidFile();
+ if (guidFile != null && !guidFile.equals(GUID.nil())) {
+ metadata.set(OneNote.SECTION_GUID, guidFile.toString());
+ }
+ }
+
private static String[] sortedValues(Set<String> values) {
String[] sorted = values.toArray(new String[0]);
Arrays.sort(sorted);
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/OneNotePtr.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/OneNotePtr.java
index 916b9bbb5a..e379f3564f 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/OneNotePtr.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/OneNotePtr.java
@@ -148,33 +148,18 @@ class OneNotePtr {
}
private GUID deserializeGUID() throws IOException {
- int[] guid = new int[16];
+ byte[] guidBytes = new byte[16];
for (int i = 0; i < 16; ++i) {
- guid[i] = dif.read();
+ int next = dif.read();
+ if (next < 0) {
+ // (byte) -1 would read as 0xFF and silently decode a wrong
GUID
+ throw new IOException("End of stream reached while reading a
GUID");
+ }
+ guidBytes[i] = (byte) next;
}
- int[] guid2 = new int[16];
- // re-order [0,1,2,3] to little endian
- guid2[0] = guid[3];
- guid2[1] = guid[2];
- guid2[2] = guid[1];
- guid2[3] = guid[0];
- // re-order [4,5,6,7] to little endian
- guid2[4] = guid[5];
- guid2[5] = guid[4];
- guid2[6] = guid[7];
- guid2[7] = guid[6];
- // the rest is already in right order.
- guid2[8] = guid[8];
- guid2[9] = guid[9];
- guid2[10] = guid[10];
- guid2[11] = guid[11];
- guid2[12] = guid[12];
- guid2[13] = guid[13];
- guid2[14] = guid[14];
- guid2[15] = guid[15];
-
+ GUID guid = GUID.fromMicrosoftBytes(guidBytes);
offset = dif.position();
- return new GUID(guid2);
+ return guid;
}
private byte[] deserializedReservedHeader() throws IOException {
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/OneNoteTreeWalker.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/OneNoteTreeWalker.java
index a4c3902c0c..116ed2ba49 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/OneNoteTreeWalker.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/OneNoteTreeWalker.java
@@ -96,6 +96,7 @@ class OneNoteTreeWalker {
private final Set<String> authors = new HashSet<>();
private final Set<String> mostRecentAuthors = new HashSet<>();
private final Set<String> originalAuthors = new HashSet<>();
+ private final OneNoteGuidCollector guidCollector;
private final OneNoteTreeWalkerOptions options;
private final OneNoteDocument oneNoteDocument;
private final OneNoteDirectFileResource dif;
@@ -143,6 +144,7 @@ class OneNoteTreeWalker {
this.xhtml = xhtml;
this.parentMetadata = parentMetadata;
this.parseContext = parseContext;
+ this.guidCollector = new
OneNoteGuidCollector(this::recordGuidCapWarning);
this.embeddedDocumentExtractor =
EmbeddedDocumentUtil.getEmbeddedDocumentExtractor(parseContext);
}
@@ -156,8 +158,12 @@ class OneNoteTreeWalker {
public Map<String, Object> walkTree() throws IOException, TikaException,
SAXException {
Map<String, Object> structure = new HashMap<>();
structure.put("header", oneNoteDocument.header);
- structure.put("rootFileNodes", walkRootFileNodes());
- return structure;
+ try {
+ structure.put("rootFileNodes", walkRootFileNodes());
+ return structure;
+ } finally {
+ publishGuidBags();
+ }
}
/**
@@ -384,7 +390,8 @@ class OneNoteTreeWalker {
structure.put("childFileNodeList",
walkFileNodeList(fileNode.childFileNodeList, parentPropertyId));
}
if (fileNode.propertySet != null) {
- List<Map<String, Object>> propSet =
processPropertySet(fileNode.propertySet, parentPropertyId);
+ List<Map<String, Object>> propSet =
+ processPropertySet(fileNode.propertySet, parentPropertyId,
jcidIndex(fileNode));
if (!propSet.isEmpty()) {
structure.put("propertySet", propSet);
}
@@ -461,12 +468,13 @@ class OneNoteTreeWalker {
* @throws IOException Can throw these when manipulating the seekable byte
channel.
*/
private List<Map<String, Object>> processPropertySet(PropertySet
propertySet,
- OneNotePropertyId
parentPropertyId)
+ OneNotePropertyId
parentPropertyId,
+ int objectType)
throws IOException, TikaException, SAXException {
List<Map<String, Object>> propValues = new ArrayList<>();
for (int i = 0; i < propertySet.rgPridsData.size(); ++i) {
PropertyValue propertyValue = propertySet.rgPridsData.get(i);
- propValues.add(processPropertyValue(propertyValue,
parentPropertyId));
+ propValues.add(processPropertyValue(propertyValue,
parentPropertyId, objectType));
}
return propValues;
}
@@ -495,8 +503,9 @@ class OneNoteTreeWalker {
* @return The map parsed by this property value.
* @throws IOException Can throw these when manipulating the seekable byte
channel.
*/
- private Map<String, Object> processPropertyValue(PropertyValue
propertyValue,
- OneNotePropertyId
parentPropertyId)
+ Map<String, Object> processPropertyValue(PropertyValue propertyValue,
+ OneNotePropertyId
parentPropertyId,
+ int objectType)
throws IOException, TikaException, SAXException {
Map<String, Object> propMap = new HashMap<>();
propMap.put("oneNoteType", "PropertyValue");
@@ -615,6 +624,20 @@ class OneNoteTreeWalker {
// have the onlyLatestRevision = false
handleRichEditTextUnicode(content.size());
}
+ } else if (propertyValue.propertyId.propertyEnum ==
+ OneNotePropertyEnum.NotebookManagementEntityGuid
+ && content.size() == 16) {
+ ByteBuffer guidBuffer = ByteBuffer.allocate(16);
+ dif.read(guidBuffer);
+ String guid =
pooledGuid(GUID.fromMicrosoftBytes(guidBuffer.array()).toString());
+ propMap.put("notebookManagementEntityGuid", guid);
+ // Version-history copies of a page are object-space
roots; only the
+ // current page's metadata is referenced via
MetaDataObjectsAboveGraphSpace.
+ if (objectType != OneNoteJcid.PAGE_METADATA ||
(parentPropertyId != null
+ && parentPropertyId.propertyEnum ==
+
OneNotePropertyEnum.MetaDataObjectsAboveGraphSpace)) {
+ addClassicEntityGuid(objectType, guid);
+ }
} else {
//TODO -- these seem to be somewhat broken font files and
other
//odds and ends...what are they and how should we process
them?
@@ -633,7 +656,8 @@ class OneNoteTreeWalker {
}
}
if (propertyValue.propertySet != null &&
propertyValue.propertySet.rgPridsData != null) {
- List<Map<String, Object>> propSet =
processPropertySet(propertyValue.propertySet, parentPropertyId);
+ List<Map<String, Object>> propSet =
+ processPropertySet(propertyValue.propertySet,
parentPropertyId, objectType);
if (!propSet.isEmpty()) {
propMap.put("propertySet", propSet);
}
@@ -703,6 +727,49 @@ class OneNoteTreeWalker {
}
}
+ void addClassicEntityGuid(int objectType, String guid) {
+ guidCollector.addForObjectType(objectType, guid);
+ }
+
+ private void publishGuidBags() {
+ if (parentMetadata != null) {
+ guidCollector.publish(parentMetadata);
+ }
+ }
+
+ /**
+ * Exempt from any warning budget: capped GUID metadata must always carry
its own
+ * incompleteness signal.
+ */
+ private void recordGuidCapWarning(String warning) {
+ LOG.warn(warning);
+ if (parentMetadata != null) {
+ parentMetadata.add(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING,
warning);
+ }
+ }
+
+ // revisit-heavy graphs decode the same GUID many times; pool the
canonical string so the
+ // retained per-node maps share one instance instead of one string per
visit
+ final Map<String, String> guidStringPool = new HashMap<>();
+
+ String pooledGuid(String guid) {
+ if (guidStringPool.size() >= OneNoteGuidCollector.MAX_GUID_COUNT) {
+ return guid;
+ }
+ String pooled = guidStringPool.putIfAbsent(guid, guid);
+ return pooled != null ? pooled : guid;
+ }
+
+ static int jcidIndex(FileNode fileNode) {
+ if (fileNode == null || fileNode.subType == null
+ || fileNode.subType.objectDeclarationWithRefCount == null
+ || fileNode.subType.objectDeclarationWithRefCount.body == null
+ || fileNode.subType.objectDeclarationWithRefCount.body.jcid ==
null) {
+ return -1;
+ }
+ return (int)
fileNode.subType.objectDeclarationWithRefCount.body.jcid.index;
+ }
+
public Set<String> getAuthors() {
return authors;
}
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStorePackage.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStorePackage.java
index ca1a724ee1..a5cad9d413 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStorePackage.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStorePackage.java
@@ -33,6 +33,7 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.function.Consumer;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -49,6 +50,9 @@ import org.apache.tika.metadata.Metadata;
import org.apache.tika.metadata.OneNote;
import org.apache.tika.metadata.TikaCoreProperties;
import org.apache.tika.parser.ParseContext;
+import org.apache.tika.parser.microsoft.onenote.GUID;
+import org.apache.tika.parser.microsoft.onenote.OneNoteGuidCollector;
+import org.apache.tika.parser.microsoft.onenote.OneNoteJcid;
import org.apache.tika.parser.microsoft.onenote.OneNotePropertyEnum;
import org.apache.tika.parser.microsoft.onenote.OneNoteTreeWalkerOptions;
import org.apache.tika.parser.microsoft.onenote.fsshttpb.property.ArrayNumber;
@@ -99,6 +103,10 @@ public class MSOneStorePackage {
private static final int MAX_REFERENCE_COUNT = 100000;
private static final int MAX_PARSE_WARNINGS = 100;
+ /** Defensive bound on root objects resolved per cell; real cells declare
a handful. */
+ static final int MAX_CELL_ROOTS = 10_000;
+ private static final int FILE_IDENTITY_GUID_PROPERTY_ID = 0x1C001D94;
+
static {
LocalDateTime time32Epoch1980 = LocalDateTime.of(1980, Month.JANUARY,
1, 0, 0);
Instant instant = time32Epoch1980.atZone(ZoneOffset.UTC).toInstant();
@@ -114,6 +122,7 @@ public class MSOneStorePackage {
private final Set<String> authors = new HashSet<>();
private final Set<String> mostRecentAuthors = new HashSet<>();
private final Set<String> originalAuthors = new HashSet<>();
+ private final OneNoteGuidCollector guids = new
OneNoteGuidCollector(this::recordGuidCapWarning);
/**
* The fully populated storage index. Set this before performing
storage-index lookups; its
* mapping lists are indexed once and must not be mutated afterward.
@@ -235,45 +244,71 @@ public class MSOneStorePackage {
metadata.add(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING,
warning);
}
parseWarnings.clear();
- if (!cells.isEmpty()) {
- // Walk each page cell (object space) as a tree, starting from the
root objects of
- // its current revision and following the object references in
property order. This
- // emits the text in document order. The pages are walked in the
order in which the
- // section object space references them; cells that hold older
versions of a page
- // (the same object space in a different revision context) are
skipped.
- List<RevisionStoreCell> pageCells = new ArrayList<>();
- List<RevisionStoreCell> otherCells = new ArrayList<>();
- splitCells(pageCells, otherCells);
- for (RevisionStoreCell cell : pageCells) {
- xhtml.startElement("div", "class", "page");
- try {
- walkCell(cell, options, metadata, xhtml);
- } finally {
- xhtml.endElement("div");
+ try {
+ collectSectionFileIdentityGuids();
+ if (!cells.isEmpty()) {
+ // Walk each page cell (object space) as a tree, starting from
the root objects of
+ // its current revision and following the object references in
property order. This
+ // emits the text in document order. The pages are walked in
the order in which the
+ // section object space references them; cells that hold older
versions of a page
+ // (the same object space in a different revision context) are
skipped.
+ List<RevisionStoreCell> pageCells = new ArrayList<>();
+ List<RevisionStoreCell> otherCells = new ArrayList<>();
+ splitCells(pageCells, otherCells);
+ for (RevisionStoreCell cell : pageCells) {
+ emitPage(cell, options, metadata, xhtml);
+ }
+ for (RevisionStoreCell cell : otherCells) {
+ walkCell(cell, indexObjectsById(cell.objectGroups),
options, metadata, xhtml);
+ }
+ } else {
+ // no cell information available - walk the object groups in
revision order
+ Map<ExGuid, RevisionStoreObject> objectsById =
indexObjectsById(OtherFileNodeList);
+ Set<ExGuid> visited = new HashSet<>();
+ for (RevisionStoreObjectGroup objectGroup : OtherFileNodeList)
{
+ if (objectGroup == null || objectGroup.objects == null) {
+ continue;
+ }
+ for (RevisionStoreObject object : objectGroup.objects) {
+ walkObject(object, objectsById, visited,
AuthorRole.NONE, options, metadata,
+ xhtml, 0);
+ }
}
}
- for (RevisionStoreCell cell : otherCells) {
- walkCell(cell, options, metadata, xhtml);
+ if (!authors.isEmpty()) {
+ metadata.set(TikaCoreProperties.CREATOR,
sortedValues(authors));
}
- } else {
- // no cell information available - walk the object groups in
revision order
- Map<ExGuid, RevisionStoreObject> objectsById =
indexObjectsById(OtherFileNodeList);
- Set<ExGuid> visited = new HashSet<>();
- for (RevisionStoreObjectGroup objectGroup : OtherFileNodeList) {
- for (RevisionStoreObject object : objectGroup.objects) {
- walkObject(object, objectsById, visited, AuthorRole.NONE,
options, metadata,
- xhtml, 0);
- }
+ if (!mostRecentAuthors.isEmpty()) {
+ metadata.set(OneNote.MOST_RECENT_AUTHORS,
sortedValues(mostRecentAuthors));
}
+ if (!originalAuthors.isEmpty()) {
+ metadata.set(OneNote.ORIGINAL_AUTHORS,
sortedValues(originalAuthors));
+ }
+ } finally {
+ guids.publish(metadata);
}
- if (!authors.isEmpty()) {
- metadata.set(TikaCoreProperties.CREATOR, sortedValues(authors));
- }
- if (!mostRecentAuthors.isEmpty()) {
- metadata.set(OneNote.MOST_RECENT_AUTHORS,
sortedValues(mostRecentAuthors));
+ }
+
+ OneNoteGuidCollector guidCollector() {
+ return guids;
+ }
+
+ void emitPage(RevisionStoreCell cell, OneNoteTreeWalkerOptions options,
+ Metadata metadata, XHTMLContentHandler xhtml)
+ throws SAXException, TikaException, IOException {
+ Map<ExGuid, RevisionStoreObject> objectsById =
indexObjectsById(cell.objectGroups);
+ String pageGuid = findPageGuid(cell, objectsById);
+ guids.add(OneNoteGuidCollector.Category.PAGE, pageGuid);
+ AttributesImpl attributes = new AttributesImpl();
+ attributes.addAttribute("", "class", "class", "CDATA", "page");
+ if (pageGuid != null) {
+ attributes.addAttribute("", "id", "id", "CDATA", pageGuid);
}
- if (!originalAuthors.isEmpty()) {
- metadata.set(OneNote.ORIGINAL_AUTHORS,
sortedValues(originalAuthors));
+ xhtml.startElement("div", attributes);
+ try {
+ walkCell(cell, objectsById, options, metadata, xhtml);
+ } finally {
+ xhtml.endElement("div");
}
}
@@ -355,6 +390,7 @@ public class MSOneStorePackage {
if (object.objectID != null && !visited.add(object.objectID)) {
return;
}
+ recordEntityGuid(object, jcidIndex(object));
List<PropertyAction> actions = collectObjectActions(object);
for (PropertyAction action : actions) {
if (action.spaceReference != null) {
@@ -370,36 +406,28 @@ public class MSOneStorePackage {
}
}
- private void walkCell(RevisionStoreCell cell, OneNoteTreeWalkerOptions
options,
- Metadata metadata, XHTMLContentHandler xhtml)
+ private void walkCell(RevisionStoreCell cell,
+ Map<ExGuid, RevisionStoreObject> objectsById,
+ OneNoteTreeWalkerOptions options, Metadata metadata,
+ XHTMLContentHandler xhtml)
throws SAXException, TikaException, IOException {
- Map<ExGuid, RevisionStoreObject> objectsById =
indexObjectsById(cell.objectGroups);
Set<ExGuid> visited = new HashSet<>();
// Only objects reachable from the root objects of the current
revision are part of
// the current content. The object groups may also contain older,
superseded versions
// of objects (under a different object ID); those are intentionally
not walked.
- // A blob-only root (no property set) cannot reach the page body, so
it does not
- // count as a resolved content root: if the content root declare
dangles next to it,
- // the walk-everything fallback must still fire or the page body is
silently lost.
- boolean resolvedContentRoot = false;
- boolean unresolvedRoot = false;
- for (RevisionManifestRootDeclare rootDeclare : cell.rootDeclares) {
- RevisionStoreObject rootObject =
objectsById.get(rootDeclare.objectExGuid);
- if (rootObject == null) {
- recordParseWarning("OneNote cell root object " +
rootDeclare.objectExGuid +
- " could not be resolved");
- unresolvedRoot = true;
- } else {
- if (rootObject.propertySet != null) {
- resolvedContentRoot = true;
- }
- walkObject(rootObject, objectsById, visited, AuthorRole.NONE,
options,
- metadata, xhtml, 0);
- }
+ CellRoots roots = cellRoots(cell, objectsById,
this::recordParseWarning);
+ for (RevisionStoreObject rootObject : roots.declared) {
+ walkObject(rootObject, objectsById, visited, AuthorRole.NONE,
options, metadata,
+ xhtml, 0);
}
- if (cell.rootDeclares.isEmpty() || (unresolvedRoot &&
!resolvedContentRoot)) {
- if (cell.rootDeclares.isEmpty()) {
+ if (roots.walkAll) {
+ if (cell.rootDeclares != null && cell.rootDeclares.isEmpty()) {
recordParseWarning("OneNote cell has no declared root objects;
walking all objects");
+ } else if (roots.truncated) {
+ // reachable only when no content root survived the cap, so
the plain
+ // check is enough here
+ recordParseWarning("OneNote cell has more than " +
MAX_CELL_ROOTS
+ + " root objects; walking all objects");
} else {
recordParseWarning("OneNote cell root objects could not be
resolved; walking all objects");
}
@@ -412,6 +440,57 @@ public class MSOneStorePackage {
}
}
+ /**
+ * Resolves the objects a cell walk starts from: the root objects declared
by the current
+ * revision, then, when the declares are empty or dangle without a
property-set root, every
+ * object in the groups. A blob-only root (no property set) cannot reach
the page body, so
+ * it does not count as a resolved content root: if the content root
declare dangles next
+ * to it, the walk-everything fallback must still fire or the page body is
silently lost.
+ * The same holds when the root cap truncates the declares before reaching
a content root.
+ */
+ private CellRoots cellRoots(RevisionStoreCell cell,
+ Map<ExGuid, RevisionStoreObject> objectsById,
+ Consumer<String> unresolvedRootWarning) {
+ CellRoots roots = new CellRoots();
+ if (cell.rootDeclares != null) {
+ Set<RevisionStoreObject> seenRoots = new HashSet<>();
+ for (RevisionManifestRootDeclare rootDeclare : cell.rootDeclares) {
+ RevisionStoreObject rootObject =
objectsById.get(rootDeclare.objectExGuid);
+ if (rootObject == null) {
+ unresolvedRootWarning.accept("OneNote cell root object " +
+ rootDeclare.objectExGuid + " could not be
resolved");
+ roots.unresolvedRoot = true;
+ } else if (seenRoots.add(rootObject)) {
+ if (roots.declared.size() < MAX_CELL_ROOTS) {
+ if (rootObject.propertySet != null) {
+ roots.resolvedContentRoot = true;
+ }
+ roots.declared.add(rootObject);
+ } else if (!roots.truncated) {
+ roots.truncated = true;
+ unresolvedRootWarning.accept("OneNote cell has more
than " + MAX_CELL_ROOTS
+ + " root objects; ignoring the rest");
+ }
+ }
+ }
+ }
+ roots.walkAll = cell.rootDeclares == null ||
cell.rootDeclares.isEmpty()
+ || ((roots.unresolvedRoot || roots.truncated) &&
!roots.resolvedContentRoot);
+ return roots;
+ }
+
+ /**
+ * Root resolution shared by {@link #walkCell} and
+ * {@link #findPageGuid(RevisionStoreCell, Map)}.
+ */
+ private static final class CellRoots {
+ final List<RevisionStoreObject> declared = new ArrayList<>();
+ boolean resolvedContentRoot;
+ boolean unresolvedRoot;
+ boolean truncated;
+ boolean walkAll;
+ }
+
private String[] sortedValues(Set<String> values) {
String[] sorted = values.toArray(new String[0]);
Arrays.sort(sorted);
@@ -452,16 +531,27 @@ public class MSOneStorePackage {
}
}
+ /**
+ * Exempt from the parse-warning budget: capped GUID metadata must always
carry its own
+ * incompleteness signal, even when generic warnings were suppressed.
+ */
+ private void recordGuidCapWarning(String warning) {
+ emitParseWarning(warning);
+ }
+
/**
* Builds a map of object ID to object. The object groups are ordered from
the oldest
* revision to the newest, so a newer version of an object wins over an
older one.
*/
- private Map<ExGuid, RevisionStoreObject> indexObjectsById(
+ Map<ExGuid, RevisionStoreObject> indexObjectsById(
List<RevisionStoreObjectGroup> objectGroups) {
Map<ExGuid, RevisionStoreObject> objectsById = new HashMap<>();
for (RevisionStoreObjectGroup objectGroup : objectGroups) {
+ if (objectGroup == null || objectGroup.objects == null) {
+ continue;
+ }
for (RevisionStoreObject object : objectGroup.objects) {
- if (object.objectID != null) {
+ if (object != null && object.objectID != null) {
objectsById.put(object.objectID, object);
}
}
@@ -530,6 +620,8 @@ public class MSOneStorePackage {
break;
}
}
+ int objectType = jcidIndex(object);
+ recordEntityGuid(object, objectType);
// The title structure of a page (StructureElementChildNodes) appears
above the page
// body on screen, but is declared after the body child nodes. Emit it
first so the
// text comes out in visual order.
@@ -872,7 +964,8 @@ public class MSOneStorePackage {
lastModified = lastMod;
}
metadata.set(TikaCoreProperties.MODIFIED,
String.valueOf(lastModified));
- } else if (oneNotePropertyEnum == OneNotePropertyEnum.Author) {
+ } else if (oneNotePropertyEnum == OneNotePropertyEnum.Author
+ && property instanceof PrtFourBytesOfLengthFollowedByData) {
recordAuthor(decodeOneNoteText(
((PrtFourBytesOfLengthFollowedByData) property).data),
authorRole);
} else if (propertyType ==
PropertyType.FourBytesOfLengthFollowedByData) {
@@ -939,6 +1032,169 @@ public class MSOneStorePackage {
}
}
+ private void collectSectionFileIdentityGuids() {
+ if (headerCell != null && headerCell.objectData != null
+ && headerCell.objectData.body != null) {
+ collectFileIdentityGuids(headerCell.objectData.body, 0);
+ }
+ if (!sectionGuidCollected() && dataRootCell != null) {
+ collectFileIdentityGuidsFromGroups(dataRootCell.objectGroups);
+ }
+ if (!sectionGuidCollected()) {
+ collectFileIdentityGuidsFromGroups(OtherFileNodeList);
+ }
+ }
+
+ // sectionGuid is a scalar: past the first candidate, further scanning
cannot change
+ // the output, only burn work on values the collector will refuse
+ private boolean sectionGuidCollected() {
+ return !guids.values(OneNoteGuidCollector.Category.SECTION).isEmpty();
+ }
+
+ void collectFileIdentityGuidsFromGroups(List<RevisionStoreObjectGroup>
groups) {
+ if (groups == null || guids.isFull()) {
+ return;
+ }
+ for (RevisionStoreObjectGroup group : groups) {
+ if (guids.isFull() || sectionGuidCollected()) {
+ return;
+ }
+ if (group == null || group.objects == null) {
+ continue;
+ }
+ for (RevisionStoreObject object : group.objects) {
+ if (guids.isFull()) {
+ return;
+ }
+ if (object == null || object.propertySet == null
+ || object.propertySet.objectSpaceObjectPropSet ==
null) {
+ continue;
+ }
+
collectFileIdentityGuids(object.propertySet.objectSpaceObjectPropSet.body, 0);
+ }
+ }
+ }
+
+ void collectFileIdentityGuids(PropertySet propertySet, int depth) {
+ if (propertySet == null || propertySet.rgPrids == null ||
propertySet.rgData == null
+ || depth >= PropertySet.MAX_PROPERTY_NESTING ||
guids.isFull()) {
+ return;
+ }
+ int count = Math.min(propertySet.rgPrids.length,
propertySet.rgData.size());
+ for (int i = 0; i < count && !guids.isFull() &&
!sectionGuidCollected(); i++) {
+ PropertyID propertyID = propertySet.rgPrids[i];
+ IProperty property = propertySet.rgData.get(i);
+ long propertyIdValue = Unsigned.uint(propertyID.value).longValue();
+ PropertyType propertyType =
PropertyType.fromIntVal(propertyID.type);
+ if (propertyIdValue == FILE_IDENTITY_GUID_PROPERTY_ID
+ && property instanceof PrtFourBytesOfLengthFollowedByData)
{
+ GUID guid = GUID.fromMicrosoftBytes(
+ ((PrtFourBytesOfLengthFollowedByData) property).data);
+ if (guid != null) {
+ guids.add(OneNoteGuidCollector.Category.SECTION,
guid.toString());
+ }
+ } else if (propertyType == PropertyType.PropertySet && property
instanceof PropertySet) {
+ collectFileIdentityGuids((PropertySet) property, depth + 1);
+ } else if (propertyType == PropertyType.ArrayOfPropertyValues
+ && property instanceof PrtArrayOfPropertyValues
+ && ((PrtArrayOfPropertyValues) property).data != null) {
+ for (PropertySet nested : ((PrtArrayOfPropertyValues)
property).data) {
+ if (guids.isFull() || sectionGuidCollected()) {
+ return;
+ }
+ collectFileIdentityGuids(nested, depth + 1);
+ }
+ }
+ }
+ }
+
+ void recordEntityGuid(RevisionStoreObject object, int objectType) {
+ guids.addForObjectType(objectType,
notebookManagementEntityGuid(object));
+ }
+
+ String findPageGuid(RevisionStoreCell cell,
+ Map<ExGuid, RevisionStoreObject> objectsById) {
+ if (cell == null || cell.objectGroups == null) {
+ return null;
+ }
+ Set<ExGuid> visited = new HashSet<>();
+ CellRoots roots = cellRoots(cell, objectsById, warning -> { });
+ for (RevisionStoreObject rootObject : roots.declared) {
+ String guid = findPageGuid(rootObject, objectsById, visited, 0);
+ if (guid != null) {
+ return guid;
+ }
+ }
+ if (roots.walkAll) {
+ for (RevisionStoreObjectGroup group : cell.objectGroups) {
+ if (group == null || group.objects == null) {
+ continue;
+ }
+ for (RevisionStoreObject object : group.objects) {
+ String guid = findPageGuid(object, objectsById, visited,
0);
+ if (guid != null) {
+ return guid;
+ }
+ }
+ }
+ }
+ return null;
+ }
+
+ String findPageGuid(RevisionStoreObject object,
+ Map<ExGuid, RevisionStoreObject> objectsById,
+ Set<ExGuid> visited, int depth) {
+ if (object == null || depth >= MAX_OBJECT_WALK_DEPTH
+ || object.propertySet == null
+ || object.propertySet.objectSpaceObjectPropSet == null) {
+ return null;
+ }
+ if (object.objectID != null && !visited.add(object.objectID)) {
+ return null;
+ }
+ if (jcidIndex(object) == OneNoteJcid.PAGE_METADATA) {
+ String guid = notebookManagementEntityGuid(object);
+ if (guid != null) {
+ return guid;
+ }
+ }
+ for (PropertyAction action : collectObjectActions(object)) {
+ if (action.isChildReference && action.childReference != null) {
+ String guid =
findPageGuid(objectsById.get(action.childReference), objectsById,
+ visited, depth + 1);
+ if (guid != null) {
+ return guid;
+ }
+ }
+ }
+ return null;
+ }
+
+ String notebookManagementEntityGuid(RevisionStoreObject object) {
+ if (object == null || object.propertySet == null
+ || object.propertySet.objectSpaceObjectPropSet == null) {
+ return null;
+ }
+ for (PropertyAction action : collectObjectActions(object)) {
+ if (action.oneNotePropertyEnum ==
OneNotePropertyEnum.NotebookManagementEntityGuid
+ && action.property instanceof
PrtFourBytesOfLengthFollowedByData) {
+ GUID guid = GUID.fromMicrosoftBytes(
+ ((PrtFourBytesOfLengthFollowedByData)
action.property).data);
+ if (guid != null) {
+ return guid.toString();
+ }
+ }
+ }
+ return null;
+ }
+
+ static int jcidIndex(RevisionStoreObject object) {
+ if (object == null || object.jcid == null || object.jcid.jcid == null)
{
+ return -1;
+ }
+ return object.jcid.jcid.index;
+ }
+
private EmbeddedResourceInfo embeddedResourceInfo(List<PropertyAction>
actions) {
for (PropertyAction action : actions) {
if (action.property instanceof PrtFourBytesOfLengthFollowedByData
&&
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/GUIDTest.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/GUIDTest.java
index 0f21f861db..7e3c965f4d 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/GUIDTest.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/GUIDTest.java
@@ -17,6 +17,7 @@
package org.apache.tika.parser.microsoft.onenote;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import java.nio.charset.StandardCharsets;
@@ -67,4 +68,23 @@ public class GUIDTest {
malformedGuid.getBytes(StandardCharsets.UTF_16LE)));
}
}
+
+ @Test
+ public void testFromMicrosoftBytes() {
+ byte[] data = new byte[] {
+ 0x33, 0x22, 0x11, 0x00, 0x55, 0x44, 0x77, 0x66,
+ (byte) 0x88, (byte) 0x99, (byte) 0xaa, (byte) 0xbb,
+ (byte) 0xcc, (byte) 0xdd, (byte) 0xee, (byte) 0xff
+ };
+ assertEquals("{00112233-4455-6677-8899-AABBCCDDEEFF}",
+ GUID.fromMicrosoftBytes(data).toString());
+ }
+
+ @Test
+ public void testFromMicrosoftBytesRejectsInvalidLengths() {
+ assertNull(GUID.fromMicrosoftBytes(null));
+ assertNull(GUID.fromMicrosoftBytes(new byte[15]));
+ assertNull(GUID.fromMicrosoftBytes(new byte[17]));
+ assertNull(GUID.fromMicrosoftBytes(new byte[32]));
+ }
}
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/OneNoteGuidCollectorTest.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/OneNoteGuidCollectorTest.java
new file mode 100644
index 0000000000..9dbcba4ec9
--- /dev/null
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/OneNoteGuidCollectorTest.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.microsoft.onenote;
+
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.junit.jupiter.api.Test;
+
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.OneNote;
+
+public class OneNoteGuidCollectorTest {
+
+ @Test
+ public void testCapsTotalGuidValuesAndWarnsOnce() {
+ List<String> warnings = new ArrayList<>();
+ OneNoteGuidCollector collector = new
OneNoteGuidCollector(warnings::add);
+ assertFalse(collector.isFull());
+ collector.add(OneNoteGuidCollector.Category.PAGE, null);
+ for (int i = 0; i < OneNoteGuidCollector.MAX_GUID_COUNT; i++) {
+ collector.add(OneNoteGuidCollector.Category.CONFLICT_PAGE, "guid-"
+ i);
+ }
+ collector.add(OneNoteGuidCollector.Category.CONFLICT_PAGE, "guid-0");
+ collector.add(OneNoteGuidCollector.Category.PAGE, "overflow-1");
+ collector.add(OneNoteGuidCollector.Category.PAGE, "overflow-2");
+
+ assertEquals(OneNoteGuidCollector.MAX_GUID_COUNT,
+
collector.values(OneNoteGuidCollector.Category.CONFLICT_PAGE).size());
+
assertTrue(collector.values(OneNoteGuidCollector.Category.PAGE).isEmpty());
+ assertEquals(1, warnings.size());
+ assertTrue(collector.isFull());
+ }
+
+ @Test
+ public void testDuplicateValuesDoNotConsumeBudgetBeforeSaturation() {
+ List<String> warnings = new ArrayList<>();
+ OneNoteGuidCollector collector = new
OneNoteGuidCollector(warnings::add);
+ collector.add(OneNoteGuidCollector.Category.PAGE, "dup");
+ collector.add(OneNoteGuidCollector.Category.PAGE, "dup");
+ for (int i = 0; i < OneNoteGuidCollector.MAX_GUID_COUNT - 1; i++) {
+ collector.add(OneNoteGuidCollector.Category.CONFLICT_PAGE, "e" +
i);
+ }
+ collector.add(OneNoteGuidCollector.Category.CONFLICT_PAGE,
"final-extra");
+
+ assertEquals(1,
collector.values(OneNoteGuidCollector.Category.PAGE).size());
+ assertEquals(OneNoteGuidCollector.MAX_GUID_COUNT - 1,
+
collector.values(OneNoteGuidCollector.Category.CONFLICT_PAGE).size());
+
assertFalse(collector.values(OneNoteGuidCollector.Category.CONFLICT_PAGE)
+ .contains("final-extra"));
+ assertEquals(1, warnings.size());
+ assertTrue(collector.isFull());
+ }
+
+ @Test
+ public void testObjectTypesMapToKeys() {
+ assertEquals(OneNoteGuidCollector.Category.PAGE,
+ OneNoteGuidCollector.categoryFor(OneNoteJcid.PAGE_METADATA));
+ assertEquals(OneNoteGuidCollector.Category.PAGE_SERIES,
+
OneNoteGuidCollector.categoryFor(OneNoteJcid.PAGE_SERIES_NODE));
+ assertEquals(OneNoteGuidCollector.Category.PAGE_NODE,
+ OneNoteGuidCollector.categoryFor(OneNoteJcid.PAGE_NODE));
+ assertEquals(OneNoteGuidCollector.Category.CONFLICT_PAGE,
+
OneNoteGuidCollector.categoryFor(OneNoteJcid.CONFLICT_PAGE_METADATA));
+ assertNull(OneNoteGuidCollector.categoryFor(OneNoteJcid.SECTION_NODE));
+ assertNull(OneNoteGuidCollector.categoryFor(0x7fff));
+ }
+
+ @Test
+ public void testPublishSortsValuesAndSkipsEmptyKeys() {
+ OneNoteGuidCollector collector = new OneNoteGuidCollector(w -> { });
+ collector.addForObjectType(OneNoteJcid.PAGE_NODE, "{B}");
+ collector.addForObjectType(OneNoteJcid.PAGE_NODE, "{A}");
+ collector.addForObjectType(OneNoteJcid.SECTION_NODE, "{section-node}");
+ collector.add(OneNoteGuidCollector.Category.SECTION, "{first}");
+ collector.add(OneNoteGuidCollector.Category.SECTION, "{second}");
+ Metadata metadata = new Metadata();
+
+ collector.publish(metadata);
+
+ assertArrayEquals(new String[] {"{A}", "{B}"},
+ metadata.getValues(OneNote.PAGE_NODE_GUIDS));
+ assertEquals("{first}", metadata.get(OneNote.SECTION_GUID));
+ assertEquals(2, metadata.names().length);
+ }
+}
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/OneNoteParserTest.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/OneNoteParserTest.java
index d61aa5666e..f1dceec2cc 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/OneNoteParserTest.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/OneNoteParserTest.java
@@ -18,9 +18,11 @@ package org.apache.tika.parser.microsoft.onenote;
import static
org.apache.tika.parser.microsoft.onenote.OneNoteParser.ONE_NOTE_PREFIX;
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -51,6 +53,7 @@ import org.apache.tika.exception.TikaMemoryLimitException;
import org.apache.tika.extractor.EmbeddedDocumentExtractor;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.OneNote;
import org.apache.tika.metadata.TikaCoreProperties;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.microsoft.onenote.fsshttpb.MSOneStorePackage;
@@ -79,6 +82,76 @@ public class OneNoteParserTest extends TikaTest {
return metadata;
}
+ @Test
+ public void testSectionGuidIsExtractedFromClassicHeader() throws Exception
{
+ Metadata metadata = new Metadata();
+ try (InputStream input =
getClass().getResourceAsStream("/test-documents/testOneNote1.one");
+ TikaInputStream tis = TikaInputStream.get(input)) {
+ new OneNoteParser().parse(tis, new ToTextContentHandler(),
metadata,
+ new ParseContext());
+ }
+ String guid = metadata.get(OneNote.SECTION_GUID);
+ assertNotNull(guid);
+ assertTrue(guid.matches("\\{[0-9A-F]{8}-[0-9A-F]{4}-[0-9A-F]{4}-"
+ + "[0-9A-F]{4}-[0-9A-F]{12}\\}"));
+ }
+
+ @Test
+ public void testClassicGuidValues() throws Exception {
+ Metadata metadata = new Metadata();
+ getText("testOneNote2.one", metadata);
+ assertEquals("{D03D94F3-AFB4-484F-A5ED-B93EBA2806B9}",
+ metadata.get(OneNote.SECTION_GUID));
+ assertArrayEquals(new String[]
{"{0AD2F2F8-F7C0-4301-82DF-064544DD31E5}",
+ "{E1FDD004-97A6-49B8-A9C9-BB781D0C5423}"},
+ metadata.getValues(OneNote.PAGE_GUIDS));
+ assertArrayEquals(new String[]
{"{135E9FB6-B9E5-42B9-B185-2A72495D0A05}",
+ "{815B61E9-5C57-43CF-BCDD-9D57A0FEDADB}"},
+ metadata.getValues(OneNote.PAGE_SERIES_GUIDS));
+ assertArrayEquals(new String[]
{"{52ADC330-8420-4DDC-A381-C8C30ECD8739}",
+ "{EE2E0DFD-9B11-4E22-AB62-53C667EF58D1}"},
+ metadata.getValues(OneNote.PAGE_NODE_GUIDS));
+ }
+
+ @Test
+ public void testClassicGuidsSkipVersionHistoryPages() throws Exception {
+ Metadata metadata = new Metadata();
+ getText("test-tika-3970-dupetext.one", metadata);
+ assertArrayEquals(new String[]
{"{70D3B6E3-B023-4A5E-AC96-6FCE9D66F63C}"},
+ metadata.getValues(OneNote.PAGE_GUIDS));
+ assertArrayEquals(new String[]
{"{BB5DE1E3-25FB-4584-A37F-69D332F40BF7}"},
+ metadata.getValues(OneNote.PAGE_SERIES_GUIDS));
+ }
+
+ @Test
+ public void testFsshttpbGuidValuesAreSorted() throws Exception {
+ Metadata metadata = new Metadata();
+ getText("testOneNoteFromOffice365.one", metadata);
+ assertEquals("{54807CE9-568A-4883-B863-BF97F35C844F}",
+ metadata.get(OneNote.SECTION_GUID));
+ assertArrayEquals(new String[]
{"{9376CA69-2A0D-440E-BFC9-A728C597ED93}",
+ "{D1962227-01A7-4F73-A1F5-410DA0C519CC}"},
+ metadata.getValues(OneNote.PAGE_GUIDS));
+ assertArrayEquals(new String[]
{"{01B28DC5-49F4-4F05-8E10-131236FAAB82}",
+ "{429CB628-77C7-4600-BE0C-83166E392792}"},
+ metadata.getValues(OneNote.PAGE_SERIES_GUIDS));
+ }
+
+ @Test
+ public void testEmitSectionGuidSkipsNullAndNil() {
+ Metadata metadata = new Metadata();
+ OneNoteParser.emitSectionFileGuid(new
OneNoteHeader().setGuidFile(null), metadata);
+ OneNoteParser.emitSectionFileGuid(new
OneNoteHeader().setGuidFile(GUID.nil()), metadata);
+ assertNull(metadata.get(OneNote.SECTION_GUID));
+
+ GUID fileGuid = new GUID(new int[] {
+ 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x08,
+ 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff, 0x00
+ });
+ OneNoteParser.emitSectionFileGuid(new
OneNoteHeader().setGuidFile(fileGuid), metadata);
+ assertEquals(fileGuid.toString(), metadata.get(OneNote.SECTION_GUID));
+ }
+
@Test
public void testFuzzerRegressionInputsFallBackToLegacyDump() throws
Exception {
// a structural failure falls back to the legacy string dump; the
human warning names
@@ -296,6 +369,14 @@ public class OneNoteParserTest extends TikaTest {
assertNotContained("Microsoft\u0000", originalAuthors);
assertContains("ndipiazza\u0000", mostRecentAuthors);
+ String[] pageGuids = metadata.getValues(OneNote.PAGE_GUIDS);
+ assertNotNull(pageGuids);
+ assertTrue(pageGuids.length > 0);
+ assertEquals(pageGuids.length,
Arrays.stream(pageGuids).distinct().count());
+ String[] pageSeriesGuids =
metadata.getValues(OneNote.PAGE_SERIES_GUIDS);
+ assertNotNull(pageSeriesGuids);
+ assertTrue(pageSeriesGuids.length > 0);
+
assertEquals(Instant.ofEpochSecond(1574426385),
Instant.ofEpochSecond(Long.parseLong(metadata.get(ONE_NOTE_PREFIX +
"creationTimestamp"))));
assertEquals(Instant.ofEpochMilli(1574426548000L),
@@ -399,6 +480,19 @@ public class OneNoteParserTest extends TikaTest {
assertNotNull(embedded.get(TikaCoreProperties.EMBEDDED_RELATIONSHIP_ID));
}
+ @Test
+ public void testHeaderGuidDeserializationRejectsTruncatedFile(@TempDir
Path tempDir)
+ throws Exception {
+ // fewer bytes than one GUID: reading must fail, not silently decode
0xFF filler
+ Path shortFile = tempDir.resolve("short.one");
+ Files.write(shortFile, new byte[10]);
+ try (OneNoteDirectFileResource dif = new
OneNoteDirectFileResource(shortFile.toFile())) {
+ OneNotePtr ptr = new OneNotePtr(new OneNoteDocument(), dif);
+ IOException exception = assertThrows(IOException.class,
ptr::deserializeHeader);
+ assertTrue(exception.getMessage().contains("while reading a
GUID"));
+ }
+ }
+
@Test
public void testPropertyValueBudgetIsSharedAcrossCopiesAndResetsPerList(
@TempDir Path tempDir) throws Exception {
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/OneNoteTreeWalkerGuidTest.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/OneNoteTreeWalkerGuidTest.java
new file mode 100644
index 0000000000..9fff0a055d
--- /dev/null
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/OneNoteTreeWalkerGuidTest.java
@@ -0,0 +1,218 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.microsoft.onenote;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.lang3.tuple.Pair;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.xml.sax.SAXException;
+
+import org.apache.tika.exception.TikaException;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.OneNote;
+import org.apache.tika.metadata.TikaCoreProperties;
+import org.apache.tika.parser.ParseContext;
+import org.apache.tika.sax.ToTextContentHandler;
+import org.apache.tika.sax.XHTMLContentHandler;
+
+public class OneNoteTreeWalkerGuidTest {
+
+ @Test
+ public void testGuidBagsPublishAfterWalkFailure() throws Exception {
+ Metadata metadata = new Metadata();
+ OneNoteTreeWalker walker = newWalker(metadata, true, null);
+ walker.addClassicEntityGuid(OneNoteJcid.PAGE_METADATA, "page-guid");
+ walker.addClassicEntityGuid(OneNoteJcid.PAGE_SERIES_NODE,
"series-guid");
+ walker.addClassicEntityGuid(OneNoteJcid.CONFLICT_PAGE_METADATA,
"conflict-guid");
+ walker.addClassicEntityGuid(OneNoteJcid.SECTION_NODE,
"section-node-guid");
+ walker.addClassicEntityGuid(OneNoteJcid.PAGE_NODE, "page-node-guid");
+ walker.addClassicEntityGuid(0x7fff, "entity-guid");
+ for (int i = 0; i < OneNoteGuidCollector.MAX_GUID_COUNT - 4; i++) {
+ walker.addClassicEntityGuid(OneNoteJcid.PAGE_SERIES_NODE,
"series-" + i);
+ }
+ walker.addClassicEntityGuid(OneNoteJcid.PAGE_SERIES_NODE,
"overflow-guid");
+
+ assertThrows(TikaException.class, walker::walkTree);
+ assertEquals(List.of("page-guid"),
+ Arrays.asList(metadata.getValues(OneNote.PAGE_GUIDS)));
+ assertEquals(List.of("conflict-guid"),
+
Arrays.asList(metadata.getValues(OneNote.CONFLICT_PAGE_GUIDS)));
+ assertEquals(List.of("page-node-guid"),
+ Arrays.asList(metadata.getValues(OneNote.PAGE_NODE_GUIDS)));
+ assertEquals(OneNoteGuidCollector.MAX_GUID_COUNT - 3,
+ metadata.getValues(OneNote.PAGE_SERIES_GUIDS).length);
+ assertTrue(Arrays.asList(metadata.getValues(OneNote.PAGE_SERIES_GUIDS))
+ .contains("series-guid"));
+
assertFalse(Arrays.asList(metadata.getValues(OneNote.PAGE_SERIES_GUIDS))
+ .contains("overflow-guid"));
+ assertEquals(1,
Arrays.stream(metadata.getValues(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING))
+ .filter(w -> w.contains("Capping OneNote GUID
metadata")).count());
+ }
+
+ @Test
+ public void testGuidLimitWithNoMetadataParent() throws Exception {
+ OneNoteTreeWalker walker = newWalker(null, false, null);
+ for (int i = 0; i < OneNoteGuidCollector.MAX_GUID_COUNT; i++) {
+ walker.addClassicEntityGuid(OneNoteJcid.PAGE_METADATA, "guid-" +
i);
+ }
+ walker.addClassicEntityGuid(OneNoteJcid.PAGE_METADATA,
"overflow-guid");
+ walker.walkTree();
+ }
+
+ @Test
+ public void testJcidIndexRequiresCompleteJcid() {
+ assertEquals(-1, OneNoteTreeWalker.jcidIndex(null));
+ FileNode fileNode = new FileNode();
+ assertEquals(0, OneNoteTreeWalker.jcidIndex(fileNode));
+ fileNode.subType = null;
+ assertEquals(-1, OneNoteTreeWalker.jcidIndex(fileNode));
+ fileNode.subType = new FileNodeUnion();
+ fileNode.subType.objectDeclarationWithRefCount = null;
+ assertEquals(-1, OneNoteTreeWalker.jcidIndex(fileNode));
+ fileNode.subType.objectDeclarationWithRefCount = new
ObjectDeclarationWithRefCount();
+ fileNode.subType.objectDeclarationWithRefCount.body = null;
+ assertEquals(-1, OneNoteTreeWalker.jcidIndex(fileNode));
+ fileNode.subType.objectDeclarationWithRefCount.body =
+ new ObjectDeclarationWithRefCountBody();
+ fileNode.subType.objectDeclarationWithRefCount.body.jcid = null;
+ assertEquals(-1, OneNoteTreeWalker.jcidIndex(fileNode));
+ fileNode.subType.objectDeclarationWithRefCount.body.jcid = new JCID();
+ fileNode.subType.objectDeclarationWithRefCount.body.jcid.index =
OneNoteJcid.PAGE_METADATA;
+ assertEquals(OneNoteJcid.PAGE_METADATA,
OneNoteTreeWalker.jcidIndex(fileNode));
+ }
+
+ @Test
+ public void testNotebookGuidRequiresSixteenBytes(@TempDir Path tempDir)
throws Exception {
+ byte[] guidBytes = new byte[] {
+ 0x33, 0x22, 0x11, 0x00, 0x55, 0x44, 0x77, 0x66,
+ (byte) 0x88, (byte) 0x99, (byte) 0xaa, (byte) 0xbb,
+ (byte) 0xcc, (byte) 0xdd, (byte) 0xee, (byte) 0xff
+ };
+ Path input = tempDir.resolve("guid-data.bin");
+ Files.write(input, guidBytes);
+ Metadata metadata = new Metadata();
+ try (OneNoteDirectFileResource dif = new
OneNoteDirectFileResource(input.toFile())) {
+ OneNoteTreeWalker walker = newWalker(metadata, false, dif);
+ PropertyValue valid = notebookGuidProperty(guidBytes.length);
+ Map<String, Object> parsed = walker.processPropertyValue(valid,
+ parent(OneNotePropertyEnum.MetaDataObjectsAboveGraphSpace),
+ OneNoteJcid.PAGE_METADATA);
+ assertEquals("{00112233-4455-6677-8899-AABBCCDDEEFF}",
+ parsed.get("notebookManagementEntityGuid"));
+
+ PropertyValue shortGuid = notebookGuidProperty(15);
+ Map<String, Object> malformed =
walker.processPropertyValue(shortGuid,
+ parent(OneNotePropertyEnum.MetaDataObjectsAboveGraphSpace),
+ OneNoteJcid.PAGE_METADATA);
+ assertFalse(malformed.containsKey("notebookManagementEntityGuid"));
+ walker.walkTree();
+ }
+ assertEquals(List.of("{00112233-4455-6677-8899-AABBCCDDEEFF}"),
+ Arrays.asList(metadata.getValues(OneNote.PAGE_GUIDS)));
+ }
+
+ @Test
+ public void testClassicPageGuidRequiresCurrentPageReference(@TempDir Path
tempDir)
+ throws Exception {
+ byte[] guidBytes = new byte[] {
+ 0x33, 0x22, 0x11, 0x00, 0x55, 0x44, 0x77, 0x66,
+ (byte) 0x88, (byte) 0x99, (byte) 0xaa, (byte) 0xbb,
+ (byte) 0xcc, (byte) 0xdd, (byte) 0xee, (byte) 0xff
+ };
+ Path input = tempDir.resolve("guid-data.bin");
+ Files.write(input, guidBytes);
+ Metadata metadata = new Metadata();
+ try (OneNoteDirectFileResource dif = new
OneNoteDirectFileResource(input.toFile())) {
+ OneNoteTreeWalker walker = newWalker(metadata, false, dif);
+ // version-history copies are reached as object-space roots or via
other properties
+ walker.processPropertyValue(notebookGuidProperty(16), null,
+ OneNoteJcid.PAGE_METADATA);
+ walker.processPropertyValue(notebookGuidProperty(16),
+
parent(OneNotePropertyEnum.ContentChildNodesOfPageManifest),
+ OneNoteJcid.PAGE_METADATA);
+ walker.processPropertyValue(notebookGuidProperty(16), null,
+ OneNoteJcid.PAGE_SERIES_NODE);
+ walker.walkTree();
+ }
+ assertEquals(0, metadata.getValues(OneNote.PAGE_GUIDS).length);
+ assertEquals(List.of("{00112233-4455-6677-8899-AABBCCDDEEFF}"),
+ Arrays.asList(metadata.getValues(OneNote.PAGE_SERIES_GUIDS)));
+ }
+
+ private static OneNotePropertyId parent(OneNotePropertyEnum propertyEnum) {
+ return new OneNotePropertyId().setPropertyEnum(propertyEnum);
+ }
+
+ private static PropertyValue notebookGuidProperty(int length) {
+ PropertyValue value = new PropertyValue();
+ value.propertyId.propertyEnum =
OneNotePropertyEnum.NotebookManagementEntityGuid;
+ value.propertyId.type = 7;
+ value.rawData.setStp(0).setCb(length);
+ return value;
+ }
+
+ @Test
+ public void testGuidStringPoolSharesAndBoundsInstances() {
+ OneNoteTreeWalker walker = newWalker(new Metadata(), false, null);
+ // equal-but-distinct instances: an identity implementation would fail
assertSame
+ String first = walker.pooledGuid("{A}");
+ String second = walker.pooledGuid(new String("{A}"));
+ assertSame(first, second);
+
+ walker.guidStringPool.clear();
+ for (int i = 0; i < OneNoteGuidCollector.MAX_GUID_COUNT; i++) {
+ walker.guidStringPool.put("g" + i, "g" + i);
+ }
+ String input = new String("{B}");
+ assertSame(input, walker.pooledGuid(input));
+ // a full pool must not grow: without the size guard this becomes MAX
+ 1
+ assertEquals(OneNoteGuidCollector.MAX_GUID_COUNT,
walker.guidStringPool.size());
+ }
+
+ private static OneNoteTreeWalker newWalker(Metadata metadata, boolean fail,
+ OneNoteDirectFileResource dif)
{
+ ParseContext context = new ParseContext();
+ Metadata handlerMetadata = metadata == null ? new Metadata() :
metadata;
+ XHTMLContentHandler xhtml = new XHTMLContentHandler(
+ new ToTextContentHandler(), handlerMetadata, context);
+ return new OneNoteTreeWalker(new OneNoteTreeWalkerOptions(), new
OneNoteDocument(), dif,
+ xhtml, metadata, context, Pair.of(1L, ExtendedGUID.nil())) {
+ @Override
+ public List<Map<String, Object>> walkRootFileNodes()
+ throws IOException, TikaException, SAXException {
+ if (fail) {
+ throw new TikaException("test walk failure");
+ }
+ return Collections.emptyList();
+ }
+ };
+ }
+}
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStorePackageTest.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStorePackageTest.java
index bbeadea253..9ff8a0bc14 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStorePackageTest.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStorePackageTest.java
@@ -16,6 +16,7 @@
*/
package org.apache.tika.parser.microsoft.onenote.fsshttpb;
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNull;
@@ -29,6 +30,8 @@ import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -42,6 +45,8 @@ import org.apache.tika.metadata.Metadata;
import org.apache.tika.metadata.OneNote;
import org.apache.tika.metadata.TikaCoreProperties;
import org.apache.tika.parser.ParseContext;
+import org.apache.tika.parser.microsoft.onenote.OneNoteGuidCollector;
+import org.apache.tika.parser.microsoft.onenote.OneNoteJcid;
import org.apache.tika.parser.microsoft.onenote.OneNoteTreeWalkerOptions;
import org.apache.tika.parser.microsoft.onenote.fsshttpb.property.ArrayNumber;
import org.apache.tika.parser.microsoft.onenote.fsshttpb.property.IProperty;
@@ -50,6 +55,7 @@ import
org.apache.tika.parser.microsoft.onenote.fsshttpb.property.PrtArrayOfProp
import
org.apache.tika.parser.microsoft.onenote.fsshttpb.property.PrtFourBytesOfLengthFollowedByData;
import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.DataElement;
import
org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.FileDataObject;
+import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.JCIDObject;
import
org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.ObjectDataBLOB;
import
org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.ObjectDataBLOBDataElementData;
import
org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.ObjectGroupObjectData;
@@ -62,6 +68,7 @@ import
org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.RevisionStore
import
org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.basic.CellID;
import
org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.basic.DataElementType;
import
org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.basic.ExGuid;
+import
org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.basic.HeaderCell;
import
org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.basic.PropertyID;
import
org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.basic.PropertyType;
import
org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.space.ObjectSpaceObjectPropSet;
@@ -72,6 +79,814 @@ import org.apache.tika.sax.XHTMLContentHandler;
public class MSOneStorePackageTest {
+ @Test
+ public void testSectionAndPageGuidsAreExtracted() throws Exception {
+ String sectionGuid = "{00112233-4455-6677-8899-AABBCCDDEEFF}";
+ String pageGuid = "{10213243-5465-7687-98A9-BACBDCEDFE0F}";
+ String stalePageGuid = "{20314253-6475-8697-A8B9-CADBECFD0E1F}";
+ String pageSeriesGuid = "{30415263-7485-96A7-B8C9-DAEBFC0D1E2F}";
+ String conflictPageGuid = "{20314253-6475-8697-A8B9-CADBECFD0E1F}";
+ String entityGuid = "{40516273-8495-A6B7-C8D9-EAFB0C1D2E3F}";
+ String sectionNodeGuid = "{50617283-94A5-B6C7-D8E9-FA0B1C2D3E4F}";
+ String pageNodeGuid = "{60718293-A4B5-C6D7-E8F9-0A1B2C3D4E5F}";
+ byte[] pageNodeGuidBytes = new byte[] {
+ (byte) 0x93, (byte) 0x82, 0x71, 0x60, (byte) 0xb5, (byte)
0xa4, (byte) 0xd7,
+ (byte) 0xc6, (byte) 0xe8, (byte) 0xf9, 0x0a, 0x1b, 0x2c, 0x3d,
0x4e, 0x5f
+ };
+ byte[] guidBytes = sectionGuidBytes();
+ byte[] pageGuidBytes = new byte[] {
+ 0x43, 0x32, 0x21, 0x10, 0x65, 0x54, (byte) 0x87, 0x76,
+ (byte) 0x98, (byte) 0xa9, (byte) 0xba, (byte) 0xcb,
+ (byte) 0xdc, (byte) 0xed, (byte) 0xfe, 0x0f
+ };
+ byte[] stalePageGuidBytes = new byte[] {
+ 0x53, 0x42, 0x31, 0x20, 0x75, 0x64, (byte) 0x97, (byte) 0x86,
+ (byte) 0xa8, (byte) 0xb9, (byte) 0xca, (byte) 0xdb,
+ (byte) 0xec, (byte) 0xfd, 0x0e, 0x1f
+ };
+ byte[] pageSeriesGuidBytes = new byte[] {
+ 0x63, 0x52, 0x41, 0x30, (byte) 0x85, 0x74, (byte) 0xa7, (byte)
0x96,
+ (byte) 0xb8, (byte) 0xc9, (byte) 0xda, (byte) 0xeb,
+ (byte) 0xfc, 0x0d, 0x1e, 0x2f
+ };
+ byte[] conflictPageGuidBytes = new byte[] {
+ 0x53, 0x42, 0x31, 0x20, 0x75, 0x64, (byte) 0x97, (byte) 0x86,
+ (byte) 0xa8, (byte) 0xb9, (byte) 0xca, (byte) 0xdb,
+ (byte) 0xec, (byte) 0xfd, 0x0e, 0x1f
+ };
+ byte[] entityGuidBytes = new byte[] {
+ 0x73, 0x62, 0x51, 0x40, (byte) 0x95, (byte) 0x84, (byte) 0xb7,
(byte) 0xa6,
+ (byte) 0xc8, (byte) 0xd9, (byte) 0xea, (byte) 0xfb,
+ 0x0c, 0x1d, 0x2e, 0x3f
+ };
+ byte[] sectionNodeGuidBytes = new byte[] {
+ (byte) 0x83, 0x72, 0x61, 0x50, (byte) 0xa5, (byte) 0x94,
(byte) 0xc7, (byte) 0xb6,
+ (byte) 0xd8, (byte) 0xe9, (byte) 0xfa, 0x0b, 0x1c, 0x2d, 0x3e,
0x4f
+ };
+ RevisionStoreObject stalePageMetadata = object(id(1199),
+ propertySet(new
PropertySpec(PropertyType.FourBytesOfLengthFollowedByData,
+ 0x1C001C30, bytes(stalePageGuidBytes))),
Collections.emptyList(),
+ Collections.emptyList());
+ RevisionStoreObject pageMetadata = object(id(1200),
+ propertySet(new
PropertySpec(PropertyType.FourBytesOfLengthFollowedByData,
+ 0x1C001C30, bytes(pageGuidBytes))),
Collections.emptyList(),
+ Collections.emptyList());
+ setJcid(stalePageMetadata, OneNoteJcid.PAGE_METADATA);
+ setJcid(pageMetadata, OneNoteJcid.PAGE_METADATA);
+ RevisionStoreCell pageCell = new RevisionStoreCell();
+ pageCell.cellID = cell(12, 1200);
+ pageCell.objectGroups.add(group(stalePageMetadata, pageMetadata));
+ pageCell.rootDeclares.add(rootDeclare(pageMetadata.objectID));
+
+ ExGuid sectionRootId = id(1201);
+ ExGuid pageSeriesId = id(1203);
+ RevisionStoreObject pageSeries = object(pageSeriesId,
+ propertySet(new
PropertySpec(PropertyType.FourBytesOfLengthFollowedByData,
+ 0x1C001C30, bytes(pageSeriesGuidBytes))),
Collections.emptyList(),
+ Collections.emptyList());
+ setJcid(pageSeries, OneNoteJcid.PAGE_SERIES_NODE);
+ ExGuid conflictPageId = id(1204);
+ RevisionStoreObject conflictPage = object(conflictPageId,
+ propertySet(new
PropertySpec(PropertyType.FourBytesOfLengthFollowedByData,
+ 0x1C001C30, bytes(conflictPageGuidBytes))),
Collections.emptyList(),
+ Collections.emptyList());
+ setJcid(conflictPage, OneNoteJcid.CONFLICT_PAGE_METADATA);
+ ExGuid entityId = id(1205);
+ RevisionStoreObject entity = object(entityId,
+ propertySet(new
PropertySpec(PropertyType.FourBytesOfLengthFollowedByData,
+ 0x1C001C30, bytes(entityGuidBytes))),
Collections.emptyList(),
+ Collections.emptyList());
+ setJcid(entity, 0x7fff);
+ ExGuid sectionNodeId = id(1206);
+ RevisionStoreObject sectionNode = object(sectionNodeId,
+ propertySet(new
PropertySpec(PropertyType.FourBytesOfLengthFollowedByData,
+ 0x1C001C30, bytes(sectionNodeGuidBytes))),
Collections.emptyList(),
+ Collections.emptyList());
+ setJcid(sectionNode, OneNoteJcid.SECTION_NODE);
+ ExGuid pageNodeId = id(1207);
+ RevisionStoreObject pageNode = object(pageNodeId,
+ propertySet(new
PropertySpec(PropertyType.FourBytesOfLengthFollowedByData,
+ 0x1C001C30, bytes(pageNodeGuidBytes))),
Collections.emptyList(),
+ Collections.emptyList());
+ setJcid(pageNode, OneNoteJcid.PAGE_NODE);
+ RevisionStoreObject sectionRoot = object(sectionRootId,
+ propertySet(new PropertySpec(PropertyType.ArrayOfObjectIDs,
0x24001C20,
+ arrayNumber(5)),
+ new PropertySpec(PropertyType.ObjectSpaceID,
0x20001D78, new NoData())),
+ Arrays.asList(pageSeriesId, conflictPageId, entityId,
sectionNodeId, pageNodeId),
+ Collections.singletonList(pageCell.cellID));
+ RevisionStoreObject sectionIdentity = object(id(1202),
+ propertySet(new
PropertySpec(PropertyType.FourBytesOfLengthFollowedByData,
+ 0x1C001D94, bytes(guidBytes))),
Collections.emptyList(),
+ Collections.emptyList());
+ RevisionStoreCell sectionCell = new RevisionStoreCell();
+ sectionCell.objectGroups.add(group(sectionRoot, sectionIdentity,
pageSeries,
+ conflictPage, entity, sectionNode, pageNode));
+ sectionCell.rootDeclares.add(rootDeclare(sectionRootId));
+
+ MSOneStorePackage pkg = new MSOneStorePackage();
+ pkg.dataRootCell = sectionCell;
+ pkg.cells.add(pageCell);
+ Metadata metadata = new Metadata();
+ walk(pkg, metadata);
+
+ assertArrayEquals(new String[] {sectionGuid},
metadata.getValues(OneNote.SECTION_GUID));
+ assertArrayEquals(new String[] {pageGuid},
metadata.getValues(OneNote.PAGE_GUIDS));
+ assertFalse(Arrays.asList(metadata.getValues(OneNote.SECTION_GUID))
+ .contains(sectionNodeGuid));
+ assertArrayEquals(new String[] {pageSeriesGuid},
+ metadata.getValues(OneNote.PAGE_SERIES_GUIDS));
+ assertArrayEquals(new String[] {conflictPageGuid},
+ metadata.getValues(OneNote.CONFLICT_PAGE_GUIDS));
+ assertArrayEquals(new String[] {pageNodeGuid},
+ metadata.getValues(OneNote.PAGE_NODE_GUIDS));
+ // a JCID with no GUID key (0x7fff here) contributes no GUID metadata
+ for (String name : metadata.names()) {
+
assertFalse(Arrays.asList(metadata.getValues(name)).contains(entityGuid), name);
+ }
+ String xml = walkXml(pkg);
+ assertTrue(xml.contains("id=\"" + pageGuid + "\""));
+ assertFalse(xml.contains("id=\"" + stalePageGuid + "\""));
+ }
+
+ @Test
+ public void testWalksOtherFileNodeListWhenCellsAreMissing() throws
Exception {
+ byte[] guidBytes = new byte[] {
+ 0x43, 0x32, 0x21, 0x10, 0x65, 0x54, (byte) 0x87, 0x76,
+ (byte) 0x98, (byte) 0xa9, (byte) 0xba, (byte) 0xcb,
+ (byte) 0xdc, (byte) 0xed, (byte) 0xfe, 0x0f
+ };
+ String pageGuid = "{10213243-5465-7687-98A9-BACBDCEDFE0F}";
+ MSOneStorePackage pkg = new MSOneStorePackage();
+ pkg.OtherFileNodeList.add(group(notebookGuidObject(1400,
OneNoteJcid.PAGE_METADATA,
+ guidBytes)));
+ Metadata metadata = new Metadata();
+
+ walk(pkg, metadata);
+
+ assertArrayEquals(new String[] {pageGuid},
metadata.getValues(OneNote.PAGE_GUIDS));
+ }
+
+ @Test
+ public void testFallbackWalkSkipsMalformedGroups() throws Exception {
+ // a null group and a group without objects must be skipped, not crash
the walk
+ MSOneStorePackage pkg = new MSOneStorePackage();
+ pkg.OtherFileNodeList.add(null);
+ RevisionStoreObjectGroup noObjects = group();
+ noObjects.objects = null;
+ pkg.OtherFileNodeList.add(noObjects);
+ RevisionStoreObjectGroup withNullObject = group();
+ withNullObject.objects.add(null);
+ RevisionStoreObject noId = object(id(2401),
+ propertySet(new
PropertySpec(PropertyType.FourBytesOfLengthFollowedByData,
+ 0x1C003498, text("no-id text"))),
Collections.emptyList(),
+ Collections.emptyList());
+ noId.objectID = null;
+ withNullObject.objects.add(noId);
+ withNullObject.objects.add(object(id(2400),
+ propertySet(new
PropertySpec(PropertyType.FourBytesOfLengthFollowedByData,
+ 0x1C003498, text("fallback text"))),
Collections.emptyList(),
+ Collections.emptyList()));
+ pkg.OtherFileNodeList.add(withNullObject);
+ Metadata metadata = new Metadata();
+
+ String text = walk(pkg, metadata);
+
+ assertTrue(text.contains("fallback text"));
+ assertTrue(text.contains("no-id text"));
+ }
+
+ @Test
+ public void
testFileIdentityScannerHandlesNestedSetsArraysAndMalformedValues() {
+ byte[] firstGuidBytes = sectionGuidBytes();
+ PropertySet noProperties = new PropertySet();
+ noProperties.rgData = new ArrayList<>();
+ PropertySet noData = new PropertySet();
+ noData.rgPrids = new PropertyID[0];
+ // guid-free nested structures first: once the winning GUID is found,
the scan stops,
+ // so anything placed later would never be scanned
+ PrtArrayOfPropertyValues guidFreeArray = new
PrtArrayOfPropertyValues();
+ guidFreeArray.data = new PropertySet[] {
+ propertySet(new
PropertySpec(PropertyType.FourBytesOfLengthFollowedByData,
+ 0x1C003498, text("nested"))), null, noProperties,
noData
+ };
+ PropertySet deep = noProperties;
+ for (int i = 0; i <= PropertySet.MAX_PROPERTY_NESTING; i++) {
+ deep = propertySet(new PropertySpec(PropertyType.PropertySet,
0x24000001, deep));
+ }
+ PrtArrayOfPropertyValues winningArray = new PrtArrayOfPropertyValues();
+ winningArray.data = new PropertySet[] {
+ fileIdentitySet(firstGuidBytes), noProperties
+ };
+ PrtArrayOfPropertyValues emptyArray = new PrtArrayOfPropertyValues();
+ PropertySet top = propertySet(
+ new PropertySpec(PropertyType.FourBytesOfLengthFollowedByData,
+ 0x1C001D94, bytes(new byte[17])),
+ new PropertySpec(PropertyType.PropertySet, 0x24000002,
+ propertySet(new
PropertySpec(PropertyType.FourBytesOfLengthFollowedByData,
+ 0x1C003498, text("nested")))),
+ new PropertySpec(PropertyType.PropertySet, 0x1C001D94,
+ propertySet(new
PropertySpec(PropertyType.FourBytesOfLengthFollowedByData,
+ 0x1C003498, text("nested")))),
+ new PropertySpec(PropertyType.PropertySet, 0x24000003, new
NoData()),
+ new PropertySpec(PropertyType.ArrayOfPropertyValues,
0x24000004, guidFreeArray),
+ new PropertySpec(PropertyType.ArrayOfPropertyValues,
0x24000005, emptyArray),
+ new PropertySpec(PropertyType.ArrayOfPropertyValues,
0x24000007, new NoData()),
+ new PropertySpec(PropertyType.PropertySet, 0x24000006, deep),
+ new PropertySpec(PropertyType.ArrayOfPropertyValues,
0x24000008, winningArray),
+ new PropertySpec(PropertyType.FourBytesOfLengthFollowedByData,
+ 0x1C003498, text("after the guid")));
+ MSOneStorePackage pkg = new MSOneStorePackage();
+
+ pkg.collectFileIdentityGuids(null, 0);
+ pkg.collectFileIdentityGuids(top, 0);
+
+
assertEquals(Collections.singletonList("{00112233-4455-6677-8899-AABBCCDDEEFF}"),
+ new ArrayList<>(sectionGuidValues(pkg)));
+
+ MSOneStorePackage full = new MSOneStorePackage();
+ fillGuidBudget(full, OneNoteGuidCollector.MAX_GUID_COUNT);
+ full.collectFileIdentityGuids(fileIdentitySet(firstGuidBytes), 0);
+ assertTrue(sectionGuidValues(full).isEmpty());
+ }
+
+ @Test
+ public void testFileIdentityGroupScannerSkipsMalformedGroups() throws
Exception {
+ String sectionGuid = "{00112233-4455-6677-8899-AABBCCDDEEFF}";
+ byte[] guidBytes = sectionGuidBytes();
+ RevisionStoreObjectGroup noObjects = group();
+ noObjects.objects = null;
+ RevisionStoreObject noPropertySet = new RevisionStoreObject();
+ RevisionStoreObject noObjectSpace = object(id(1450),
fileIdentitySet(guidBytes),
+ Collections.emptyList(), Collections.emptyList());
+ noObjectSpace.propertySet.objectSpaceObjectPropSet = null;
+ RevisionStoreObject valid = object(id(1451),
fileIdentitySet(guidBytes),
+ Collections.emptyList(), Collections.emptyList());
+ List<RevisionStoreObjectGroup> groups = new ArrayList<>(Arrays.asList(
+ null, noObjects, group((RevisionStoreObject) null),
group(noPropertySet),
+ group(noObjectSpace), group(valid)));
+ MSOneStorePackage pkg = new MSOneStorePackage();
+
+ pkg.collectFileIdentityGuidsFromGroups(null);
+ pkg.collectFileIdentityGuidsFromGroups(groups);
+
+ assertEquals(Collections.singletonList(sectionGuid), new
ArrayList<>(sectionGuidValues(pkg)));
+ }
+
+ @Test
+ public void testHeaderCellGuidAndMissingHeaderDataFallbacks() throws
Exception {
+ String sectionGuid = "{00112233-4455-6677-8899-AABBCCDDEEFF}";
+ byte[] guidBytes = sectionGuidBytes();
+ HeaderCell noObjectData = new HeaderCell();
+ HeaderCell noBody = new HeaderCell();
+ noBody.objectData = new ObjectSpaceObjectPropSet();
+ for (HeaderCell header : Arrays.asList(null, noObjectData, noBody)) {
+ MSOneStorePackage pkg = new MSOneStorePackage();
+ pkg.headerCell = header;
+ pkg.OtherFileNodeList.add(group(object(id(1460),
fileIdentitySet(guidBytes),
+ Collections.emptyList(), Collections.emptyList())));
+ pkg.cells.add(new RevisionStoreCell());
+ Metadata metadata = new Metadata();
+ walk(pkg, metadata);
+ assertArrayEquals(new String[] {sectionGuid},
+ metadata.getValues(OneNote.SECTION_GUID));
+ }
+
+ HeaderCell header = new HeaderCell();
+ header.objectData = new ObjectSpaceObjectPropSet();
+ header.objectData.body = fileIdentitySet(guidBytes);
+ MSOneStorePackage pkg = new MSOneStorePackage();
+ pkg.headerCell = header;
+ pkg.cells.add(new RevisionStoreCell());
+ Metadata metadata = new Metadata();
+ walk(pkg, metadata);
+ assertArrayEquals(new String[] {sectionGuid},
metadata.getValues(OneNote.SECTION_GUID));
+ }
+
+ @Test
+ public void testSectionShortCircuitStopsWithBudgetToSpare() throws
Exception {
+ byte[] secondGuidBytes = new byte[] {
+ 0x43, 0x32, 0x21, 0x10, 0x65, 0x54, (byte) 0x87, 0x76,
+ (byte) 0x98, (byte) 0xa9, (byte) 0xba, (byte) 0xcb,
+ (byte) 0xdc, (byte) 0xed, (byte) 0xfe, 0x0f
+ };
+ // the winning identity is NOT the cap-filling value, so every stop
below is
+ // attributable to the sectionCollected short-circuit alone: removing
the property
+ // loop or nested-array operand sends the scan into the deep chain,
removing the
+ // group-loop operand enters the second group - either way the entry
count grows
+ PrtArrayOfPropertyValues array = new PrtArrayOfPropertyValues();
+ array.data = new PropertySet[] {
+ fileIdentitySet(sectionGuidBytes()), propertySet()
+ };
+ PropertySet deep = propertySet();
+ for (int i = 0; i <= PropertySet.MAX_PROPERTY_NESTING; i++) {
+ deep = propertySet(new PropertySpec(PropertyType.PropertySet,
0x24000006, deep));
+ }
+ RevisionStoreObject first = object(id(1474),
+ propertySet(new
PropertySpec(PropertyType.ArrayOfPropertyValues,
+ 0x24000007, array),
+ new PropertySpec(PropertyType.PropertySet, 0x24000006,
deep)),
+ Collections.emptyList(), Collections.emptyList());
+ RevisionStoreObject afterLimit = object(id(1475),
fileIdentitySet(secondGuidBytes),
+ Collections.emptyList(), Collections.emptyList());
+ CountingMSOneStorePackage pkg = new CountingMSOneStorePackage();
+ fillGuidBudget(pkg, OneNoteGuidCollector.MAX_GUID_COUNT - 5);
+ pkg.dataRootCell = new RevisionStoreCell();
+ pkg.dataRootCell.objectGroups.add(group(first));
+ pkg.dataRootCell.objectGroups.add(group(afterLimit));
+ Metadata metadata = new Metadata();
+
+ walk(pkg, metadata);
+
+ assertArrayEquals(new String[]
{"{00112233-4455-6677-8899-AABBCCDDEEFF}"},
+ metadata.getValues(OneNote.SECTION_GUID));
+ assertEquals(2, pkg.identityScanEntries);
+ assertEquals(0,
Arrays.stream(metadata.getValues(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING))
+ .filter(w -> w.contains("Capping OneNote GUID
metadata")).count());
+ }
+
+ @Test
+ public void testSectionIdentityScanStopsAtGuidLimit() throws Exception {
+ byte[] firstGuidBytes = sectionGuidBytes();
+ byte[] secondGuidBytes = new byte[] {
+ 0x43, 0x32, 0x21, 0x10, 0x65, 0x54, (byte) 0x87, 0x76,
+ (byte) 0x98, (byte) 0xa9, (byte) 0xba, (byte) 0xcb,
+ (byte) 0xdc, (byte) 0xed, (byte) 0xfe, 0x0f
+ };
+ PrtArrayOfPropertyValues array = new PrtArrayOfPropertyValues();
+ array.data = new PropertySet[] {
+ fileIdentitySet(firstGuidBytes),
fileIdentitySet(secondGuidBytes)
+ };
+ RevisionStoreObject withArray = object(id(1470),
+ propertySet(new
PropertySpec(PropertyType.ArrayOfPropertyValues,
+ 0x24000007, array)), Collections.emptyList(),
Collections.emptyList());
+ RevisionStoreObject afterLimit = object(id(1471),
fileIdentitySet(secondGuidBytes),
+ Collections.emptyList(), Collections.emptyList());
+ CountingMSOneStorePackage pkg = new CountingMSOneStorePackage();
+ fillGuidBudget(pkg, OneNoteGuidCollector.MAX_GUID_COUNT - 1);
+ pkg.dataRootCell = new RevisionStoreCell();
+ pkg.dataRootCell.objectGroups.add(group(withArray));
+ pkg.dataRootCell.objectGroups.add(group(afterLimit));
+ Metadata metadata = new Metadata();
+
+ walk(pkg, metadata);
+
+ assertArrayEquals(new String[]
{"{00112233-4455-6677-8899-AABBCCDDEEFF}"},
+ metadata.getValues(OneNote.SECTION_GUID));
+ // exact-work check: the nested array's first element wins, so the
scan must not
+ // enter the second element, the rest of the property list, or the
second group
+ assertEquals(2, pkg.identityScanEntries);
+ assertEquals(1,
Arrays.stream(metadata.getValues(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING))
+ .filter(w -> w.contains("Capping OneNote GUID
metadata")).count());
+ }
+
+ @Test
+ public void testSectionScanStopsAtLimitWithExactWork() throws Exception {
+ byte[] secondGuidBytes = new byte[] {
+ 0x43, 0x32, 0x21, 0x10, 0x65, 0x54, (byte) 0x87, 0x76,
+ (byte) 0x98, (byte) 0xa9, (byte) 0xba, (byte) 0xcb,
+ (byte) 0xdc, (byte) 0xed, (byte) 0xfe, 0x0f
+ };
+ // a nested property-set chain deeper than MAX_PROPERTY_NESTING:
isFull and
+ // sectionCollected both fire at the winning identity here, so the
chain is reached
+ // only when BOTH stop conditions are removed (removing one leaves the
other
+ // stopping the scan); single-operand attribution is covered by
+ // testSectionShortCircuitStopsWithBudgetToSpare
+ PropertySet deep = propertySet();
+ for (int i = 0; i <= PropertySet.MAX_PROPERTY_NESTING; i++) {
+ deep = propertySet(new PropertySpec(PropertyType.PropertySet,
0x24000006, deep));
+ }
+ RevisionStoreObject first = object(id(1472),
+ propertySet(new
PropertySpec(PropertyType.FourBytesOfLengthFollowedByData,
+ 0x1C001D94, bytes(sectionGuidBytes())),
+ new PropertySpec(PropertyType.PropertySet, 0x24000006,
deep)),
+ Collections.emptyList(), Collections.emptyList());
+ RevisionStoreObject afterLimit = object(id(1473),
fileIdentitySet(secondGuidBytes),
+ Collections.emptyList(), Collections.emptyList());
+ CountingMSOneStorePackage pkg = new CountingMSOneStorePackage();
+ fillGuidBudget(pkg, OneNoteGuidCollector.MAX_GUID_COUNT - 1);
+ pkg.dataRootCell = new RevisionStoreCell();
+ pkg.dataRootCell.objectGroups.add(group(first, afterLimit));
+ Metadata metadata = new Metadata();
+
+ walk(pkg, metadata);
+
+ assertArrayEquals(new String[]
{"{00112233-4455-6677-8899-AABBCCDDEEFF}"},
+ metadata.getValues(OneNote.SECTION_GUID));
+ assertEquals(1, pkg.identityScanEntries);
+ assertEquals(1,
Arrays.stream(metadata.getValues(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING))
+ .filter(w -> w.contains("Capping OneNote GUID
metadata")).count());
+ }
+
+ @Test
+ public void testSectionGuidSourcePrecedence() throws Exception {
+ byte[] dataRootBytes = new byte[] {
+ 0x43, 0x32, 0x21, 0x10, 0x65, 0x54, (byte) 0x87, 0x76,
+ (byte) 0x98, (byte) 0xa9, (byte) 0xba, (byte) 0xcb,
+ (byte) 0xdc, (byte) 0xed, (byte) 0xfe, 0x0f
+ };
+ byte[] fallbackBytes = new byte[] {
+ 0x63, 0x52, 0x41, 0x30, (byte) 0x85, 0x74, (byte) 0xa7, (byte)
0x96,
+ (byte) 0xb8, (byte) 0xc9, (byte) 0xda, (byte) 0xeb,
+ (byte) 0xfc, 0x0d, 0x1e, 0x2f
+ };
+
+ // the header wins over both lower-priority sources
+ MSOneStorePackage withHeader = new MSOneStorePackage();
+ withHeader.headerCell = new HeaderCell();
+ withHeader.headerCell.objectData = new ObjectSpaceObjectPropSet();
+ withHeader.headerCell.objectData.body =
fileIdentitySet(sectionGuidBytes());
+ withHeader.dataRootCell = new RevisionStoreCell();
+ withHeader.dataRootCell.objectGroups.add(group(object(id(1500),
+ fileIdentitySet(dataRootBytes), Collections.emptyList(),
+ Collections.emptyList())));
+ withHeader.OtherFileNodeList.add(group(object(id(1501),
+ fileIdentitySet(fallbackBytes), Collections.emptyList(),
+ Collections.emptyList())));
+ Metadata headerMetadata = new Metadata();
+ walk(withHeader, headerMetadata);
+ assertEquals("{00112233-4455-6677-8899-AABBCCDDEEFF}",
+ headerMetadata.get(OneNote.SECTION_GUID));
+
+ // without a usable header, the data root wins over the
OtherFileNodeList fallback
+ MSOneStorePackage withoutHeader = new MSOneStorePackage();
+ withoutHeader.dataRootCell = new RevisionStoreCell();
+ withoutHeader.dataRootCell.objectGroups.add(group(object(id(1502),
+ fileIdentitySet(dataRootBytes), Collections.emptyList(),
+ Collections.emptyList())));
+ withoutHeader.OtherFileNodeList.add(group(object(id(1503),
+ fileIdentitySet(fallbackBytes), Collections.emptyList(),
+ Collections.emptyList())));
+ Metadata fallbackMetadata = new Metadata();
+ walk(withoutHeader, fallbackMetadata);
+ assertEquals("{10213243-5465-7687-98A9-BACBDCEDFE0F}",
+ fallbackMetadata.get(OneNote.SECTION_GUID));
+
+ // a scan entered after the section GUID was found must stop at the
group check
+ // (budget not full, so the short-circuit is the sectionCollected
operand): the
+ // assertion pins the collector at exactly one SECTION value, so
collecting the
+ // fallback GUID here would fail
+
withHeader.collectFileIdentityGuidsFromGroups(withHeader.OtherFileNodeList);
+
assertEquals(Collections.singleton("{00112233-4455-6677-8899-AABBCCDDEEFF}"),
+ sectionGuidValues(withHeader));
+ assertEquals("{00112233-4455-6677-8899-AABBCCDDEEFF}",
+ headerMetadata.get(OneNote.SECTION_GUID));
+ }
+
+ @Test
+ public void testCellRootsDeduplicatesAndCapsRoots() throws Exception {
+ RevisionStoreCell cell = new RevisionStoreCell();
+ RevisionStoreObject textRoot = object(id(2100), propertySet(
+ new PropertySpec(PropertyType.FourBytesOfLengthFollowedByData,
+ 0x1C003498, text("root text"))),
Collections.emptyList(),
+ Collections.emptyList());
+ cell.rootDeclares.add(rootDeclare(textRoot.objectID));
+ cell.rootDeclares.add(rootDeclare(textRoot.objectID));
+ List<RevisionStoreObject> roots = new ArrayList<>();
+ roots.add(textRoot);
+ for (int i = 0; i <= MSOneStorePackage.MAX_CELL_ROOTS; i++) {
+ RevisionStoreObject root = object(id(2101 + i), propertySet(),
+ Collections.emptyList(), Collections.emptyList());
+ roots.add(root);
+ cell.rootDeclares.add(rootDeclare(root.objectID));
+ }
+ cell.objectGroups.add(group(roots.toArray(new
RevisionStoreObject[0])));
+ MSOneStorePackage pkg = new MSOneStorePackage();
+ pkg.cells.add(cell);
+ Metadata metadata = new Metadata();
+
+ String xml = walk(pkg, metadata);
+
+ assertTrue(xml.contains("root text"));
+ assertEquals(1,
Arrays.stream(metadata.getValues(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING))
+ .filter(w -> w.contains("more than " +
MSOneStorePackage.MAX_CELL_ROOTS
+ + " root objects")).count());
+ }
+
+ @Test
+ public void testCellRootCapTruncationStillWalksContentRoot() throws
Exception {
+ // the cap must not silence the walk-everything fallback: 10,000
blob-only roots
+ // fill it, so the real content root is declare #10,001 and gets
truncated
+ RevisionStoreCell cell = new RevisionStoreCell();
+ List<RevisionStoreObject> roots = new ArrayList<>();
+ for (int i = 0; i < MSOneStorePackage.MAX_CELL_ROOTS; i++) {
+ RevisionStoreObject blob = new RevisionStoreObject();
+ blob.objectID = id(2200 + i);
+ roots.add(blob);
+ cell.rootDeclares.add(rootDeclare(blob.objectID));
+ }
+ RevisionStoreObject contentRoot = object(id(2199),
+ propertySet(new
PropertySpec(PropertyType.FourBytesOfLengthFollowedByData,
+ 0x1C003498, text("root text"))),
Collections.emptyList(),
+ Collections.emptyList());
+ roots.add(contentRoot);
+ cell.rootDeclares.add(rootDeclare(contentRoot.objectID));
+ cell.objectGroups.add(group(roots.toArray(new
RevisionStoreObject[0])));
+ MSOneStorePackage pkg = new MSOneStorePackage();
+ pkg.cells.add(cell);
+ Metadata metadata = new Metadata();
+
+ String xml = walk(pkg, metadata);
+
+ assertTrue(xml.contains("root text"));
+ assertEquals(1,
Arrays.stream(metadata.getValues(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING))
+ .filter(w -> w.contains("more than " +
MSOneStorePackage.MAX_CELL_ROOTS
+ + " root objects; ignoring the rest")).count());
+ assertEquals(1,
Arrays.stream(metadata.getValues(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING))
+ .filter(w -> w.contains("more than " +
MSOneStorePackage.MAX_CELL_ROOTS
+ + " root objects; walking all objects")).count());
+ }
+
+ @Test
+ public void testCellRootsDeduplicatesRootDeclares() throws Exception {
+ // two declares of the same object plus MAX_CELL_ROOTS - 1 distinct
roots fit the cap
+ // exactly when deduplicated; without dedupe the duplicate consumes a
slot and the
+ // last distinct root is truncated, so the absent-truncation assertion
fails
+ RevisionStoreCell cell = new RevisionStoreCell();
+ RevisionStoreObject textRoot = object(id(2300), propertySet(
+ new PropertySpec(PropertyType.FourBytesOfLengthFollowedByData,
+ 0x1C003498, text("root text"))),
Collections.emptyList(),
+ Collections.emptyList());
+ cell.rootDeclares.add(rootDeclare(textRoot.objectID));
+ cell.rootDeclares.add(rootDeclare(textRoot.objectID));
+ List<RevisionStoreObject> roots = new ArrayList<>();
+ roots.add(textRoot);
+ for (int i = 0; i < MSOneStorePackage.MAX_CELL_ROOTS - 1; i++) {
+ RevisionStoreObject root = object(id(2301 + i), propertySet(),
+ Collections.emptyList(), Collections.emptyList());
+ roots.add(root);
+ cell.rootDeclares.add(rootDeclare(root.objectID));
+ }
+ cell.objectGroups.add(group(roots.toArray(new
RevisionStoreObject[0])));
+ MSOneStorePackage pkg = new MSOneStorePackage();
+ pkg.cells.add(cell);
+ Metadata metadata = new Metadata();
+
+ String xml = walk(pkg, metadata);
+
+ assertTrue(xml.contains("root text"));
+ assertEquals(0,
Arrays.stream(metadata.getValues(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING))
+ .filter(w -> w.contains("more than " +
MSOneStorePackage.MAX_CELL_ROOTS
+ + " root objects")).count());
+ }
+
+ @Test
+ public void testPageGuidLookupFollowsReferencesAndFallbackRoots() throws
Exception {
+ String pageGuid = "{10213243-5465-7687-98A9-BACBDCEDFE0F}";
+ byte[] guidBytes = new byte[] {
+ 0x43, 0x32, 0x21, 0x10, 0x65, 0x54, (byte) 0x87, 0x76,
+ (byte) 0x98, (byte) 0xa9, (byte) 0xba, (byte) 0xcb,
+ (byte) 0xdc, (byte) 0xed, (byte) 0xfe, 0x0f
+ };
+ MSOneStorePackage pkg = new MSOneStorePackage();
+ RevisionStoreObject page = notebookGuidObject(1480,
OneNoteJcid.PAGE_METADATA, guidBytes);
+ RevisionStoreObject parent = object(id(1481),
+ propertySet(new PropertySpec(PropertyType.ObjectID,
0x20001D78, new NoData()),
+ new
PropertySpec(PropertyType.FourBytesOfLengthFollowedByData,
+ 0x1C003498, text("non-reference property"))),
+ Collections.singletonList(page.objectID),
Collections.emptyList());
+ Map<ExGuid, RevisionStoreObject> objectsById = new HashMap<>();
+ objectsById.put(parent.objectID, parent);
+ objectsById.put(page.objectID, page);
+ RevisionStoreCell cell = new RevisionStoreCell();
+ cell.objectGroups.add(group(parent, page));
+ cell.rootDeclares.add(rootDeclare(parent.objectID));
+
+ assertEquals(pageGuid, pkg.findPageGuid(cell, objectsById));
+
+ RevisionStoreCell fallbackCell = new RevisionStoreCell();
+ RevisionStoreObjectGroup noObjects = group();
+ noObjects.objects = null;
+ fallbackCell.objectGroups.add(null);
+ fallbackCell.objectGroups.add(noObjects);
+ fallbackCell.objectGroups.add(group(parent, page));
+ assertEquals(pageGuid, pkg.findPageGuid(fallbackCell, objectsById));
+ fallbackCell.rootDeclares.add(rootDeclare(id(1482)));
+ assertEquals(pageGuid, pkg.findPageGuid(fallbackCell, objectsById));
+ fallbackCell.rootDeclares = null;
+ assertEquals(pageGuid, pkg.findPageGuid(fallbackCell, objectsById));
+ assertNull(pkg.findPageGuid(null, objectsById));
+ RevisionStoreCell noGroups = new RevisionStoreCell();
+ noGroups.objectGroups = null;
+ assertNull(pkg.findPageGuid(noGroups, objectsById));
+
+ RevisionStoreObject resolvedRoot = object(id(1483),
+ propertySet(new
PropertySpec(PropertyType.FourBytesOfLengthFollowedByData,
+ 0x1C003498, text("resolved root without page GUID"))),
+ Collections.emptyList(), Collections.emptyList());
+ Map<ExGuid, RevisionStoreObject> roots = new HashMap<>(objectsById);
+ roots.put(resolvedRoot.objectID, resolvedRoot);
+ RevisionStoreCell noUnreferencedFallback = new RevisionStoreCell();
+ noUnreferencedFallback.objectGroups.add(group(resolvedRoot, page));
+
noUnreferencedFallback.rootDeclares.add(rootDeclare(resolvedRoot.objectID));
+ assertNull(pkg.findPageGuid(noUnreferencedFallback, roots));
+ noUnreferencedFallback.rootDeclares.clear();
+ noUnreferencedFallback.rootDeclares.add(rootDeclare(id(1482)));
+
noUnreferencedFallback.rootDeclares.add(rootDeclare(resolvedRoot.objectID));
+ assertNull(pkg.findPageGuid(noUnreferencedFallback, roots));
+ RevisionStoreObject rootWithoutPropertySet = new RevisionStoreObject();
+ rootWithoutPropertySet.objectID = id(1486);
+ Map<ExGuid, RevisionStoreObject> noPropertyRoots = new
HashMap<>(objectsById);
+ noPropertyRoots.put(rootWithoutPropertySet.objectID,
rootWithoutPropertySet);
+ RevisionStoreCell noPropertyRoot = new RevisionStoreCell();
+ noPropertyRoot.objectGroups.add(group(rootWithoutPropertySet, page));
+
noPropertyRoot.rootDeclares.add(rootDeclare(rootWithoutPropertySet.objectID));
+ assertNull(pkg.findPageGuid(noPropertyRoot, noPropertyRoots));
+ RevisionStoreCell emptyFallback = new RevisionStoreCell();
+ assertNull(pkg.findPageGuid(emptyFallback, objectsById));
+ RevisionStoreCell failedFallback = new RevisionStoreCell();
+ failedFallback.objectGroups.add(group());
+ failedFallback.objectGroups.add(group(rootWithoutPropertySet));
+ assertNull(pkg.findPageGuid(failedFallback, objectsById));
+
+ assertNull(pkg.findPageGuid((RevisionStoreObject) null, objectsById,
+ new HashSet<>(), 0));
+ assertNull(pkg.findPageGuid(page, objectsById,
+ new HashSet<>(Collections.singleton(page.objectID)), 0));
+ assertNull(pkg.findPageGuid(page, objectsById, new HashSet<>(), 1000));
+ RevisionStoreObject malformedPage = notebookGuidObject(1487,
+ OneNoteJcid.PAGE_METADATA, new byte[17]);
+ assertNull(pkg.findPageGuid(malformedPage, objectsById, new
HashSet<>(), 0));
+ RevisionStoreObject noChildReference = object(id(1488),
+ propertySet(new PropertySpec(PropertyType.ObjectID,
0x20001D78, new NoData())),
+ Collections.emptyList(), Collections.emptyList());
+ assertNull(pkg.findPageGuid(noChildReference, objectsById, new
HashSet<>(), 0));
+ RevisionStoreObject noPropertySet = new RevisionStoreObject();
+ assertNull(pkg.findPageGuid(noPropertySet, objectsById, new
HashSet<>(), 0));
+ RevisionStoreObject noObjectSpace = object(id(1484),
+ propertySet(new PropertySpec(PropertyType.NoData, 0x20001D78,
new NoData())),
+ Collections.emptyList(), Collections.emptyList());
+ noObjectSpace.propertySet.objectSpaceObjectPropSet = null;
+ assertNull(pkg.findPageGuid(noObjectSpace, objectsById, new
HashSet<>(), 0));
+ RevisionStoreObject noObjectId = notebookGuidObject(1485,
+ OneNoteJcid.PAGE_METADATA, guidBytes);
+ noObjectId.objectID = null;
+ assertEquals(pageGuid, pkg.findPageGuid(noObjectId, objectsById, new
HashSet<>(), 0));
+ }
+
+ @Test
+ public void testPageEmissionBuildsOneObjectIndex() throws Exception {
+ String pageGuid = "{10213243-5465-7687-98A9-BACBDCEDFE0F}";
+ byte[] guidBytes = new byte[] {
+ 0x43, 0x32, 0x21, 0x10, 0x65, 0x54, (byte) 0x87, 0x76,
+ (byte) 0x98, (byte) 0xa9, (byte) 0xba, (byte) 0xcb,
+ (byte) 0xdc, (byte) 0xed, (byte) 0xfe, 0x0f
+ };
+ CountingMSOneStorePackage pkg = new CountingMSOneStorePackage();
+ RevisionStoreCell cell = new RevisionStoreCell();
+ cell.objectGroups.add(group(notebookGuidObject(1498,
OneNoteJcid.PAGE_METADATA,
+ guidBytes)));
+ Metadata metadata = new Metadata();
+ ParseContext context = new ParseContext();
+ ToXMLContentHandler xml = new ToXMLContentHandler();
+ XHTMLContentHandler xhtml = new XHTMLContentHandler(xml, metadata,
context);
+ xhtml.startDocument();
+
+ pkg.emitPage(cell, new OneNoteTreeWalkerOptions(), metadata, xhtml);
+
+ xhtml.endDocument();
+ assertEquals(1, pkg.objectIndexBuilds);
+ assertTrue(xml.toString().contains("id=\"" + pageGuid + "\""));
+ }
+
+ @Test
+ public void
testNotebookManagementGuidRejectsMissingAndMalformedProperties() throws
Exception {
+ MSOneStorePackage pkg = new MSOneStorePackage();
+ assertNull(pkg.notebookManagementEntityGuid(null));
+ assertNull(pkg.notebookManagementEntityGuid(new
RevisionStoreObject()));
+
+ RevisionStoreObject noObjectSpace = object(id(1490),
+ propertySet(new PropertySpec(PropertyType.NoData, 0x20001D78,
new NoData())),
+ Collections.emptyList(), Collections.emptyList());
+ noObjectSpace.propertySet.objectSpaceObjectPropSet = null;
+ assertNull(pkg.notebookManagementEntityGuid(noObjectSpace));
+
+ byte[] validGuid = sectionGuidBytes();
+ RevisionStoreObject unrelated = object(id(1491),
+ propertySet(new
PropertySpec(PropertyType.FourBytesOfLengthFollowedByData,
+ 0x1C003498, bytes(validGuid))),
Collections.emptyList(),
+ Collections.emptyList());
+ assertNull(pkg.notebookManagementEntityGuid(unrelated));
+ RevisionStoreObject wrongType = object(id(1492),
+ propertySet(new PropertySpec(PropertyType.NoData, 0x1C001C30,
new NoData())),
+ Collections.emptyList(), Collections.emptyList());
+ assertNull(pkg.notebookManagementEntityGuid(wrongType));
+ RevisionStoreObject malformed = object(id(1493),
+ propertySet(new
PropertySpec(PropertyType.FourBytesOfLengthFollowedByData,
+ 0x1C001C30, bytes(new byte[17]))),
Collections.emptyList(),
+ Collections.emptyList());
+ assertNull(pkg.notebookManagementEntityGuid(malformed));
+ RevisionStoreObject valid = notebookGuidObject(1494,
OneNoteJcid.PAGE_METADATA,
+ validGuid);
+ assertEquals("{00112233-4455-6677-8899-AABBCCDDEEFF}",
+ pkg.notebookManagementEntityGuid(valid));
+ pkg.recordEntityGuid(null, OneNoteJcid.PAGE_METADATA);
+ }
+
+ @Test
+ public void testFssJcidIndexRequiresCompleteJcid() throws Exception {
+ assertEquals(-1, MSOneStorePackage.jcidIndex(null));
+ RevisionStoreObject object = new RevisionStoreObject();
+ assertEquals(-1, MSOneStorePackage.jcidIndex(object));
+ object.jcid = new JCIDObject(null, emptyObjectData());
+ object.jcid.jcid = null;
+ assertEquals(-1, MSOneStorePackage.jcidIndex(object));
+ object.jcid = new JCIDObject(null, emptyObjectData());
+ object.jcid.jcid.index = OneNoteJcid.PAGE_METADATA;
+ assertEquals(OneNoteJcid.PAGE_METADATA,
MSOneStorePackage.jcidIndex(object));
+ }
+
+ @Test
+ public void testGuidCapWarningIsPublishedAsParseWarning() throws Exception
{
+ MSOneStorePackage pkg = new MSOneStorePackage();
+ fillGuidBudget(pkg, OneNoteGuidCollector.MAX_GUID_COUNT);
+ pkg.guidCollector().add(OneNoteGuidCollector.Category.PAGE,
"overflow-1");
+
+ Metadata metadata = new Metadata();
+ walk(pkg, metadata);
+ assertEquals(OneNoteGuidCollector.MAX_GUID_COUNT,
+ metadata.getValues(OneNote.PAGE_GUIDS).length);
+ String[] warnings =
metadata.getValues(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING);
+ assertEquals(1, Arrays.stream(warnings)
+ .filter(warning -> warning.contains("Capping OneNote GUID
metadata")).count());
+ }
+
+ @Test
+ public void testGuidCapWarningSurvivesSuppressedParseWarnings() throws
Exception {
+ MSOneStorePackage pkg = new MSOneStorePackage();
+ for (int i = 0; i < 101; i++) {
+ pkg.recordParseWarning("warning " + i);
+ }
+ fillGuidBudget(pkg, OneNoteGuidCollector.MAX_GUID_COUNT - 1);
+ RevisionStoreObject pageMetadata = notebookGuidObject(1304,
OneNoteJcid.PAGE_METADATA,
+ new byte[] {
+ 0x43, 0x32, 0x21, 0x10, 0x65, 0x54, (byte) 0x87, 0x76,
+ (byte) 0x98, (byte) 0xa9, (byte) 0xba, (byte) 0xcb,
+ (byte) 0xdc, (byte) 0xed, (byte) 0xfe, 0x0f
+ });
+ RevisionStoreCell pageCell = new RevisionStoreCell();
+ pageCell.objectGroups.add(group(pageMetadata));
+ pageCell.rootDeclares.add(rootDeclare(pageMetadata.objectID));
+ pkg.cells.add(pageCell);
+ Metadata metadata = new Metadata();
+
+ walk(pkg, metadata);
+
+ String[] warnings =
metadata.getValues(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING);
+ assertTrue(Arrays.stream(warnings)
+ .anyMatch(warning -> warning.contains("Additional OneNote
parse warnings were suppressed")));
+ // the cap signal is exempt from the budget the generic notice consumed
+ assertEquals(1, Arrays.stream(warnings)
+ .filter(warning -> warning.contains("Capping OneNote GUID
metadata")).count());
+ assertEquals(OneNoteGuidCollector.MAX_GUID_COUNT,
+ metadata.getValues(OneNote.PAGE_GUIDS).length);
+ }
+
+ @Test
+ public void testGuidsCollectedBeforeWalkFailureArePublished() throws
Exception {
+ String sectionGuid = "{00112233-4455-6677-8899-AABBCCDDEEFF}";
+ String pageGuid = "{10213243-5465-7687-98A9-BACBDCEDFE0F}";
+ RevisionStoreObject sectionIdentity = object(id(1302),
+ propertySet(new
PropertySpec(PropertyType.FourBytesOfLengthFollowedByData,
+ 0x1C001D94, bytes(sectionGuidBytes()))),
Collections.emptyList(),
+ Collections.emptyList());
+ RevisionStoreObject pageMetadata = object(id(1303),
+ propertySet(new
PropertySpec(PropertyType.FourBytesOfLengthFollowedByData,
+ 0x1C001C30, bytes(new byte[] {
+ 0x43, 0x32, 0x21, 0x10, 0x65, 0x54, (byte)
0x87, 0x76,
+ (byte) 0x98, (byte) 0xa9, (byte) 0xba, (byte)
0xcb,
+ (byte) 0xdc, (byte) 0xed, (byte) 0xfe, 0x0f
+ }))), Collections.emptyList(),
+ Collections.emptyList());
+ setJcid(pageMetadata, OneNoteJcid.PAGE_METADATA);
+ RevisionStoreCell validCell = new RevisionStoreCell();
+ validCell.objectGroups.add(group(pageMetadata));
+ validCell.rootDeclares.add(rootDeclare(pageMetadata.objectID));
+ RevisionStoreCell damagedCell = new RevisionStoreCell();
+ damagedCell.objectGroups = null;
+
+ // deliberate failure seam: a future null guard inside
indexObjectsById must not
+ // turn this test green for the wrong reason
+ MSOneStorePackage pkg = new MSOneStorePackage() {
+ @Override
+ Map<ExGuid, RevisionStoreObject> indexObjectsById(
+ List<RevisionStoreObjectGroup> objectGroups) {
+ if (objectGroups == null) {
+ throw new IllegalStateException("forced index failure");
+ }
+ return super.indexObjectsById(objectGroups);
+ }
+ };
+ pkg.OtherFileNodeList.add(group(sectionIdentity));
+ pkg.cells.add(validCell);
+ pkg.cells.add(damagedCell);
+ Metadata metadata = new Metadata();
+
+ assertThrows(IllegalStateException.class, () -> walk(pkg, metadata));
+ assertArrayEquals(new String[] {sectionGuid},
metadata.getValues(OneNote.SECTION_GUID));
+ assertArrayEquals(new String[] {pageGuid},
metadata.getValues(OneNote.PAGE_GUIDS));
+ }
+
@Test
public void testParseWarningsAreBoundedAcrossParserAndWalkPhases() throws
Exception {
MSOneStorePackage pkg = new MSOneStorePackage();
@@ -135,6 +950,10 @@ public class MSOneStorePackageTest {
RevisionStoreCell oldPageOneCell = cellWithText(oldPageOne, "old page
one");
RevisionStoreCell pageOneCell = cellWithText(pageOne, "page one");
RevisionStoreCell unrelatedCell = cellWithText(cell(30, 300),
"unrelated");
+ // reverse-lexical page GUIDs: page one's GUID sorts after page two's,
so the sorted
+ // bag order cannot be mistaken for document order
+ attachPageGuid(pageOneCell, 3100, (byte) 0x60);
+ attachPageGuid(pageTwoCell, 3200, (byte) 0x50);
RevisionStoreObject sectionRoot = object(sectionRootId,
propertySet(new PropertySpec(PropertyType.ObjectSpaceID,
0x20001D78,
@@ -156,12 +975,25 @@ public class MSOneStorePackageTest {
pkg.cells.addAll(Arrays.asList(pageTwoCell, oldPageOneCell,
pageOneCell, unrelatedCell));
Metadata metadata = new Metadata();
- String text = walk(pkg, metadata);
+ // div ids are attributes, so the pairing assertions need the XML
output
+ String text = walkXml(pkg, metadata);
assertTrue(text.indexOf("page one") >= 0);
assertTrue(text.indexOf("page two") >= 0);
assertTrue(text.indexOf("page one") < text.indexOf("page two"));
assertFalse(text.contains("old page one"));
assertTrue(text.contains("unrelated"));
+ // each page div carries its own page GUID, in document order
+ int idPageOne =
text.indexOf("id=\"{60000000-0000-0000-0000-000000000000}\"");
+ int idPageTwo =
text.indexOf("id=\"{50000000-0000-0000-0000-000000000000}\"");
+ assertTrue(idPageOne >= 0);
+ assertTrue(idPageTwo >= 0);
+ assertTrue(idPageOne < text.indexOf("page one"));
+ assertTrue(text.indexOf("page one") < idPageTwo);
+ assertTrue(idPageTwo < text.indexOf("page two"));
+ // the published bag is sorted, not document-ordered
+ assertArrayEquals(new String[]
{"{50000000-0000-0000-0000-000000000000}",
+ "{60000000-0000-0000-0000-000000000000}"},
+ metadata.getValues(OneNote.PAGE_GUIDS));
assertTrue(Arrays.stream(metadata.getValues(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING))
.anyMatch(warning -> warning.contains("could not be
resolved")));
}
@@ -313,6 +1145,90 @@ public class MSOneStorePackageTest {
assertEquals("Single Author",
metadata.get(OneNote.MOST_RECENT_AUTHORS));
}
+ @Test
+ public void testDualRoleAuthorSkipsMissingPropertySetAndMalformedValue()
throws Exception {
+ RevisionStoreObject noPropertySet = new RevisionStoreObject();
+ noPropertySet.objectID = id(712);
+ RevisionStoreObject malformedAuthor = object(id(714), propertySet(
+ new PropertySpec(PropertyType.NoData, 0x1C001D75, new
NoData())),
+ Collections.emptyList(), Collections.emptyList());
+ RevisionStoreObject nullSetRoot = object(id(713), propertySet(
+ new PropertySpec(PropertyType.ObjectID, 0x20001D78,
new NoData()),
+ new PropertySpec(PropertyType.ObjectID, 0x20001D79,
new NoData())),
+ Arrays.asList(noPropertySet.objectID, noPropertySet.objectID),
+ Collections.emptyList());
+ RevisionStoreObject malformedRoot = object(id(715), propertySet(
+ new PropertySpec(PropertyType.ObjectID, 0x20001D78,
new NoData()),
+ new PropertySpec(PropertyType.ObjectID, 0x20001D79,
new NoData())),
+ Arrays.asList(malformedAuthor.objectID,
malformedAuthor.objectID),
+ Collections.emptyList());
+ RevisionStoreObject noObjectSpace = object(id(716), propertySet(
+ new PropertySpec(PropertyType.ObjectID, 0x20001D78,
new NoData())),
+ Collections.emptyList(), Collections.emptyList());
+ noObjectSpace.propertySet.objectSpaceObjectPropSet = null;
+ RevisionStoreObject emptySpaceRoot = object(id(717), propertySet(
+ new PropertySpec(PropertyType.ObjectID, 0x20001D78,
new NoData()),
+ new PropertySpec(PropertyType.ObjectID, 0x20001D79,
new NoData())),
+ Arrays.asList(noObjectSpace.objectID, noObjectSpace.objectID),
+ Collections.emptyList());
+ RevisionStoreCell cell = new RevisionStoreCell();
+ cell.objectGroups.add(group(nullSetRoot, noPropertySet, malformedRoot,
malformedAuthor,
+ emptySpaceRoot, noObjectSpace));
+ cell.rootDeclares.add(rootDeclare(nullSetRoot.objectID));
+ cell.rootDeclares.add(rootDeclare(malformedRoot.objectID));
+ cell.rootDeclares.add(rootDeclare(emptySpaceRoot.objectID));
+ MSOneStorePackage pkg = new MSOneStorePackage();
+ pkg.cells.add(cell);
+ Metadata metadata = new Metadata();
+
+ walk(pkg, metadata);
+
+ assertEquals(0, metadata.getValues(TikaCoreProperties.CREATOR).length);
+ }
+
+ @Test
+ public void testHeaderGuidWinsOverDataRootScan() throws Exception {
+ byte[] dataRootGuidBytes = new byte[] {
+ 0x43, 0x32, 0x21, 0x10, 0x65, 0x54, (byte) 0x87, 0x76,
+ (byte) 0x98, (byte) 0xa9, (byte) 0xba, (byte) 0xcb,
+ (byte) 0xdc, (byte) 0xed, (byte) 0xfe, 0x0f
+ };
+ HeaderCell header = new HeaderCell();
+ header.objectData = new ObjectSpaceObjectPropSet();
+ header.objectData.body = fileIdentitySet(sectionGuidBytes());
+ MSOneStorePackage pkg = new MSOneStorePackage();
+ pkg.headerCell = header;
+ pkg.dataRootCell = new RevisionStoreCell();
+ pkg.dataRootCell.objectGroups.add(group(object(id(1500),
+ fileIdentitySet(dataRootGuidBytes), Collections.emptyList(),
+ Collections.emptyList())));
+ Metadata metadata = new Metadata();
+
+ walk(pkg, metadata);
+
+ assertEquals("{00112233-4455-6677-8899-AABBCCDDEEFF}",
+ metadata.get(OneNote.SECTION_GUID));
+ }
+
+ @Test
+ public void testWalksCellWithMissingRootDeclares() throws Exception {
+ RevisionStoreCell cell = new RevisionStoreCell();
+ cell.rootDeclares = null;
+ cell.objectGroups.add(group(object(id(630), propertySet(
+ new
PropertySpec(PropertyType.FourBytesOfLengthFollowedByData,
+ 0x1C003498, text("late body text"))),
+ Collections.emptyList(), Collections.emptyList())));
+ MSOneStorePackage pkg = new MSOneStorePackage();
+ pkg.cells.add(cell);
+ Metadata metadata = new Metadata();
+
+ String xml = walk(pkg, metadata);
+
+ assertTrue(xml.contains("late body text"));
+
assertTrue(Arrays.stream(metadata.getValues(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING))
+ .anyMatch(warning -> warning.contains("could not be resolved;
walking all objects")));
+ }
+
@Test
public void testBlobOnlyRootWithDanglingContentRootWalksAllObjects()
throws Exception {
RevisionStoreObject blobRoot = object(id(620), propertySet(),
@@ -690,6 +1606,56 @@ public class MSOneStorePackageTest {
assertTrue(text.contains("hi"));
}
+ private static class CountingMSOneStorePackage extends MSOneStorePackage {
+ private int objectIndexBuilds;
+ private int identityScanEntries;
+
+ @Override
+ Map<ExGuid, RevisionStoreObject> indexObjectsById(
+ List<RevisionStoreObjectGroup> objectGroups) {
+ objectIndexBuilds++;
+ return super.indexObjectsById(objectGroups);
+ }
+
+ @Override
+ void collectFileIdentityGuids(PropertySet propertySet, int depth) {
+ identityScanEntries++;
+ super.collectFileIdentityGuids(propertySet, depth);
+ }
+ }
+
+ private static byte[] sectionGuidBytes() {
+ return new byte[] {
+ 0x33, 0x22, 0x11, 0x00, 0x55, 0x44, 0x77, 0x66,
+ (byte) 0x88, (byte) 0x99, (byte) 0xaa, (byte) 0xbb,
+ (byte) 0xcc, (byte) 0xdd, (byte) 0xee, (byte) 0xff
+ };
+ }
+
+ private static PropertySet fileIdentitySet(byte[] value) {
+ return propertySet(new
PropertySpec(PropertyType.FourBytesOfLengthFollowedByData,
+ 0x1C001D94, bytes(value)));
+ }
+
+ private static RevisionStoreObject notebookGuidObject(int objectId, int
jcid, byte[] value)
+ throws Exception {
+ RevisionStoreObject object = object(id(objectId),
+ propertySet(new
PropertySpec(PropertyType.FourBytesOfLengthFollowedByData,
+ 0x1C001C30, bytes(value))), Collections.emptyList(),
Collections.emptyList());
+ setJcid(object, jcid);
+ return object;
+ }
+
+ private static void fillGuidBudget(MSOneStorePackage pkg, int count) {
+ for (int i = 0; i < count; i++) {
+ pkg.guidCollector().add(OneNoteGuidCollector.Category.PAGE,
"budget-" + i);
+ }
+ }
+
+ private static Set<String> sectionGuidValues(MSOneStorePackage pkg) {
+ return
pkg.guidCollector().values(OneNoteGuidCollector.Category.SECTION);
+ }
+
private static String walk(MSOneStorePackage pkg) throws Exception {
Metadata metadata = new Metadata();
return walk(pkg, metadata);
@@ -707,7 +1673,10 @@ public class MSOneStorePackageTest {
}
private static String walkXml(MSOneStorePackage pkg) throws Exception {
- Metadata metadata = new Metadata();
+ return walkXml(pkg, new Metadata());
+ }
+
+ private static String walkXml(MSOneStorePackage pkg, Metadata metadata)
throws Exception {
ParseContext context = new ParseContext();
ToXMLContentHandler xml = new ToXMLContentHandler();
XHTMLContentHandler xhtml = new XHTMLContentHandler(xml, metadata,
context);
@@ -721,6 +1690,29 @@ public class MSOneStorePackageTest {
return value.split(java.util.regex.Pattern.quote(needle), -1).length -
1;
}
+ private static RevisionManifestRootDeclare rootDeclare(ExGuid objectID) {
+ RevisionManifestRootDeclare declare = new
RevisionManifestRootDeclare();
+ declare.objectExGuid = objectID;
+ return declare;
+ }
+
+ private static void setJcid(RevisionStoreObject object, int index) throws
Exception {
+ JCIDObject jcidObject = new JCIDObject(null, emptyObjectData());
+ jcidObject.jcid.index = index;
+ object.jcid = jcidObject;
+ }
+
+ /** Adds a PAGE_METADATA guid object as a second root so the page div
carries its id. */
+ private static void attachPageGuid(RevisionStoreCell cell, int objectId,
byte firstGroupByte)
+ throws Exception {
+ byte[] guidBytes = new byte[16];
+ guidBytes[3] = firstGroupByte;
+ RevisionStoreObject pageMetadata = notebookGuidObject(objectId,
+ OneNoteJcid.PAGE_METADATA, guidBytes);
+ cell.objectGroups.add(group(pageMetadata));
+ cell.rootDeclares.add(rootDeclare(pageMetadata.objectID));
+ }
+
private static RevisionStoreCell cellWithText(CellID cellID, String value)
throws Exception {
RevisionStoreObject object = object(id(cellID.extendGUID1.hashCode()),
propertySet(new
PropertySpec(PropertyType.FourBytesOfLengthFollowedByData,
@@ -786,6 +1778,8 @@ public class MSOneStorePackageTest {
return set;
}
+ // PropertyID.type and .value are set independently here; the real
deserializer derives
+ // the type from the value's high bits, so some fixtures are deliberately
impossible inputs
private static PropertyID propertyID(PropertyType type, int value) {
PropertyID id = new PropertyID();
id.type = type.getIntVal();
