This is an automated email from the ASF dual-hosted git repository.
tballison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git
The following commit(s) were added to refs/heads/main by this push:
new 9c2a853f94 [TIKA-5055] Improve onenote performance (#3565)
9c2a853f94 is described below
commit 9c2a853f94ba3e439c1eb4b11e473fa65d1f5dc1
Author: Henry Lindeman <[email protected]>
AuthorDate: Fri Nov 6 16:42:41 2026 -0800
[TIKA-5055] Improve onenote performance (#3565)
* Improve onenote parsing performance
Optimization | Section-1 Time Saved | Section-2 Time Saved
---|---|---
BitReader.getBytes shifts instead of BitSet | 0.78 ms | 1.17 ms
BinaryItem stores byte[], not List<Byte> | 0.09 ms | 5.33 ms
GUID→UUID cache | 0.08 ms | 0.12 ms
Generated-by: Glean Tau <various models>
Signed-off-by: Henry Lindeman <[email protected]>
* AI Review comments
Generated-by: Glean Tau <various models>
Signed-off-by: Henry Lindeman <[email protected]>
---------
Signed-off-by: Henry Lindeman <[email protected]>
---
CHANGES.txt | 6 +
.../onenote/fsshttpb/MSOneStorePackage.java | 14 ++-
.../onenote/fsshttpb/streamobj/FileDataObject.java | 30 +++--
.../onenote/fsshttpb/streamobj/JCIDObject.java | 3 +-
.../onenote/fsshttpb/streamobj/LeafNodeObject.java | 6 +-
.../onenote/fsshttpb/streamobj/ObjectDataBLOB.java | 15 ++-
.../fsshttpb/streamobj/PropertySetObject.java | 4 +-
.../streamobj/RevisionStoreObjectGroup.java | 3 +-
.../fsshttpb/streamobj/basic/BinaryItem.java | 105 ++++++++++++++---
.../fsshttpb/streamobj/basic/HeaderCell.java | 4 +-
.../microsoft/onenote/fsshttpb/util/BitReader.java | 85 +++++++++-----
.../onenote/fsshttpb/MSOneStoreBlobTest.java | 16 +--
.../onenote/fsshttpb/MSOneStorePackageTest.java | 6 +-
.../fsshttpb/streamobj/basic/BinaryItemTest.java | 129 +++++++++++++++++++++
.../onenote/fsshttpb/util/BitReaderTest.java | 118 +++++++++++++++++++
15 files changed, 453 insertions(+), 91 deletions(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index 6b61b28444..def32a6c9c 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -611,6 +611,12 @@ Release 4.2.0 - 10/25/2026
* OneNote: table and list structure in FSSHTTPB files (TIKA-4943); section
and page
GUID metadata (TIKA-4944).
+ * OneNote FSSHTTPB parsing is ~6% faster with ~20% fewer GCs: binary items
are read
+ in place instead of copied, bit reads assemble whole bytes, and GUIDs are
cached.
+ Compat: BinaryItem.content (a List<Byte>) is replaced by
getContent()/setContent()
+ on byte[]; a binary item whose length decodes as 2^63 or more now fails
the
+ FSSHTTPB parse (TIKA-5055).
+
* Detection: *.rs is text/x-rust (TIKA-4977); a MIME email nesting
multipart/related is message/rfc822, not MHTML (TIKA-4973); XML with an
element
like <htmlPath> is no longer HTML (TIKA-4962); XUL is detected by content
and
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStorePackage.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStorePackage.java
index 0a02f61e93..c77156dd3a 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStorePackage.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStorePackage.java
@@ -64,6 +64,7 @@ import
org.apache.tika.parser.microsoft.onenote.fsshttpb.property.IProperty;
import
org.apache.tika.parser.microsoft.onenote.fsshttpb.property.PrtArrayOfPropertyValues;
import
org.apache.tika.parser.microsoft.onenote.fsshttpb.property.PrtFourBytesOfLengthFollowedByData;
import
org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.CellManifestDataElementData;
+import
org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.FileDataObject;
import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.PropertySet;
import
org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.RevisionManifestDataElementData;
import
org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.RevisionManifestRootDeclare;
@@ -614,7 +615,7 @@ public class MSOneStorePackage {
}
if (object.fileDataObject != null) {
// the object carries opaque binary data, e.g. an embedded image
or file
- handleEmbedded(object.fileDataObject.getData(), xhtml,
resourceInfo, object.objectID);
+ handleEmbedded(object.fileDataObject, xhtml, resourceInfo,
object.objectID);
}
if (object.propertySet == null ||
object.propertySet.objectSpaceObjectPropSet == null) {
@@ -1115,8 +1116,7 @@ public class MSOneStorePackage {
if (object == null) {
return false;
}
- byte[] data = object.fileDataObject == null ? null :
object.fileDataObject.getData();
- if (data != null && data.length > 0) {
+ if (object.fileDataObject != null && object.fileDataObject.hasData()) {
return true;
}
if (object.propertySet == null ||
object.propertySet.objectSpaceObjectPropSet == null) {
@@ -1691,10 +1691,10 @@ public class MSOneStorePackage {
* Hands the binary data of an embedded object (e.g. an image or an
attached file) to the
* embedded document extractor.
*/
- private void handleEmbedded(byte[] data, XHTMLContentHandler xhtml,
+ private void handleEmbedded(FileDataObject fileData, XHTMLContentHandler
xhtml,
EmbeddedResourceInfo resourceInfo, ExGuid
objectID)
throws SAXException, IOException {
- if (data == null || data.length == 0 || embeddedDocumentExtractor ==
null) {
+ if (!fileData.hasData() || embeddedDocumentExtractor == null) {
return;
}
Metadata embeddedMetadata = Metadata.newInstance(this.parseContext);
@@ -1716,7 +1716,9 @@ public class MSOneStorePackage {
}
xhtml.startElement("div", attributes);
xhtml.endElement("div");
- try (TikaInputStream tis = TikaInputStream.get(data)) {
+ // A byte-array stream, not one re-opened over the parsed buffer:
parsers that need a
+ // seekable channel (zip containers, PDF) get the array zero-copy
instead of a temp file.
+ try (TikaInputStream tis = TikaInputStream.get(fileData.getData())) {
if
(embeddedDocumentExtractor.shouldParseEmbedded(embeddedMetadata, parseContext))
{
// only counts as content when the extractor accepts it, so a
declined
// embedded object in an otherwise-empty file still triggers
the fallback
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/FileDataObject.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/FileDataObject.java
index d3a70780a9..e7a33c20b8 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/FileDataObject.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/FileDataObject.java
@@ -25,17 +25,33 @@ public class FileDataObject {
public DataElement objectDataBLOBDataElement;
/**
- * @return the opaque binary data of this file data object, or null if it
could not be
- * resolved.
+ * @return a copy of the opaque binary data of this file data object, or
null if it could
+ * not be resolved.
*/
public byte[] getData() {
+ ObjectDataBLOB blob = getBlob();
+ return blob == null ? null : blob.getData();
+ }
+
+ /**
+ * @return the length of the binary data, 0 if it could not be resolved
+ */
+ public int getDataLength() {
+ ObjectDataBLOB blob = getBlob();
+ return blob == null ? 0 : blob.getDataLength();
+ }
+
+ /**
+ * @return whether the file data resolves to non-empty binary data,
without copying it.
+ */
+ public boolean hasData() {
+ return getDataLength() > 0;
+ }
+
+ private ObjectDataBLOB getBlob() {
if (objectDataBLOBDataElement != null &&
objectDataBLOBDataElement.data instanceof
ObjectDataBLOBDataElementData) {
- ObjectDataBLOBDataElementData blobData =
- (ObjectDataBLOBDataElementData)
objectDataBLOBDataElement.data;
- if (blobData.objectDataBLOB != null) {
- return blobData.objectDataBLOB.getData();
- }
+ return ((ObjectDataBLOBDataElementData)
objectDataBLOBDataElement.data).objectDataBLOB;
}
return null;
}
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/JCIDObject.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/JCIDObject.java
index 465a264205..28b5b605a4 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/JCIDObject.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/JCIDObject.java
@@ -19,7 +19,6 @@ package
org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj;
import java.io.IOException;
import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.basic.JCID;
-import org.apache.tika.parser.microsoft.onenote.fsshttpb.util.ByteUtil;
/**
@@ -39,6 +38,6 @@ public class JCIDObject {
ObjectGroupObjectData objectData) throws IOException {
this.objectDeclaration = objectDeclaration;
this.jcid = new JCID();
-
this.jcid.doDeserializeFromByteArray(ByteUtil.toByteArray(objectData.data.content),
0);
+ this.jcid.doDeserializeFromByteArray(objectData.data.getContent(), 0);
}
}
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/LeafNodeObject.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/LeafNodeObject.java
index 3630dfc741..98fe156c5b 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/LeafNodeObject.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/LeafNodeObject.java
@@ -128,7 +128,7 @@ public class LeafNodeObject extends NodeObject {
AtomicReference<IntermediateNodeObject> rootNode = new
AtomicReference<>();
AtomicInteger index = new AtomicInteger(0);
- if
(StreamObject.tryGetCurrent(ByteUtil.toByteArray(dataObj.data.content), index,
node,
+ if (StreamObject.tryGetCurrent(dataObj.data.getContent(), index,
node,
LeafNodeObject.class)) {
if (dataObj.objectExGUIDArray == null) {
throw new TikaException(
@@ -146,7 +146,7 @@ public class LeafNodeObject extends NodeObject {
BinaryItem data = dataNodeData.data;
node.get().dataNodeObjectData =
- new
DataNodeObjectData(ByteUtil.toByteArray(data.content), 0,
+ new DataNodeObjectData(data.getContent(), 0,
(int) data.length.getDecodedValue());
node.get().dataNodeObjectData.exGuid =
dataObj.objectExGUIDArray.content.get(0);
node.get().intermediateNodeObjectList = null;
@@ -164,7 +164,7 @@ public class LeafNodeObject extends NodeObject {
intermediateData, extGuid));
}
}
- } else if
(StreamObject.tryGetCurrent(ByteUtil.toByteArray(dataObj.data.content), index,
+ } else if (StreamObject.tryGetCurrent(dataObj.data.getContent(),
index,
rootNode, IntermediateNodeObject.class)) {
// In Sub chunking for larger than 1MB zip file, MOSS2010
could return IntermediateNodeObject.
// For easy further process, the rootNode will be replaced by
intermediate node instead.
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/ObjectDataBLOB.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/ObjectDataBLOB.java
index 2873e1d013..2b5e359492 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/ObjectDataBLOB.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/ObjectDataBLOB.java
@@ -23,7 +23,6 @@ import java.util.concurrent.atomic.AtomicInteger;
import org.apache.tika.exception.TikaException;
import
org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.basic.BasicObject;
import
org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.basic.BinaryItem;
-import org.apache.tika.parser.microsoft.onenote.fsshttpb.util.ByteUtil;
/**
* Specifies an object data BLOB stream object - the opaque binary data of an
object,
@@ -38,13 +37,17 @@ public class ObjectDataBLOB extends StreamObject {
}
/**
- * @return the opaque binary data as a byte array, or null if not present.
+ * @return a copy of the opaque binary data, or null if not present.
*/
public byte[] getData() {
- if (this.data == null || this.data.content == null) {
- return null;
- }
- return ByteUtil.toByteArray(this.data.content);
+ return this.data == null ? null : this.data.copyContent();
+ }
+
+ /**
+ * @return the length of the binary data, 0 if not present
+ */
+ public int getDataLength() {
+ return this.data == null ? 0 : this.data.getContentLength();
}
@Override
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/PropertySetObject.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/PropertySetObject.java
index 89843a903c..85e883548a 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/PropertySetObject.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/PropertySetObject.java
@@ -19,7 +19,6 @@ package
org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj;
import java.io.IOException;
import
org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.space.ObjectSpaceObjectPropSet;
-import org.apache.tika.parser.microsoft.onenote.fsshttpb.util.ByteUtil;
/**
@@ -39,7 +38,6 @@ public class PropertySetObject {
ObjectGroupObjectData objectData) throws
IOException {
this.objectDeclaration = objectDeclaration;
this.objectSpaceObjectPropSet = new ObjectSpaceObjectPropSet();
- this.objectSpaceObjectPropSet.doDeserializeFromByteArray(
- ByteUtil.toByteArray(objectData.data.content), 0);
+
this.objectSpaceObjectPropSet.doDeserializeFromByteArray(objectData.data.getContent(),
0);
}
}
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/RevisionStoreObjectGroup.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/RevisionStoreObjectGroup.java
index fb16a8402a..4f1e34b617 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/RevisionStoreObjectGroup.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/RevisionStoreObjectGroup.java
@@ -23,7 +23,6 @@ import java.util.List;
import java.util.Map;
import
org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.basic.ExGuid;
-import org.apache.tika.parser.microsoft.onenote.fsshttpb.util.ByteUtil;
public class RevisionStoreObjectGroup {
public ExGuid objectGroupID;
@@ -124,7 +123,7 @@ public class RevisionStoreObjectGroup {
if (objectDeclaration.objectPartitionID.getDecodedValue() ==
1) {
EncryptionObject encrypObject = new EncryptionObject();
encrypObject.objectDeclaration = objectDeclaration;
- encrypObject.objectData =
ByteUtil.toByteArray(objectData.data.content);
+ encrypObject.objectData = objectData.data.getContent();
objectGroup.encryptionObjects.add(encrypObject);
}
}
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/basic/BinaryItem.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/basic/BinaryItem.java
index 2db0be6b42..b69b518eb4 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/basic/BinaryItem.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/basic/BinaryItem.java
@@ -18,22 +18,46 @@ package
org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.basic;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.tika.exception.TikaException;
+import org.apache.tika.parser.microsoft.onenote.fsshttpb.util.ByteUtil;
+/**
+ * A length-prefixed run of bytes.
+ *
+ * <p>The content is read in place, never copied on the way in: a deserialized
item is a range
+ * of the buffer it was parsed from, and an item built from an array keeps
that array. Neither
+ * may be modified while the item is in use. {@link #getContent()} swaps a
range for an array of
+ * its own on the first call; {@link #copyContent()} always returns a fresh
array.
+ *
+ * <p>Construction, {@link #setContent} and serialization keep {@link #length}
equal to the
+ * content size, which is 0 for content set to null.
+ */
public class BinaryItem extends BasicObject {
+ private static final byte[] EMPTY = new byte[0];
+
public Compact64bitInt length;
- public List<Byte> content;
+
+ /**
+ * The payload is {@code source[offset, offset + count)}. Deserializing
keeps a view of the
+ * parsed buffer, so items that are never read, or only measured, cost no
copy; object data
+ * BLOBs hold whole embedded files. {@code source} is null when the
content was explicitly
+ * set to null.
+ */
+ private byte[] source;
+ private int offset;
+ private int count;
/**
* Initializes a new instance of the BinaryItem class.
*/
public BinaryItem() {
this.length = new Compact64bitInt();
- this.content = new ArrayList<>();
+ this.source = EMPTY;
}
/**
@@ -42,10 +66,56 @@ public class BinaryItem extends BasicObject {
* @param content Specify the binary content.
*/
public BinaryItem(Collection<Byte> content) {
+ this(ByteUtil.toByteArray(content instanceof List<Byte> list ? list :
new ArrayList<>(content)));
+ }
+
+ /**
+ * Initializes a new instance of the BinaryItem class with the specified
content. The
+ * array is used as-is, not copied.
+ *
+ * @param content Specify the binary content, or null for none.
+ */
+ public BinaryItem(byte[] content) {
this.length = new Compact64bitInt();
- this.content = new ArrayList<>();
- this.content.addAll(content);
- this.length.setDecodedValue(this.content.size());
+ setContent(content);
+ }
+
+ /**
+ * @return the content as an array of exactly its length, or null if the
content was set
+ * to null. A deserialized item copies its range out of the parsed buffer
on the first
+ * call and returns that same array afterwards.
+ */
+ public byte[] getContent() {
+ if (source != null && (offset != 0 || count != source.length)) {
+ source = Arrays.copyOfRange(source, offset, offset + count);
+ offset = 0;
+ }
+ return source;
+ }
+
+ /**
+ * @return a new copy of the content, or null if the content was set to
null. Unlike
+ * {@link #getContent()}, the item does not keep the copy.
+ */
+ public byte[] copyContent() {
+ return source == null ? null : Arrays.copyOfRange(source, offset,
offset + count);
+ }
+
+ /**
+ * @return the content length in bytes, 0 if the content was set to null
+ */
+ public int getContentLength() {
+ return count;
+ }
+
+ /**
+ * Replaces the content. The array is used as-is, not copied.
+ */
+ public void setContent(byte[] content) {
+ this.source = content;
+ this.offset = 0;
+ this.count = content == null ? 0 : content.length;
+ this.length.setDecodedValue(this.count);
}
/**
@@ -55,12 +125,12 @@ public class BinaryItem extends BasicObject {
*/
@Override
public List<Byte> serializeToByteList() throws IOException {
- this.length.setDecodedValue(this.content.size());
-
- List<Byte> result = new ArrayList<>();
- result.addAll(this.length.serializeToByteList());
- result.addAll(this.content);
-
+ List<Byte> length =
this.length.setDecodedValue(count).serializeToByteList();
+ List<Byte> result = new ArrayList<>(length.size() + count);
+ result.addAll(length);
+ for (int i = offset; i < offset + count; i++) {
+ result.add(source[i]);
+ }
return result;
}
@@ -78,10 +148,17 @@ public class BinaryItem extends BasicObject {
this.length = BasicObject.parse(byteArray, index,
Compact64bitInt.class);
- this.content.clear();
- for (long i = 0; i < this.length.getDecodedValue(); i++) {
- this.content.add(byteArray[index.getAndIncrement()]);
+ long contentLength = this.length.getDecodedValue();
+ int contentStart = index.get();
+ if (contentLength < 0 || contentLength > byteArray.length -
contentStart) {
+ throw new TikaException("BinaryItem length " + contentLength + "
at offset " +
+ contentStart + " exceeds the " + (byteArray.length -
contentStart) +
+ " bytes remaining");
}
+ this.source = byteArray;
+ this.offset = contentStart;
+ this.count = (int) contentLength;
+ index.addAndGet((int) contentLength);
return index.get() - startIndex;
}
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/basic/HeaderCell.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/basic/HeaderCell.java
index 24a8e664b3..e2db132dc1 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/basic/HeaderCell.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/basic/HeaderCell.java
@@ -22,7 +22,6 @@ import
org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.ObjectGroupDa
import
org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.ObjectGroupObjectData;
import
org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.ObjectGroupObjectDeclare;
import
org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.space.ObjectSpaceObjectPropSet;
-import org.apache.tika.parser.microsoft.onenote.fsshttpb.util.ByteUtil;
public class HeaderCell {
public ObjectGroupObjectDeclare objectDeclaration;
@@ -49,8 +48,7 @@ public class HeaderCell {
ObjectGroupObjectData objectData =
objectElement.objectGroupData.objectGroupObjectDataList.get(i);
instance.objectData = new ObjectSpaceObjectPropSet();
- instance.objectData.doDeserializeFromByteArray(
- ByteUtil.toByteArray(objectData.data.content), 0);
+
instance.objectData.doDeserializeFromByteArray(objectData.data.getContent(), 0);
break;
}
}
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/util/BitReader.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/util/BitReader.java
index 73b3f4a61c..d36b34a050 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/util/BitReader.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/util/BitReader.java
@@ -17,14 +17,24 @@
package org.apache.tika.parser.microsoft.onenote.fsshttpb.util;
import java.io.IOException;
-import java.util.Arrays;
-import java.util.BitSet;
import java.util.UUID;
/**
* A class is used to extract values across byte boundaries with arbitrary bit
positions.
*/
public class BitReader {
+ /**
+ * Direct-mapped cache from raw GUID bytes to the derived UUID. A file
references the same
+ * few GUIDs from every ExGuid and cell ID, so most reads would otherwise
recompute the same
+ * MD5. Entries are immutable and the mapping is a pure function of the
key, so racing
+ * writers can only replace one correct entry with another.
+ */
+ private static final int GUID_CACHE_MASK = 1023;
+ private static final GuidCacheEntry[] GUID_CACHE = new
GuidCacheEntry[GUID_CACHE_MASK + 1];
+
+ private record GuidCacheEntry(long high, long low, UUID uuid) {
+ }
+
/**
* A start position which will be not changed in the process of reading.
* This value will be used for recording the start position and will be
used by the function reset.
@@ -56,18 +66,6 @@ public class BitReader {
this.byteArray = array;
}
- private static String toBinaryString(BitSet bs, int nbits) {
- final StringBuilder buffer = new StringBuilder(bs.size());
- for (int i = nbits - 1; i >= 0; --i) {
- if (i < bs.size()) {
- buffer.append(bs.get(i) ? "1" : "0");
- } else {
- buffer.append("0");
- }
- }
- return buffer.toString();
- }
-
public boolean getCurrent() {
return (byteArray[(int) offset >> 3] & (1 << ((int) offset & 7))) != 0;
}
@@ -137,7 +135,22 @@ public class BitReader {
* @return Return the GUID value.
*/
public UUID readGuid() throws IOException {
- return UUID.nameUUIDFromBytes(this.getBytes(128, 16));
+ byte[] raw = this.getBytes(128, 16);
+ long high = 0;
+ long low = 0;
+ for (int i = 0; i < 8; i++) {
+ high = (high << 8) | (raw[i] & 0xff);
+ low = (low << 8) | (raw[i + 8] & 0xff);
+ }
+ long mixed = (high ^ low) * 0x9E3779B97F4A7C15L;
+ int slot = (int) (mixed >>> 54) & GUID_CACHE_MASK;
+ GuidCacheEntry entry = GUID_CACHE[slot];
+ if (entry != null && entry.high == high && entry.low == low) {
+ return entry.uuid;
+ }
+ UUID uuid = UUID.nameUUIDFromBytes(raw);
+ GUID_CACHE[slot] = new GuidCacheEntry(high, low, uuid);
+ return uuid;
}
/**
@@ -172,25 +185,35 @@ public class BitReader {
* @return Returns the constructed byte array.
*/
private byte[] getBytes(int needReadlength, int size) throws IOException {
- BitSet retSet = new BitSet(size);
- int i = 0;
- while (i < needReadlength) {
- if (!this.moveNext()) {
- throw new IOException("Unexpected to meet the byte array
end.");
- }
- if (getCurrent()) {
- retSet.set(i);
- } else {
- retSet.clear(i);
- }
- i++;
+ // Bit i of the result is bit i after the current offset, least
significant bit first.
+ // The bounds check below guarantees every source byte read holds a
requested bit.
+ long start = this.offset + 1;
+ if (needReadlength > this.length - start) {
+ this.offset = this.length;
+ throw new IOException("Unexpected to meet the byte array end.");
}
byte[] result = new byte[size];
- Arrays.fill(result, (byte) 0);
- byte[] retSetBa = retSet.toByteArray();
- for (i = 0; i < retSetBa.length; ++i) {
- result[i] = retSetBa[i];
+ int shift = (int) (start & 7);
+ int source = (int) (start >> 3);
+ int fullBytes = needReadlength >> 3;
+ int remainingBits = needReadlength & 7;
+ if (shift == 0) {
+ System.arraycopy(byteArray, source, result, 0, fullBytes);
+ source += fullBytes;
+ } else {
+ for (int i = 0; i < fullBytes; i++, source++) {
+ result[i] = (byte) (((byteArray[source] & 0xFF) >>> shift) |
+ (byteArray[source + 1] << (8 - shift)));
+ }
+ }
+ if (remainingBits > 0) {
+ int bits = (byteArray[source] & 0xFF) >>> shift;
+ if (shift + remainingBits > 8) {
+ bits |= byteArray[source + 1] << (8 - shift);
+ }
+ result[fullBytes] = (byte) (bits & ((1 << remainingBits) - 1));
}
+ this.offset += needReadlength;
return result;
}
}
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStoreBlobTest.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStoreBlobTest.java
index b7d7508f8b..13368f8291 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStoreBlobTest.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStoreBlobTest.java
@@ -184,8 +184,7 @@ public class MSOneStoreBlobTest {
ObjectSpaceObjectPropSet propSet = new ObjectSpaceObjectPropSet();
propSet.body = propertySet;
ObjectGroupObjectData objectData = new ObjectGroupObjectData();
- objectData.data.content.addAll(ByteUtil.toListOfByte(new byte[]{0, 0,
0, (byte) 0x80,
- 0, 0, 0, 0}));
+ objectData.data.setContent(new byte[]{0, 0, 0, (byte) 0x80, 0, 0, 0,
0});
PropertySetObject propertySetObject = new PropertySetObject(null,
objectData);
propertySetObject.objectSpaceObjectPropSet = propSet;
parent.propertySet = propertySetObject;
@@ -210,14 +209,13 @@ public class MSOneStoreBlobTest {
blob.data = null;
assertNull(blob.getData());
blob.data = new BinaryItem();
- blob.data.content = null;
+ blob.data.setContent(null);
assertNull(blob.getData());
- blob.data = new BinaryItem();
- blob.data.content.add((byte) 1);
+ blob.data = new BinaryItem(new byte[]{1});
assertFalse(blob.serializeToByteList().isEmpty());
ObjectDataBLOB validBlob = new ObjectDataBLOB();
- validBlob.data.content.add((byte) 1);
+ validBlob.data.setContent(new byte[]{1});
byte[] itemBytes =
ByteUtil.toByteArray(validBlob.data.serializeToByteList());
AtomicInteger index = new AtomicInteger(0);
java.lang.reflect.Method deserialize =
ObjectDataBLOB.class.getDeclaredMethod(
@@ -282,7 +280,7 @@ public class MSOneStoreBlobTest {
ObjectGroupObjectDeclare declaration = new ObjectGroupObjectDeclare();
declaration.objectPartitionID.setDecodedValue(1);
ObjectGroupObjectData objectData = new ObjectGroupObjectData();
- objectData.data.content.add((byte) 42);
+ objectData.data.setContent(new byte[]{42});
encryptedData.objectGroupDeclarations.objectDeclarationList.add(declaration);
encryptedData.objectGroupData.objectGroupObjectDataList.add(objectData);
@@ -337,9 +335,7 @@ public class MSOneStoreBlobTest {
private DataElement roundTripBlobElement() throws Exception {
ObjectDataBLOBDataElementData blobData = new
ObjectDataBLOBDataElementData();
- for (byte b : BLOB_BYTES) {
- blobData.objectDataBLOB.data.content.add(b);
- }
+ blobData.objectDataBLOB.data.setContent(BLOB_BYTES.clone());
DataElement blobElement =
new DataElement(DataElementType.ObjectDataBLOBDataElementData,
blobData);
byte[] serialized =
ByteUtil.toByteArray(blobElement.serializeToByteList());
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStorePackageTest.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStorePackageTest.java
index b8095abd45..bff0021a4e 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStorePackageTest.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStorePackageTest.java
@@ -76,7 +76,6 @@ import
org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.basic.HeaderC
import
org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.basic.PropertyID;
import
org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.basic.PropertyType;
import
org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.space.ObjectSpaceObjectPropSet;
-import org.apache.tika.parser.microsoft.onenote.fsshttpb.util.ByteUtil;
import org.apache.tika.sax.ToTextContentHandler;
import org.apache.tika.sax.ToXMLContentHandler;
import org.apache.tika.sax.XHTMLContentHandler;
@@ -2678,7 +2677,7 @@ public class MSOneStorePackageTest {
private static FileDataObject fileData(String value) {
ObjectDataBLOB blob = new ObjectDataBLOB();
-
blob.data.content.addAll(ByteUtil.toListOfByte(value.getBytes(StandardCharsets.UTF_8)));
+ blob.data.setContent(value.getBytes(StandardCharsets.UTF_8));
ObjectDataBLOBDataElementData blobData = new
ObjectDataBLOBDataElementData();
blobData.objectDataBLOB = blob;
DataElement element = new DataElement();
@@ -2745,8 +2744,7 @@ public class MSOneStorePackageTest {
private static ObjectGroupObjectData emptyObjectData() {
ObjectGroupObjectData data = new ObjectGroupObjectData();
- data.data.content.addAll(ByteUtil.toListOfByte(new byte[]{0, 0, 0,
(byte) 0x80,
- 0, 0, 0, 0}));
+ data.data.setContent(new byte[]{0, 0, 0, (byte) 0x80, 0, 0, 0, 0});
return data;
}
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/basic/BinaryItemTest.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/basic/BinaryItemTest.java
new file mode 100644
index 0000000000..d8cde111c8
--- /dev/null
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/basic/BinaryItemTest.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.basic;
+
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotSame;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+import java.util.Arrays;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.junit.jupiter.api.Test;
+
+import org.apache.tika.exception.TikaException;
+import org.apache.tika.parser.microsoft.onenote.fsshttpb.util.ByteUtil;
+
+public class BinaryItemTest {
+
+ /**
+ * Deserializing keeps only a view of the parsed buffer, so the accessors
must still
+ * return exactly the item's bytes, surrounded by unrelated data, and hand
out arrays
+ * that cannot reach back into that buffer.
+ */
+ @Test
+ public void testDeserializedContentIsExactAndDetached() throws Exception {
+ byte[] payload = {10, 20, 30, 40, 50};
+ byte[] item = serialize(new BinaryItem(payload.clone()));
+ byte[] buffer = new byte[item.length + 6];
+ Arrays.fill(buffer, (byte) 99);
+ System.arraycopy(item, 0, buffer, 3, item.length);
+
+ AtomicInteger index = new AtomicInteger(3);
+ BinaryItem parsed = BasicObject.parse(buffer, index, BinaryItem.class);
+ assertEquals(3 + item.length, index.get());
+ assertEquals(payload.length, parsed.getContentLength());
+
+ byte[] copy = parsed.copyContent();
+ assertArrayEquals(payload, copy);
+ copy[0] = 0;
+ assertArrayEquals(payload, parsed.copyContent());
+
+ byte[] content = parsed.getContent();
+ assertArrayEquals(payload, content);
+ assertSame(content, parsed.getContent());
+ assertNotSame(buffer, content);
+ content[1] = 0;
+ assertEquals(20, buffer[3 + item.length - payload.length + 1]);
+ }
+
+ /**
+ * Replacing the content of a deserialized item must drop the view of the
parsed buffer,
+ * keep {@code length} in step, and serialize the new bytes; a
deserialized item must also
+ * reserialize to exactly the bytes it was parsed from.
+ */
+ @Test
+ public void testSetContentAfterDeserializeAndRoundTrip() throws Exception {
+ byte[] item = serialize(new BinaryItem(new byte[]{1, 2, 3, 4}));
+ byte[] buffer = new byte[item.length + 2];
+ System.arraycopy(item, 0, buffer, 1, item.length);
+ BinaryItem parsed = BasicObject.parse(buffer, new AtomicInteger(1),
BinaryItem.class);
+ assertArrayEquals(item, serialize(parsed));
+
+ byte[] replacement = {7, 8};
+ parsed.setContent(replacement);
+ assertSame(replacement, parsed.getContent());
+ assertEquals(2, parsed.getContentLength());
+ assertEquals(2, parsed.length.getDecodedValue());
+
+ byte[] reserialized = serialize(parsed);
+ BinaryItem reparsed =
+ BasicObject.parse(reserialized, new AtomicInteger(0),
BinaryItem.class);
+ assertArrayEquals(replacement, reparsed.getContent());
+ }
+
+ @Test
+ public void testLengthPastBufferEndFails() throws Exception {
+ byte[] item = serialize(new BinaryItem(new byte[]{1, 2, 3, 4}));
+ byte[] truncated = Arrays.copyOf(item, item.length - 1);
+ assertThrows(TikaException.class,
+ () -> BasicObject.parse(truncated, new AtomicInteger(0),
BinaryItem.class));
+ }
+
+ /**
+ * A 64-bit compact length of 2^63 or more decodes as a negative long; it
must fail like any
+ * other impossible length rather than produce an empty item.
+ */
+ @Test
+ public void testNegativeLengthFails() {
+ byte[] item = {(byte) 0x80, 0, 0, 0, 0, 0, 0, 0, (byte) 0x80};
+ assertThrows(TikaException.class,
+ () -> BasicObject.parse(item, new AtomicInteger(0),
BinaryItem.class));
+ }
+
+ @Test
+ public void testNullContent() throws Exception {
+ BinaryItem item = new BinaryItem(new byte[]{1});
+ item.setContent(null);
+ assertNull(item.getContent());
+ assertNull(item.copyContent());
+ assertEquals(0, item.getContentLength());
+ assertEquals(0, item.length.getDecodedValue());
+
+ BinaryItem constructed = new BinaryItem((byte[]) null);
+ assertNull(constructed.getContent());
+ assertEquals(0, constructed.length.getDecodedValue());
+ assertArrayEquals(serialize(new BinaryItem()), serialize(constructed));
+ }
+
+ private static byte[] serialize(BinaryItem item) throws Exception {
+ return ByteUtil.toByteArray(item.serializeToByteList());
+ }
+}
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/util/BitReaderTest.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/util/BitReaderTest.java
new file mode 100644
index 0000000000..0c1b3ee809
--- /dev/null
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/util/BitReaderTest.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.microsoft.onenote.fsshttpb.util;
+
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+import java.io.IOException;
+import java.util.Random;
+import java.util.UUID;
+
+import org.junit.jupiter.api.Test;
+
+public class BitReaderTest {
+
+ /**
+ * Every read must match a reference reader stepping
moveNext()/getCurrent() one bit at a
+ * time over the same bytes.
+ */
+ @Test
+ public void testReadsMatchBitByBitReference() throws IOException {
+ Random random = new Random(20261104L);
+ for (int trial = 0; trial < 200; trial++) {
+ byte[] data = new byte[1 + random.nextInt(64)];
+ random.nextBytes(data);
+ int startByte = random.nextInt(data.length);
+ BitReader reader = new BitReader(data, startByte);
+ BitReader reference = new BitReader(data, startByte);
+ long bitsLeft = (long) (data.length - startByte) * 8;
+ while (bitsLeft > 0) {
+ int kind = random.nextInt(3);
+ if (kind == 0 && bitsLeft >= 128) {
+
assertEquals(UUID.nameUUIDFromBytes(referenceBytes(reference, 128, 16)),
+ reader.readGuid());
+ bitsLeft -= 128;
+ } else if (kind == 1 && bitsLeft >= 8) {
+ int bytes = 1 + random.nextInt((int) Math.min(bitsLeft /
8, 20));
+ assertArrayEquals(referenceBytes(reference, bytes * 8,
bytes),
+ reader.readBytes(bytes));
+ bitsLeft -= bytes * 8L;
+ } else {
+ int bits = 1 + random.nextInt((int) Math.min(bitsLeft,
64));
+ assertEquals(LittleEndianBitConverter.toUInt64(
+ referenceBytes(reference, bits, 8), 0),
+ reader.readUInt64(bits), "trial " + trial + ",
width " + bits);
+ bitsLeft -= bits;
+ }
+ }
+ assertThrows(IOException.class, () -> reader.readUInt32(1));
+ }
+ }
+
+ /**
+ * The GUID cache slot depends on high ^ low, so XORing both 8-byte halves
of a GUID with the
+ * same mask gives a different GUID in the same slot. Reads alternating
between the two must
+ * each return their own MD5-derived UUID; a repeat read of the resident
one is a cache hit.
+ */
+ @Test
+ public void testGuidCacheCollision() throws IOException {
+ byte[] first = new byte[16];
+ new Random(20261106L).nextBytes(first);
+ byte[] second = first.clone();
+ second[0] ^= (byte) 0x5A;
+ second[8] ^= (byte) 0x5A;
+ UUID firstUuid = UUID.nameUUIDFromBytes(first);
+ UUID secondUuid = UUID.nameUUIDFromBytes(second);
+
+ byte[] data = new byte[16 * 5];
+ for (int i = 0; i < 5; i++) {
+ System.arraycopy(i % 2 == 0 ? first : second, 0, data, i * 16, 16);
+ }
+ BitReader reader = new BitReader(data, 0);
+ assertEquals(firstUuid, reader.readGuid());
+ assertEquals(secondUuid, reader.readGuid());
+ assertEquals(firstUuid, reader.readGuid());
+ assertEquals(secondUuid, reader.readGuid());
+
+ BitReader repeat = new BitReader(data, 64);
+ UUID resident = repeat.readGuid();
+ assertEquals(firstUuid, resident);
+ assertSame(resident, new BitReader(data, 64).readGuid());
+ }
+
+ @Test
+ public void testReadPastEndThrows() {
+ BitReader reader = new BitReader(new byte[]{1, 2, 3}, 1);
+ assertThrows(IOException.class, () -> reader.readUInt32(17));
+ }
+
+ private static byte[] referenceBytes(BitReader reference, int bits, int
size) {
+ byte[] result = new byte[size];
+ for (int i = 0; i < bits; i++) {
+ if (!reference.moveNext()) {
+ throw new IllegalStateException("reference reader ran out of
bits");
+ }
+ if (reference.getCurrent()) {
+ result[i >> 3] |= (byte) (1 << (i & 7));
+ }
+ }
+ return result;
+ }
+}
