This is an automated email from the ASF dual-hosted git repository.
jadams pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-daffodil.git
The following commit(s) were added to refs/heads/master by this push:
new 1b84e9e Fix issue unparsing large blobs
1b84e9e is described below
commit 1b84e9e7d6aff9d1339dd39c86719bb0e5e25c4b
Author: Josh Adams <[email protected]>
AuthorDate: Mon Oct 26 21:19:47 2020 -0400
Fix issue unparsing large blobs
With large blob files it is possible that the blob size will be greater
than Int.MaxValue, which was resulting in an overflow and causing the
check to ensure that we have written out all bytes to fail. Simply
moving the byte length int conversion to the section of code for
non-blob files prevents this issue from occurring.
Also cleaned up some duplicated code and removed old commented code.
DAFFODIL-2425
---
.../io/DirectOrBufferedDataOutputStream.scala | 151 ++++++++-------------
1 file changed, 57 insertions(+), 94 deletions(-)
diff --git
a/daffodil-io/src/main/scala/org/apache/daffodil/io/DirectOrBufferedDataOutputStream.scala
b/daffodil-io/src/main/scala/org/apache/daffodil/io/DirectOrBufferedDataOutputStream.scala
index 3c28a0c..256ecb5 100644
---
a/daffodil-io/src/main/scala/org/apache/daffodil/io/DirectOrBufferedDataOutputStream.scala
+++
b/daffodil-io/src/main/scala/org/apache/daffodil/io/DirectOrBufferedDataOutputStream.scala
@@ -845,107 +845,70 @@ class DirectOrBufferedDataOutputStream private[io] (
}
/**
- * Convenience methods that temporarily set and (reliably) restore the
bitLimit.
- * The argument gives the limit length. Note this is a length, not a bit
position.
- *
- * This is added to the current bit position to get the limiting bit position
- * which is then set as the bitLimit when
- * the body is evaluated. On return the bit limit is restored to its
- * prior value.
- * <p>
- * The return value is false if the new bit limit is beyond the existing bit
limit range.
- * Otherwise the return value is true.
- * <p>
- * The prior value is restored even if an Error/Exception is thrown. (ie.,
via a try-finally)
- * <p>
- * These are intended for use implementing specified-length types (simple or
complex).
- * <p>
- * Note that length limits in lengthUnits Characters are not implemented
- * this way. See fillCharBuffer(cb) method.
+ * Delivers the bits of this DOS's buffer into the given directDOS's output
stream.
+ * Deals with the possibility that the directDOS ends with a fragment byte,
or
+ * the bufDOS does, or both. Handles file based buffers as well.
*/
- // private def withBitLengthLimit(lengthLimitInBits: Long)(body: => Unit):
Boolean = macro IOMacros.withBitLengthLimitMacroForOutput
-
- protected def deliverContent(directDOS: DirectOrBufferedDataOutputStream,
finfo: FormatInfo) = {
+ protected def deliverContent(directDOS: DirectOrBufferedDataOutputStream,
finfo: FormatInfo): Unit = {
val bufferNBits = this.relBitPos0b // don't have to subtract a starting
offset. It's always zero in buffered case.
val bufOS = this.bufferingJOS
-
- if (directDOS.isEndOnByteBoundary && this.isEndOnByteBoundary) {
- // no fragment bytes anywhere - just take the bytes
-
- val nBytes = (bufferNBits / 8).toInt
- val nBytesPut = {
- if (bufOS.isFile) {
- bufOS.close
- val nBitsPut = try {
- directDOS.putFile(bufOS.getFile.toPath, bufferNBits.toLong,
chunkSizeInBytes, finfo)
- } finally {
- // Make sure we delete the file after it was put in the directDOS
or
- // if we encountered an error
- if (bufOS.isTempFile)
- bufOS.getFile.delete()
- }
- nBitsPut / 8
- } else
- directDOS.putBytes(bufOS.getBuf, 0, nBytes, finfo)
- }
- Assert.invariant(nBytesPut == nBytes)
-
- } else {
- // fragment byte on directDOS, fragment byte on bufDOS, or both.
-
- val nFragBits = this.fragmentLastByteLimit
- val byteCount = bufferNBits / 8
- val wholeBytesWritten = {
- if (bufOS.isFile) {
- bufOS.close
- val nBitsPut = try {
- directDOS.putFile(bufOS.getFile.toPath, bufferNBits.toLong,
chunkSizeInBytes, finfo)
- } finally {
- // Make sure we delete the file after it was put in the directDOS
or
- // if we encountered an error
- if (bufOS.isTempFile)
- bufOS.getFile.delete()
- }
- nBitsPut / 8
- } else
- directDOS.putBytes(bufOS.getBuf, 0, byteCount.toInt, finfo)
+ val nBytes = (bufferNBits / 8)
+ val nFragBits = this.fragmentLastByteLimit
+
+ val wholeBytesWritten = {
+ if (bufOS.isFile) {
+ bufOS.close
+ val nBitsPut = try {
+ directDOS.putFile(bufOS.getFile.toPath, bufferNBits.toLong,
chunkSizeInBytes, finfo)
+ } finally {
+ // Make sure we delete the file after it was put in the directDOS or
+ // if we encountered an error
+ if (bufOS.isTempFile)
+ bufOS.getFile.delete()
+ }
+ nBitsPut / 8
+ } else {
+ Assert.invariant(nBytes <= Int.MaxValue)
+ directDOS.putBytes(bufOS.getBuf, 0, nBytes.toInt, finfo)
}
+ }
- Assert.invariant(byteCount == wholeBytesWritten)
- if (nFragBits > 0) {
- if (directDOS.isEndOnByteBoundary) {
- // We cannot use putLong like below because it's possible that
- // the fragment byte has a different bitOrder than the finfo
- // passed in, since that came from a suspension. However, if
- // the directDOS ended on a byte boundary, that means that its
- // new fragment byte should be exactly the same as the buffered
- // DOS fragment byte. So in this case, just copy the frag byte
- // information from buffered to direct.
- directDOS.setFragmentLastByte(this.fragmentLastByte,
this.fragmentLastByteLimit)
- } else {
- // If the direct DOS wasn't byte aligned, then we need logic to
- // write the buffered DOS fragment after the direct DOS
- // fragment. Fortunately, putLong has all of this logic. Like
- // above, the call to putLong potentially uses the wrong finfo
- // since it may have come from a suspension. However, all that
- // putLong really uses from the finfo is the bitOrder. And
- // because the directDOS isn't byte aligned we know it must
- // have the same bitOrder as the buffered DOS. So even though
- // it could be the wrong format info, it's safe to use in this
- // case.
- val origfrag = this.fragmentLastByte
- val fragNum =
- if (finfo.bitOrder eq BitOrder.MostSignificantBitFirst)
- origfrag >> (8 - nFragBits)
- else
- origfrag
-
- Assert.invariant(directDOS.putLongUnchecked(
- fragNum, nFragBits, finfo,
- ignoreByteOrder = this.bufferingJOS.isFile))
- }
+ Assert.invariant(nBytes == wholeBytesWritten)
+
+ if (nFragBits > 0) {
+ if (directDOS.isEndOnByteBoundary) {
+ // We cannot use putLong like below because it's possible that
+ // the fragment byte has a different bitOrder than the finfo
+ // passed in, since that came from a suspension. However, if
+ // the directDOS ended on a byte boundary, that means that its
+ // new fragment byte should be exactly the same as the buffered
+ // DOS fragment byte. So in this case, just copy the frag byte
+ // information from buffered to direct.
+ directDOS.setFragmentLastByte(this.fragmentLastByte,
this.fragmentLastByteLimit)
+ } else {
+ // If the direct DOS wasn't byte aligned, then we need logic to
+ // write the buffered DOS fragment after the direct DOS
+ // fragment. Fortunately, putLong has all of this logic. Like
+ // above, the call to putLong potentially uses the wrong finfo
+ // since it may have come from a suspension. However, all that
+ // putLong really uses from the finfo is the bitOrder. And
+ // because the directDOS isn't byte aligned we know it must
+ // have the same bitOrder as the buffered DOS. So even though
+ // it could be the wrong format info, it's safe to use in this
+ // case.
+ val origfrag = this.fragmentLastByte
+ val fragNum =
+ if (finfo.bitOrder eq BitOrder.MostSignificantBitFirst)
+ origfrag >> (8 - nFragBits)
+ else
+ origfrag
+
+ Assert.invariant(directDOS.putLongUnchecked(
+ fragNum, nFragBits, finfo,
+ ignoreByteOrder = this.bufferingJOS.isFile))
}
}
+
bufOS.close()
}
