This is an automated email from the ASF dual-hosted git repository.
mmcgann pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/daffodil.git
The following commit(s) were added to refs/heads/main by this push:
new 6590127af Update parse error message
6590127af is described below
commit 6590127afe09aaa5a8bcffa4296fa6ec852d0ebe
Author: Mike McGann <[email protected]>
AuthorDate: Wed Mar 15 10:41:41 2023 -0400
Update parse error message
When there are not enough bits for parsing, an error message will be
displayed showing the number of bits available. Sometimes this value uses
the number of bits remaining in the limit and doesn't consider that this
could be greater than the number of bits actually available in the data
stream. This change selects the lesser of the two values when that
information is known.
The TDML runner has also been updated to only place a limit on the parser
when the length of the document under test doesn't end on a byte boundary.
DAFFODIL-2645
---
.../main/scala/org/apache/daffodil/cli/Main.scala | 4 +-
.../daffodil/io/DataInputStreamImplMixin.scala | 11 ----
.../daffodil/io/DataOutputStreamImplMixin.scala | 5 --
.../org/apache/daffodil/io/DataStreamCommon.scala | 8 ---
.../scala/org/apache/daffodil/io/InputSource.scala | 10 +--
.../daffodil/io/InputSourceDataInputStream.scala | 11 +++-
.../runtime1/processors/PackedBinaryTraits.scala | 4 +-
.../processors/parsers/BinaryBooleanParsers.scala | 2 +-
.../processors/parsers/BinaryNumberParsers.scala | 8 +--
.../processors/parsers/BlobLengthParsers.scala | 13 +---
.../parsers/HexBinaryLengthParsers.scala | 2 +-
.../processors/parsers/LayeredSequenceParser.scala | 2 -
.../runtime1/processors/parsers/Parser.scala | 73 ++++++++++++++++++----
.../processors/parsers/PrimitivesDateTime1.scala | 2 +-
.../parsers/SpecifiedLengthParsers.scala | 4 +-
.../processor/tdml/DaffodilTDMLDFDLProcessor.scala | 17 +++--
.../daffodil/section05/simple_types/Blobs.tdml | 34 +++++++++-
.../section12/lengthKind/ExplicitTests.tdml | 53 +++++++++++++++-
.../section05/simple_types/TestBlobs.scala | 4 ++
.../lengthKind/TestLengthKindExplicit.scala | 8 +++
20 files changed, 197 insertions(+), 78 deletions(-)
diff --git a/daffodil-cli/src/main/scala/org/apache/daffodil/cli/Main.scala
b/daffodil-cli/src/main/scala/org/apache/daffodil/cli/Main.scala
index ee5139f2b..26141b797 100644
--- a/daffodil-cli/src/main/scala/org/apache/daffodil/cli/Main.scala
+++ b/daffodil-cli/src/main/scala/org/apache/daffodil/cli/Main.scala
@@ -1138,7 +1138,7 @@ class Main(
if (loc.bitLimit0b.isDefined) {
(loc.bitLimit0b.get - loc.bitPos0b).toString
} else {
- "at least " + (inStream.inputSource.bytesAvailable *
8)
+ "at least " +
(inStream.inputSource.knownBytesAvailable * 8)
}
Logger.log.error(
s"Left over data after consuming 0 bits while
streaming. Stopped after consuming ${loc.bitPos0b} bit(s) with ${remainingBits}
bit(s) remaining.",
@@ -1176,7 +1176,7 @@ class Main(
dumpString
} else ""
val curBytePosition1b = inStream.inputSource.position + 1
- val bytesAvailable = inStream.inputSource.bytesAvailable
+ val bytesAvailable =
inStream.inputSource.knownBytesAvailable
val bytesLimit = math.min(8, bytesAvailable).toInt
val destArray = new Array[Byte](bytesLimit)
val destArrayFilled = inStream.inputSource.get(destArray,
0, bytesLimit)
diff --git
a/daffodil-io/src/main/scala/org/apache/daffodil/io/DataInputStreamImplMixin.scala
b/daffodil-io/src/main/scala/org/apache/daffodil/io/DataInputStreamImplMixin.scala
index c4dcd8254..045eef054 100644
---
a/daffodil-io/src/main/scala/org/apache/daffodil/io/DataInputStreamImplMixin.scala
+++
b/daffodil-io/src/main/scala/org/apache/daffodil/io/DataInputStreamImplMixin.scala
@@ -18,7 +18,6 @@
package org.apache.daffodil.io
import org.apache.daffodil.lib.exceptions.Assert
-import org.apache.daffodil.lib.util.MaybeULong
trait DataInputStreamImplMixin
extends DataInputStream
@@ -45,14 +44,4 @@ trait DataInputStreamImplMixin
skip(deltaBits, finfo)
}
- final override def remainingBits = {
- if (this.bitLimit0b.isEmpty) MaybeULong.Nope
- else {
- val lim = bitLimit0b.get
- Assert.invariant(lim >= 0)
- val nBits = lim - bitPos0b
- MaybeULong(nBits)
- }
- }
-
}
diff --git
a/daffodil-io/src/main/scala/org/apache/daffodil/io/DataOutputStreamImplMixin.scala
b/daffodil-io/src/main/scala/org/apache/daffodil/io/DataOutputStreamImplMixin.scala
index 6ead5579b..4aeb4a792 100644
---
a/daffodil-io/src/main/scala/org/apache/daffodil/io/DataOutputStreamImplMixin.scala
+++
b/daffodil-io/src/main/scala/org/apache/daffodil/io/DataOutputStreamImplMixin.scala
@@ -313,11 +313,6 @@ trait DataOutputStreamImplMixin
*/
var bitStartOffset0b: Int = 0
- final override def remainingBits: MaybeULong = {
- if (maybeRelBitLimit0b.isEmpty) MaybeULong.Nope
- else MaybeULong(maybeRelBitLimit0b.get - relBitPos0b.toLong)
- }
-
var debugOutputStream: Maybe[ByteArrayOutputStream] = Nope
/**
diff --git
a/daffodil-io/src/main/scala/org/apache/daffodil/io/DataStreamCommon.scala
b/daffodil-io/src/main/scala/org/apache/daffodil/io/DataStreamCommon.scala
index f4056f86c..474aa247c 100644
--- a/daffodil-io/src/main/scala/org/apache/daffodil/io/DataStreamCommon.scala
+++ b/daffodil-io/src/main/scala/org/apache/daffodil/io/DataStreamCommon.scala
@@ -19,8 +19,6 @@ package org.apache.daffodil.io
import java.nio.ByteBuffer
-import org.apache.daffodil.lib.util.MaybeULong
-
/**
* This is an interface trait, and it defines methods shared by
* both DataInputStream and DataOutputStream.
@@ -30,12 +28,6 @@ import org.apache.daffodil.lib.util.MaybeULong
*/
trait DataStreamCommon {
- /**
- * Returns number of bits remaining (if a limit is defined). Nope if not
defined.
- */
-
- def remainingBits: MaybeULong
-
/*
* Methods for moving through data.
*/
diff --git
a/daffodil-io/src/main/scala/org/apache/daffodil/io/InputSource.scala
b/daffodil-io/src/main/scala/org/apache/daffodil/io/InputSource.scala
index e3da0cb78..141049ecd 100644
--- a/daffodil-io/src/main/scala/org/apache/daffodil/io/InputSource.scala
+++ b/daffodil-io/src/main/scala/org/apache/daffodil/io/InputSource.scala
@@ -116,9 +116,9 @@ abstract class InputSource {
*
* This should not be used to determine the length of the data, as more bytes
* may become available in the future. This should really only be used for
- * debug purposes.
+ * debug or diagnostic purposes.
*/
- def bytesAvailable(): Long
+ def knownBytesAvailable(): Long
/**
* Return a single byte at the current byte position with a value in the
@@ -396,7 +396,7 @@ class BucketingInputSource(
* Calculate how many bytes are currently buffered starting from the current
* position
*/
- def bytesAvailable(): Long = {
+ def knownBytesAvailable(): Long = {
var available = 0L
val (curBucketIndex, curByteIndex) =
bytePositionToIndicies(curBytePosition0b)
@@ -528,7 +528,7 @@ class BucketingInputSource(
* buckets arraybuffer does not grow too big. This will move all existing
* buckets to the front of the buckets ArrayBuffer and update offsets
* accordingly. This way, there are not a bunch of null empty buckets at the
- * front of the buckets array taking up space.
+ * front of the buckets array taking up space.
*/
def compact(): Unit = {
releaseBuckets()
@@ -559,7 +559,7 @@ class ByteBufferInputSource(byteBuffer: ByteBuffer) extends
InputSource {
bb.remaining >= nBytes
}
- def bytesAvailable(): Long = {
+ def knownBytesAvailable(): Long = {
bb.remaining
}
diff --git
a/daffodil-io/src/main/scala/org/apache/daffodil/io/InputSourceDataInputStream.scala
b/daffodil-io/src/main/scala/org/apache/daffodil/io/InputSourceDataInputStream.scala
index 024e5cd5c..e745d348f 100644
---
a/daffodil-io/src/main/scala/org/apache/daffodil/io/InputSourceDataInputStream.scala
+++
b/daffodil-io/src/main/scala/org/apache/daffodil/io/InputSourceDataInputStream.scala
@@ -128,6 +128,15 @@ final class InputSourceDataInputStream private (val
inputSource: InputSource)
*/
def hasReachedEndOfData: Boolean = inputSource.hasReachedEndOfData
+ /**
+ * Return the number of currently available bytes.
+ *
+ * This should not be used to determine the length of the data, as more bytes
+ * may become available in the future. This should really only be used for
+ * debug or diagnostic purposes.
+ */
+ def knownBytesAvailable: Long = inputSource.knownBytesAvailable
+
def setBitPos0b(newBitPos0b: Long): Unit = {
// threadCheck()
Assert.invariant(newBitPos0b >= 0)
@@ -789,7 +798,7 @@ final class InputSourceDataInputStream private (val
inputSource: InputSource)
// need to call areBytesAvailable first to ensure at least length bytes are
// buffered if they exist
val available = inputSource.areBytesAvailable(nBytesRequested)
- val bytesToRead = if (available) nBytesRequested else
inputSource.bytesAvailable.toInt
+ val bytesToRead = if (available) nBytesRequested else
inputSource.knownBytesAvailable.toInt
val array = new Array[Byte](bytesToRead)
inputSource.get(array, 0, bytesToRead)
diff --git
a/daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/processors/PackedBinaryTraits.scala
b/daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/processors/PackedBinaryTraits.scala
index 09e8a2c1e..608ec58e9 100644
---
a/daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/processors/PackedBinaryTraits.scala
+++
b/daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/processors/PackedBinaryTraits.scala
@@ -55,7 +55,7 @@ abstract class PackedBinaryDecimalBaseParser(
val dis = start.dataInputStream
if (!dis.isDefinedForLength(nBits)) {
- PENotEnoughBits(start, nBits, dis.remainingBits)
+ PENotEnoughBits(start, nBits, dis)
return
}
@@ -83,7 +83,7 @@ abstract class PackedBinaryIntegerBaseParser(
val dis = start.dataInputStream
if (!dis.isDefinedForLength(nBits)) {
- PENotEnoughBits(start, nBits, dis.remainingBits)
+ PENotEnoughBits(start, nBits, dis)
return
}
diff --git
a/daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/BinaryBooleanParsers.scala
b/daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/BinaryBooleanParsers.scala
index c69f45d21..c92f8d339 100644
---
a/daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/BinaryBooleanParsers.scala
+++
b/daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/BinaryBooleanParsers.scala
@@ -62,7 +62,7 @@ abstract class BinaryBooleanParserBase(
val dis = start.dataInputStream
if (!dis.isDefinedForLength(nBits)) {
- PENotEnoughBits(start, nBits, dis.remainingBits)
+ PENotEnoughBits(start, nBits, dis)
return
}
diff --git
a/daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/BinaryNumberParsers.scala
b/daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/BinaryNumberParsers.scala
index 60da09183..c546cbe72 100644
---
a/daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/BinaryNumberParsers.scala
+++
b/daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/BinaryNumberParsers.scala
@@ -36,7 +36,7 @@ class BinaryFloatParser(override val context:
ElementRuntimeData) extends PrimPa
val dis = start.dataInputStream
if (!dis.isDefinedForLength(32)) {
- PENotEnoughBits(start, 32, dis.remainingBits)
+ PENotEnoughBits(start, 32, dis)
return
}
@@ -52,7 +52,7 @@ class BinaryDoubleParser(override val context:
ElementRuntimeData) extends PrimP
val dis = start.dataInputStream
if (!dis.isDefinedForLength(64)) {
- PENotEnoughBits(start, 64, dis.remainingBits)
+ PENotEnoughBits(start, 64, dis)
return
}
@@ -109,7 +109,7 @@ abstract class BinaryDecimalParserBase(
val nBits = getBitLength(start)
val dis = start.dataInputStream
if (!dis.isDefinedForLength(nBits)) {
- PENotEnoughBits(start, nBits, dis.remainingBits)
+ PENotEnoughBits(start, nBits, dis)
return
}
@@ -179,7 +179,7 @@ abstract class BinaryIntegerBaseParser(
}
val dis = start.dataInputStream
if (!dis.isDefinedForLength(nBits)) {
- PENotEnoughBits(start, nBits, dis.remainingBits)
+ PENotEnoughBits(start, nBits, dis)
return
}
diff --git
a/daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/BlobLengthParsers.scala
b/daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/BlobLengthParsers.scala
index 4fb5d367d..32c106606 100644
---
a/daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/BlobLengthParsers.scala
+++
b/daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/BlobLengthParsers.scala
@@ -20,7 +20,6 @@ package org.apache.daffodil.runtime1.processors.parsers
import java.nio.file.Files
import java.nio.file.StandardOpenOption
-import org.apache.daffodil.lib.util.MaybeULong
import org.apache.daffodil.runtime1.processors.ElementRuntimeData
import org.apache.daffodil.runtime1.processors.LengthInBitsEv
@@ -37,10 +36,11 @@ trait ByteChunkWriter { self: PrimParser =>
writeChunk: (Array[Byte], Int) => Unit,
): Unit = {
val dis = start.dataInputStream
+ val startLoc = start.currentLocation
+ val startSfl = start.schemaFileLocation
var remainingBitsToGet = nBits
val array = new Array[Byte](start.tunable.blobChunkSizeInBytes)
-
// Tunable restrictions ensure blobChunkSizeInBits still fits in an int.
// All the places where toInt is called below can be no larger than this
// value, and so have no issues with potential overflow.
@@ -54,14 +54,7 @@ trait ByteChunkWriter { self: PrimParser =>
writeChunk(array, bytesToPut.toInt)
remainingBitsToGet -= bitsToGet
} else {
- val remainingBits =
- if (dis.remainingBits.isDefined) {
- val totalBitsRead = nBits - remainingBitsToGet
- MaybeULong(dis.remainingBits.get + totalBitsRead)
- } else {
- MaybeULong.Nope
- }
- self.PENotEnoughBits(start, nBits, remainingBits)
+ PENotEnoughBits(start, startSfl, startLoc, nBits,
start.dataInputStream)
remainingBitsToGet = 0 // break out of the loop
}
}
diff --git
a/daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/HexBinaryLengthParsers.scala
b/daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/HexBinaryLengthParsers.scala
index ee6b5d61a..1e8ddf163 100644
---
a/daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/HexBinaryLengthParsers.scala
+++
b/daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/HexBinaryLengthParsers.scala
@@ -52,7 +52,7 @@ sealed abstract class HexBinaryLengthParser(override val
context: ElementRuntime
// create and fill the byte array
val dis = start.dataInputStream
if (!dis.isDefinedForLength(nBits)) {
- PENotEnoughBits(start, nBits, dis.remainingBits)
+ PENotEnoughBits(start, nBits, dis)
} else {
val array = start.dataInputStream.getByteArray(nBits.toInt, start)
currentElement.setDataValue(array)
diff --git
a/daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/LayeredSequenceParser.scala
b/daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/LayeredSequenceParser.scala
index 18f181f0d..25cc5c06b 100644
---
a/daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/LayeredSequenceParser.scala
+++
b/daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/LayeredSequenceParser.scala
@@ -17,7 +17,6 @@
package org.apache.daffodil.runtime1.processors.parsers
-import org.apache.daffodil.lib.util.MaybeULong
import org.apache.daffodil.runtime1.layers.LayerExecutionException
import org.apache.daffodil.runtime1.layers.LayerNotEnoughDataException
import org.apache.daffodil.runtime1.layers.LayerRuntimeInfo
@@ -62,7 +61,6 @@ class LayeredSequenceParser(
le.schemaFileLocation,
le.dataLocation,
le.nBytesRequired * 8,
- MaybeULong.Nope,
)
case e: Exception =>
throw LayerExecutionException(
diff --git
a/daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/Parser.scala
b/daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/Parser.scala
index d546b0869..d67548535 100644
---
a/daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/Parser.scala
+++
b/daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/Parser.scala
@@ -18,13 +18,13 @@
package org.apache.daffodil.runtime1.processors.parsers
import org.apache.daffodil.io.BacktrackingException
+import org.apache.daffodil.io.InputSourceDataInputStream
import org.apache.daffodil.lib.api.DataLocation
import org.apache.daffodil.lib.api.Diagnostic
import org.apache.daffodil.lib.exceptions.Assert
import org.apache.daffodil.lib.exceptions.SchemaFileLocation
import org.apache.daffodil.lib.util.Maybe.One
-import org.apache.daffodil.lib.util.MaybeULong
-import org.apache.daffodil.lib.util.Misc
+import org.apache.daffodil.lib.util.{ MaybeULong, Misc }
import org.apache.daffodil.runtime1.dsom.RuntimeSchemaDefinitionError
import org.apache.daffodil.runtime1.processors.CombinatorProcessor
import org.apache.daffodil.runtime1.processors.ElementRuntimeData
@@ -65,10 +65,49 @@ sealed trait Parser extends Processor {
pstate.setFailed(new ParseError(One(sfl), One(dataLoc), s, args: _*))
}
- def PENotEnoughBits(pstate: PState, neededBits: Long, remainingBits:
MaybeULong) = {
- val remainingStr =
- if (remainingBits.isDefined) s" but found only ${remainingBits.get}
available" else ""
- PE(pstate, "Insufficient bits in data. Needed %d bit(s)%s.", neededBits,
remainingStr)
+ def PENotEnoughBits(
+ pstate: PState,
+ sfl: SchemaFileLocation,
+ dataLoc: DataLocation,
+ neededBits: Long,
+ source: InputSourceDataInputStream,
+ ): Unit = {
+ val startPos = dataLoc.bitPos1b - 1
+ val remainingLimitedBits = {
+ if (source.bitLimit0b.isEmpty) MaybeULong.Nope
+ else {
+ val lim = source.bitLimit0b.get
+ Assert.invariant(lim >= 0)
+ val nBits = lim - startPos
+ MaybeULong(nBits)
+ }
+ }
+ val remainingBits = {
+ if (source.hasReachedEndOfData) {
+ val bitsAvailable = {
+ val fragmentBitsReadFromSourcePos = source.bitPos0b % 8
+ val bitsAvailableFromSourcePos =
+ source.knownBytesAvailable * 8 - fragmentBitsReadFromSourcePos
+ val bitsBetweenStartPosAndSourcePos = source.bitPos0b - startPos
+ val bitsAvailableFromStartPos =
+ bitsAvailableFromSourcePos + bitsBetweenStartPosAndSourcePos
+ bitsAvailableFromStartPos
+ }
+ if (remainingLimitedBits.isEmpty || bitsAvailable <
remainingLimitedBits.get)
+ bitsAvailable
+ else
+ remainingLimitedBits.get
+ } else remainingLimitedBits.get
+ }
+
+ PE(
+ pstate,
+ sfl,
+ dataLoc,
+ "Insufficient bits in data. Needed %d bit(s) but found only %d
available",
+ neededBits,
+ remainingBits,
+ )
}
def PENotEnoughBits(
@@ -76,17 +115,27 @@ sealed trait Parser extends Processor {
sfl: SchemaFileLocation,
dataLoc: DataLocation,
neededBits: Long,
- remainingBits: MaybeULong,
- ) = {
- val remainingStr =
- if (remainingBits.isDefined) s" but found only ${remainingBits.get}
available" else ""
+ ): Unit = {
PE(
pstate,
sfl,
dataLoc,
- "Insufficient bits in data. Needed %d bit(s)%s.",
+ "Insufficient bits in data. Needed %d bit(s)",
+ neededBits,
+ )
+ }
+
+ def PENotEnoughBits(
+ pstate: PState,
+ neededBits: Long,
+ source: InputSourceDataInputStream,
+ ): Unit = {
+ PENotEnoughBits(
+ pstate,
+ pstate.schemaFileLocation,
+ pstate.currentLocation,
neededBits,
- remainingStr,
+ source,
)
}
diff --git
a/daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/PrimitivesDateTime1.scala
b/daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/PrimitivesDateTime1.scala
index d075fe223..729fb42b3 100644
---
a/daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/PrimitivesDateTime1.scala
+++
b/daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/PrimitivesDateTime1.scala
@@ -203,7 +203,7 @@ case class ConvertBinaryCalendarSecMilliParser(
val dis = start.dataInputStream
if (!dis.isDefinedForLength(lengthInBits)) {
- PENotEnoughBits(start, lengthInBits, dis.remainingBits)
+ PENotEnoughBits(start, lengthInBits, dis)
return
}
diff --git
a/daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/SpecifiedLengthParsers.scala
b/daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/SpecifiedLengthParsers.scala
index 94f9145ee..07f6c1bde 100644
---
a/daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/SpecifiedLengthParsers.scala
+++
b/daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/SpecifiedLengthParsers.scala
@@ -67,7 +67,7 @@ sealed abstract class SpecifiedLengthParserBase(eParser:
Parser, erd: RuntimeDat
}
if (shouldCheckDefinedForLength && !dis.isDefinedForLength(nBits)) {
- PENotEnoughBits(pState, nBits, dis.remainingBits)
+ PENotEnoughBits(pState, nBits, dis)
return
}
@@ -96,7 +96,7 @@ sealed abstract class SpecifiedLengthParserBase(eParser:
Parser, erd: RuntimeDat
// skip left over bits
val skipSuccess = dis.skip(bitsToSkip, pState)
if (!skipSuccess) {
- PENotEnoughBits(pState, bitsToSkip, dis.remainingBits)
+ PENotEnoughBits(pState, bitsToSkip, dis)
}
}
}
diff --git
a/daffodil-tdml-processor/src/main/scala/org/apache/daffodil/processor/tdml/DaffodilTDMLDFDLProcessor.scala
b/daffodil-tdml-processor/src/main/scala/org/apache/daffodil/processor/tdml/DaffodilTDMLDFDLProcessor.scala
index 8290b9336..6bf60ebfc 100644
---
a/daffodil-tdml-processor/src/main/scala/org/apache/daffodil/processor/tdml/DaffodilTDMLDFDLProcessor.scala
+++
b/daffodil-tdml-processor/src/main/scala/org/apache/daffodil/processor/tdml/DaffodilTDMLDFDLProcessor.scala
@@ -269,13 +269,6 @@ class DaffodilTDMLDFDLProcessor private (private var dp:
DataProcessor)
saxInputStream: java.io.InputStream,
lengthLimitInBits: Long,
): TDMLParseResult = {
- //
- // TDML Tests MUST have a length limit. Otherwise they cannot determine if
- // there is left-over-data or not without doing more reading from the
input stream
- // so as to be sure to hit end-of-data.
- //
- Assert.usage(lengthLimitInBits >= 0)
-
val outputter = new TDMLInfosetOutputter()
outputter.setBlobAttributes(blobDir, blobPrefix, blobSuffix)
@@ -293,8 +286,14 @@ class DaffodilTDMLDFDLProcessor private (private var dp:
DataProcessor)
val dis = InputSourceDataInputStream(dpInputStream)
val sis = InputSourceDataInputStream(saxInputStream)
- dis.setBitLimit0b(MaybeULong(lengthLimitInBits))
- sis.setBitLimit0b(MaybeULong(lengthLimitInBits))
+ // The length limit here should be the length of the document
+ // under test. Only set a limit when the end of the document
+ // do not match a byte boundary.
+ if (lengthLimitInBits % 8 != 0) {
+ Assert.usage(lengthLimitInBits >= 0)
+ dis.setBitLimit0b(MaybeULong(lengthLimitInBits))
+ sis.setBitLimit0b(MaybeULong(lengthLimitInBits))
+ }
val actual = dp.parse(dis, outputter)
xri.parse(sis)
diff --git
a/daffodil-test/src/test/resources/org/apache/daffodil/section05/simple_types/Blobs.tdml
b/daffodil-test/src/test/resources/org/apache/daffodil/section05/simple_types/Blobs.tdml
index a71090adf..b5c982238 100644
---
a/daffodil-test/src/test/resources/org/apache/daffodil/section05/simple_types/Blobs.tdml
+++
b/daffodil-test/src/test/resources/org/apache/daffodil/section05/simple_types/Blobs.tdml
@@ -48,6 +48,16 @@
dfdl:lengthKind="explicit" dfdl:length="4"
dfdlx:objectKind="bytes" />
+ <xs:element name="blob_01_complex" dfdl:lengthKind="explicit"
dfdl:length="8">
+ <xs:complexType>
+ <xs:sequence>
+ <xs:element name="blob_01" type="xs:anyURI"
+ dfdl:lengthKind="explicit" dfdl:length="4"
+ dfdlx:objectKind="bytes" />
+ </xs:sequence>
+ </xs:complexType>
+ </xs:element>
+
<xs:element name="blob_02" type="xs:anyURI"
dfdl:lengthKind="explicit" dfdl:length="30" dfdl:lengthUnits="bits"
dfdl:bitOrder="mostSignificantBitFirst"
@@ -145,7 +155,29 @@
</tdml:dfdlInfoset>
</tdml:infoset>
</tdml:parserTestCase>
-
+
+ <tdml:parserTestCase name="blob_01_insufficient" root="blob_01"
model="Blob.dfdl.xsd">
+ <tdml:document>
+ <tdml:documentPart type="byte"><![CDATA[a1b1c1]]></tdml:documentPart>
+ <tdml:documentPart type="bits">1</tdml:documentPart>
+ </tdml:document>
+ <tdml:errors>
+ <tdml:error>Needed 32 bit(s)</tdml:error>
+ <tdml:error>found only 25</tdml:error>
+ </tdml:errors>
+ </tdml:parserTestCase>
+
+ <tdml:parserTestCase name="blob_01_insufficient_complex"
root="blob_01_complex" model="Blob.dfdl.xsd">
+ <tdml:document>
+ <tdml:documentPart type="byte"><![CDATA[a1b1c1]]></tdml:documentPart>
+ <tdml:documentPart type="bits">1</tdml:documentPart>
+ </tdml:document>
+ <tdml:errors>
+ <tdml:error>Needed 64 bit(s)</tdml:error>
+ <tdml:error>found only 25</tdml:error>
+ </tdml:errors>
+ </tdml:parserTestCase>
+
<!-- 30 bits of blob data, MSBF -->
<tdml:parserTestCase name="blob_02" root="blob_02" model="Blob.dfdl.xsd">
<tdml:document>
diff --git
a/daffodil-test/src/test/resources/org/apache/daffodil/section12/lengthKind/ExplicitTests.tdml
b/daffodil-test/src/test/resources/org/apache/daffodil/section12/lengthKind/ExplicitTests.tdml
index b9facca16..613d559ee 100644
---
a/daffodil-test/src/test/resources/org/apache/daffodil/section12/lengthKind/ExplicitTests.tdml
+++
b/daffodil-test/src/test/resources/org/apache/daffodil/section12/lengthKind/ExplicitTests.tdml
@@ -605,7 +605,8 @@
<tdml:documentPart type="text"><![CDATA[000118Ridgewood Circle
Rochester NY123]]></tdml:documentPart>
</tdml:document>
<tdml:errors>
- <tdml:error>6467715464</tdml:error>
+ <tdml:error>Needed 16</tdml:error>
+ <tdml:error>found only 8</tdml:error>
<tdml:error>insufficient</tdml:error>
<tdml:error>parse error</tdml:error>
</tdml:errors>
@@ -1338,4 +1339,54 @@
</tdml:errors>
</tdml:parserTestCase>
+ <tdml:defineSchema name="insufficientBits">
+ <xs:include
schemaLocation="org/apache/daffodil/xsd/DFDLGeneralFormat.dfdl.xsd"/>
+ <dfdl:format ref="ex:GeneralFormat"
+ representation="binary"
+ alignment="1"
+ alignmentUnits="bits"
+ />
+
+ <xs:element name="complex" dfdl:lengthKind="explicit" dfdl:length="16">
+ <xs:complexType>
+ <xs:sequence>
+ <xs:element name="child" type="xs:int" dfdl:lengthKind="explicit"
dfdl:length="4" />
+ </xs:sequence>
+ </xs:complexType>
+ </xs:element>
+
+ <xs:element name="nineBits" dfdl:lengthKind="explicit" dfdl:length="2">
+ <xs:complexType>
+ <xs:sequence>
+ <xs:element name="bit" type="xs:int" dfdl:lengthKind="explicit"
dfdl:lengthUnits="bits" dfdl:length="1"/>
+ <xs:element name="byte" type="xs:int" dfdl:lengthKind="explicit"
dfdl:lengthUnits="bits" dfdl:length="8"/>
+ </xs:sequence>
+ </xs:complexType>
+ </xs:element>
+ </tdml:defineSchema>
+
+ <tdml:parserTestCase name="insufficientBitsComplex" root="complex"
model="insufficientBits"
+ description="Tests that we get the correct error
message when there are not enough bits remaining in the stream">
+ <tdml:document>
+ <tdml:documentPart type="byte">
+ 12 34
+ </tdml:documentPart>
+ </tdml:document>
+ <tdml:errors>
+ <tdml:error>32 bit(s) but found only 16</tdml:error>
+ </tdml:errors>
+ </tdml:parserTestCase>
+
+ <tdml:parserTestCase name="insufficientBitsByte" root="nineBits"
model="insufficientBits"
+ description="Tests that we get the correct error
message when there are not enough bits remaining in the stream when unaligned">
+ <tdml:document>
+ <tdml:documentPart type="byte">
+ ff
+ </tdml:documentPart>
+ </tdml:document>
+ <tdml:errors>
+ <tdml:error>8 bit(s) but found only 7</tdml:error>
+ </tdml:errors>
+ </tdml:parserTestCase>
+
</tdml:testSuite>
diff --git
a/daffodil-test/src/test/scala/org/apache/daffodil/section05/simple_types/TestBlobs.scala
b/daffodil-test/src/test/scala/org/apache/daffodil/section05/simple_types/TestBlobs.scala
index 13b75726a..f450dd25f 100644
---
a/daffodil-test/src/test/scala/org/apache/daffodil/section05/simple_types/TestBlobs.scala
+++
b/daffodil-test/src/test/scala/org/apache/daffodil/section05/simple_types/TestBlobs.scala
@@ -37,6 +37,10 @@ class TestBlobs {
import TestBlobs._
@Test def test_blob_01(): Unit = { runner.runOneTest("blob_01") }
+ @Test def test_blob_01_insufficient(): Unit = {
runner.runOneTest("blob_01_insufficient") }
+ @Test def test_blob_01_insufficient_complex(): Unit = {
+ runner.runOneTest("blob_01_insufficient_complex")
+ }
@Test def test_blob_02(): Unit = { runner.runOneTest("blob_02") }
@Test def test_blob_03(): Unit = { runner.runOneTest("blob_03") }
@Test def test_blob_04(): Unit = { runner.runOneTest("blob_04") }
diff --git
a/daffodil-test/src/test/scala/org/apache/daffodil/section12/lengthKind/TestLengthKindExplicit.scala
b/daffodil-test/src/test/scala/org/apache/daffodil/section12/lengthKind/TestLengthKindExplicit.scala
index c5df7b6b3..e85cabf01 100644
---
a/daffodil-test/src/test/scala/org/apache/daffodil/section12/lengthKind/TestLengthKindExplicit.scala
+++
b/daffodil-test/src/test/scala/org/apache/daffodil/section12/lengthKind/TestLengthKindExplicit.scala
@@ -153,4 +153,12 @@ class TestLengthKindExplicit {
@Test def test_invalidByteBitLengthExpr(): Unit = {
runner.runOneTest("invalidByteBitLengthExpr")
}
+
+ @Test def test_insufficientBitsComplex(): Unit = {
+ runner.runOneTest("insufficientBitsComplex")
+ }
+
+ @Test def test_insufficientBitsByte(): Unit = {
+ runner.runOneTest("insufficientBitsByte")
+ }
}
