stevedlawrence closed pull request #146: Fix hexBinary with non-byte lengths or
non-byte boundaries
URL: https://github.com/apache/incubator-daffodil/pull/146
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git
a/daffodil-core/src/test/scala/org/apache/daffodil/layers/TestLayers.scala
b/daffodil-core/src/test/scala/org/apache/daffodil/layers/TestLayers.scala
index 3d88e828d..d48b68841 100644
--- a/daffodil-core/src/test/scala/org/apache/daffodil/layers/TestLayers.scala
+++ b/daffodil-core/src/test/scala/org/apache/daffodil/layers/TestLayers.scala
@@ -403,13 +403,13 @@ a few lines of pointless text like this.""".replace("\n",
" ")
val infoset =
<e1 xmlns={ example }>
<s0>00</s0>
- <s1>01</s1>
+ <s1>10</s1>
<s2>02</s2>
<s3>333333</s3>
<s4>44</s4>
- <s5>0555</s5>
- <s6>06</s6>
- <s7>07</s7>
+ <s5>5505</s5>
+ <s6>60</s6>
+ <s7>70</s7>
</e1>
val (_, actual) = TestUtils.testBinary(sch, data, areTracing = false)
diff --git
a/daffodil-runtime1-unparser/src/main/scala/org/apache/daffodil/processors/unparsers/HexBinaryLengthUnparser.scala
b/daffodil-runtime1-unparser/src/main/scala/org/apache/daffodil/processors/unparsers/HexBinaryLengthUnparser.scala
index cae458857..ce81512e3 100644
---
a/daffodil-runtime1-unparser/src/main/scala/org/apache/daffodil/processors/unparsers/HexBinaryLengthUnparser.scala
+++
b/daffodil-runtime1-unparser/src/main/scala/org/apache/daffodil/processors/unparsers/HexBinaryLengthUnparser.scala
@@ -17,15 +17,17 @@
package org.apache.daffodil.processors.unparsers
+import passera.unsigned.ULong
+
import org.apache.daffodil.exceptions.Assert
-import org.apache.daffodil.infoset.DISimple
-import org.apache.daffodil.infoset.Infoset
import org.apache.daffodil.infoset.RetryableException
import org.apache.daffodil.processors.ElementRuntimeData
import org.apache.daffodil.processors.ParseOrUnparseState
import org.apache.daffodil.processors.Processor
import org.apache.daffodil.processors.UnparseTargetLengthInBitsEv
+import org.apache.daffodil.schema.annotation.props.gen.BitOrder
import org.apache.daffodil.schema.annotation.props.gen.LengthUnits
+import org.apache.daffodil.util.Bits
import org.apache.daffodil.util.Maybe._
abstract class HexBinaryUnparserBase(override val context: ElementRuntimeData)
@@ -60,12 +62,30 @@ abstract class HexBinaryUnparserBase(override val context:
ElementRuntimeData)
val dos = state.dataOutputStream
- // put the hex binary array
- if (bitsFromValueToPut > 0) {
- val ret = dos.putByteArray(value, bitsFromValueToPut.toInt, state)
+ // We cannot use dos.putByteArray(...) because that function takes into
+ // account byteOrder. hexBinary should completely ignore byteOrder, only
+ // taking into account bitOrder. To do this, we want to just repeatedly put
+ // 8 bit's at a time, effecitively ignores byteOrder.
+ var pos = 0
+ var bitsRemaining = bitsFromValueToPut
+ while (bitsRemaining > 0) {
+ val byte = Bits.asUnsignedByte(value(pos))
+ val bitsToPut = Math.min(bitsRemaining, 8).toInt
+
+ val adjustedForBitOrder =
+ if (bitsToPut < 8 && state.bitOrder ==
BitOrder.MostSignificantBitFirst) {
+ byte >> (8 - bitsToPut)
+ } else {
+ byte
+ }
+
+ val ret = dos.putULong(ULong(adjustedForBitOrder), bitsToPut, state)
if (!ret) {
- UnparseError(One(context.schemaFileLocation),
One(state.currentLocation), "Failed to write %d hexBinary bits",
bitsFromValueToPut)
+ UnparseError(One(context.schemaFileLocation),
One(state.currentLocation), "Failed to write byte % from hexBinary data", pos)
}
+
+ pos += 1
+ bitsRemaining -= bitsToPut
}
// calculate the skip bits
diff --git
a/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/parsers/HexBinaryLengthParsers.scala
b/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/parsers/HexBinaryLengthParsers.scala
index 484ab6de5..4830f0eec 100644
---
a/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/parsers/HexBinaryLengthParsers.scala
+++
b/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/parsers/HexBinaryLengthParsers.scala
@@ -22,6 +22,7 @@ import java.lang.{ Long => JLong }
import org.apache.daffodil.processors.ElementRuntimeData
import org.apache.daffodil.processors.LengthInBitsEv
import org.apache.daffodil.processors.Processor
+import org.apache.daffodil.schema.annotation.props.gen.BitOrder
import org.apache.daffodil.schema.annotation.props.gen.LengthUnits
import org.apache.daffodil.util.Numbers
@@ -41,7 +42,30 @@ sealed abstract class HexBinaryLengthParser(override val
context: ElementRuntime
} else if (!dis.isDefinedForLength(nBits)) {
PENotEnoughBits(start, nBits, dis.remainingBits)
} else {
- val array = start.dataInputStream.getByteArray(nBits, start)
+ // We cannot use dataInputStream.getByteArray(nBits) because that
+ // function takes into account byteOrder. hexBinary should completely
+ // ignore byteOrder, only taking into account bitOrder. To do this, we
+ // want to just repeatedly read 8 bit's at a time and add them to an
+ // ByteArray, effecitively ignoring byteOrder.
+ val nBytes = (nBits + 7) >> 3
+ val array = new Array[Byte](nBytes)
+ var bitsRemaining = nBits
+ var pos = 0
+ while (bitsRemaining > 0) {
+ val bitsToGet = Math.min(bitsRemaining, 8)
+ val longVal = start.dataInputStream.getUnsignedLong(bitsToGet, start)
+
+ val adjustedForBitOrder =
+ if (bitsToGet < 8 && start.bitOrder ==
BitOrder.MostSignificantBitFirst) {
+ (longVal << (8 - bitsToGet))
+ } else {
+ longVal
+ }
+
+ array(pos) = adjustedForBitOrder.toByte
+ pos += 1
+ bitsRemaining -= bitsToGet
+ }
currentElement.setDataValue(array)
}
}
diff --git
a/daffodil-test/src/test/resources/org/apache/daffodil/section05/simple_types/BitOrder.tdml
b/daffodil-test/src/test/resources/org/apache/daffodil/section05/simple_types/BitOrder.tdml
index 696c9189b..50c805601 100644
---
a/daffodil-test/src/test/resources/org/apache/daffodil/section05/simple_types/BitOrder.tdml
+++
b/daffodil-test/src/test/resources/org/apache/daffodil/section05/simple_types/BitOrder.tdml
@@ -375,7 +375,7 @@
<tdml:infoset>
<tdml:dfdlInfoset xmlns:xs="http://www.w3.org/2001/XMLSchema";
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
xmlns="http://example.com";>
- <littleLeast>00925A</littleLeast>
+ <littleLeast>5A9200</littleLeast>
</tdml:dfdlInfoset>
</tdml:infoset>
</tdml:parserTestCase>
@@ -394,7 +394,7 @@
<tdml:infoset>
<tdml:dfdlInfoset xmlns:xs="http://www.w3.org/2001/XMLSchema";
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
xmlns="http://example.com";>
- <littleLeast>5A9200</littleLeast>
+ <littleLeast>00925A</littleLeast>
</tdml:dfdlInfoset>
</tdml:infoset>
</tdml:parserTestCase>
diff --git
a/daffodil-test/src/test/resources/org/apache/daffodil/section05/simple_types/SimpleTypes.tdml
b/daffodil-test/src/test/resources/org/apache/daffodil/section05/simple_types/SimpleTypes.tdml
index 862490c3f..2a8927366 100644
---
a/daffodil-test/src/test/resources/org/apache/daffodil/section05/simple_types/SimpleTypes.tdml
+++
b/daffodil-test/src/test/resources/org/apache/daffodil/section05/simple_types/SimpleTypes.tdml
@@ -7138,6 +7138,55 @@
</xs:complexType>
</xs:element>
+ <xs:element name="hB_bits_le_lsbf_2">
+ <xs:complexType>
+ <xs:sequence>
+ <xs:element name="hb1" type="xs:hexBinary"
dfdl:alignmentUnits="bits" dfdl:lengthKind="explicit"
+ dfdl:lengthUnits="bits" dfdl:length="5"
dfdl:bitOrder="leastSignificantBitFirst"
+ dfdl:byteOrder="littleEndian" />
+ <xs:element name="hb2" type="xs:hexBinary"
dfdl:alignmentUnits="bits" dfdl:lengthKind="explicit"
+ dfdl:lengthUnits="bits" dfdl:length="22"
dfdl:bitOrder="leastSignificantBitFirst"
+ dfdl:byteOrder="littleEndian" />
+ <xs:element name="hb3" type="xs:hexBinary"
dfdl:alignmentUnits="bits" dfdl:lengthKind="explicit"
+ dfdl:lengthUnits="bits" dfdl:length="5"
dfdl:bitOrder="leastSignificantBitFirst"
+ dfdl:byteOrder="littleEndian" />
+ </xs:sequence>
+ </xs:complexType>
+ </xs:element>
+
+ <xs:element name="hB_bits_be_msbf_2">
+ <xs:complexType>
+ <xs:sequence>
+ <xs:element name="hb1" type="xs:hexBinary"
dfdl:alignmentUnits="bits" dfdl:lengthKind="explicit"
+ dfdl:lengthUnits="bits" dfdl:length="5"
dfdl:bitOrder="mostSignificantBitFirst"
+ dfdl:byteOrder="bigEndian" />
+ <xs:element name="hb2" type="xs:hexBinary"
dfdl:alignmentUnits="bits" dfdl:lengthKind="explicit"
+ dfdl:lengthUnits="bits" dfdl:length="22"
dfdl:bitOrder="mostSignificantBitFirst"
+ dfdl:byteOrder="bigEndian" />
+ <xs:element name="hb3" type="xs:hexBinary"
dfdl:alignmentUnits="bits" dfdl:lengthKind="explicit"
+ dfdl:lengthUnits="bits" dfdl:length="5"
dfdl:bitOrder="mostSignificantBitFirst"
+ dfdl:byteOrder="bigEndian" />
+ </xs:sequence>
+ </xs:complexType>
+ </xs:element>
+
+ <xs:element name="hB_bits_le_msbf_2"
+ dfdl:bitOrder="mostSignificantBitFirst" dfdl:byteOrder="littleEndian">
+ <xs:complexType>
+ <xs:sequence>
+ <xs:element name="hb1" type="xs:hexBinary"
dfdl:alignmentUnits="bits" dfdl:lengthKind="explicit"
+ dfdl:lengthUnits="bits" dfdl:length="5"
dfdl:bitOrder="mostSignificantBitFirst"
+ dfdl:byteOrder="littleEndian" />
+ <xs:element name="hb2" type="xs:hexBinary"
dfdl:alignmentUnits="bits" dfdl:lengthKind="explicit"
+ dfdl:lengthUnits="bits" dfdl:length="22"
dfdl:bitOrder="mostSignificantBitFirst"
+ dfdl:byteOrder="littleEndian" />
+ <xs:element name="hb3" type="xs:hexBinary"
dfdl:alignmentUnits="bits" dfdl:lengthKind="explicit"
+ dfdl:lengthUnits="bits" dfdl:length="5"
dfdl:bitOrder="mostSignificantBitFirst"
+ dfdl:byteOrder="littleEndian" />
+ </xs:sequence>
+ </xs:complexType>
+ </xs:element>
+
</tdml:defineSchema>
<!--
@@ -7491,7 +7540,7 @@
<tdml:infoset>
<tdml:dfdlInfoset>
<hB_bits_le_lsbf>
- <hb1>0A35</hb1>
+ <hb1>350A</hb1>
<hb2>05</hb2>
</hB_bits_le_lsbf>
</tdml:dfdlInfoset>
@@ -7507,8 +7556,8 @@
<tdml:infoset>
<tdml:dfdlInfoset>
<hB_bits_be_msbf>
- <hb1>06B5</hb1>
- <hb2>02</hb2>
+ <hb1>35A8</hb1>
+ <hb2>40</hb2>
</hB_bits_be_msbf>
</tdml:dfdlInfoset>
</tdml:infoset>
@@ -7523,12 +7572,63 @@
<tdml:infoset>
<tdml:dfdlInfoset>
<hB_bits_le_msbf>
- <hb1>1535</hb1>
- <hb2>02</hb2>
+ <hb1>35A8</hb1>
+ <hb2>40</hb2>
</hB_bits_le_msbf>
</tdml:dfdlInfoset>
</tdml:infoset>
</tdml:parserTestCase>
+
+ <tdml:parserTestCase name="hexBinary_bits_le_lsbf_2"
+ root="hB_bits_le_lsbf_2" model="HexBinary"
+ description="Section 5 Simple Types - hexBinary - DFDL-5-025R"
roundTrip="true">
+ <tdml:document>
+ <tdml:documentPart type="byte">DEADBEEF</tdml:documentPart>
+ </tdml:document>
+ <tdml:infoset>
+ <tdml:dfdlInfoset>
+ <hB_bits_le_lsbf_2>
+ <hb1>1E</hb1>
+ <hb2>6EF53D</hb2>
+ <hb3>1D</hb3>
+ </hB_bits_le_lsbf_2>
+ </tdml:dfdlInfoset>
+ </tdml:infoset>
+ </tdml:parserTestCase>
+
+ <tdml:parserTestCase name="hexBinary_bits_be_msbf_2"
+ root="hB_bits_be_msbf_2" model="HexBinary"
+ description="Section 5 Simple Types - hexBinary - DFDL-5-025R"
roundTrip="true">
+ <tdml:document>
+ <tdml:documentPart type="byte">DEADBEEF</tdml:documentPart>
+ </tdml:document>
+ <tdml:infoset>
+ <tdml:dfdlInfoset>
+ <hB_bits_be_msbf_2>
+ <hb1>D8</hb1>
+ <hb2>D5B7DC</hb2>
+ <hb3>78</hb3>
+ </hB_bits_be_msbf_2>
+ </tdml:dfdlInfoset>
+ </tdml:infoset>
+ </tdml:parserTestCase>
+
+ <tdml:parserTestCase name="hexBinary_bits_le_msbf_2"
+ root="hB_bits_le_msbf_2" model="HexBinary"
+ description="Section 5 Simple Types - hexBinary - DFDL-5-025R"
roundTrip="true">
+ <tdml:document>
+ <tdml:documentPart type="byte">DEADBEEF</tdml:documentPart>
+ </tdml:document>
+ <tdml:infoset>
+ <tdml:dfdlInfoset>
+ <hB_bits_le_msbf_2>
+ <hb1>D8</hb1>
+ <hb2>D5B7DC</hb2>
+ <hb3>78</hb3>
+ </hB_bits_le_msbf_2>
+ </tdml:dfdlInfoset>
+ </tdml:infoset>
+ </tdml:parserTestCase>
<!--
Test name: NonNegativeInteger
diff --git
a/daffodil-test/src/test/scala/org/apache/daffodil/section05/simple_types/TestSimpleTypes.scala
b/daffodil-test/src/test/scala/org/apache/daffodil/section05/simple_types/TestSimpleTypes.scala
index cac9fd549..f19da1f47 100644
---
a/daffodil-test/src/test/scala/org/apache/daffodil/section05/simple_types/TestSimpleTypes.scala
+++
b/daffodil-test/src/test/scala/org/apache/daffodil/section05/simple_types/TestSimpleTypes.scala
@@ -57,11 +57,14 @@ class TestSimpleTypes {
@Test def test_hexBinary_func() { runner.runOneTest("hexBinary_func") }
@Test def test_hexBinary_func_neg() {
runner.runOneTest("hexBinary_func_neg") }
- // DFDL-1707
@Test def test_hexBinary_bits_be_msbf() {
runner.runOneTest("hexBinary_bits_be_msbf") }
@Test def test_hexBinary_bits_le_msbf() {
runner.runOneTest("hexBinary_bits_le_msbf") }
@Test def test_hexBinary_bits_le_lsbf() {
runner.runOneTest("hexBinary_bits_le_lsbf") }
+ @Test def test_hexBinary_bits_be_msbf_2() {
runner.runOneTest("hexBinary_bits_be_msbf_2") }
+ @Test def test_hexBinary_bits_le_msbf_2() {
runner.runOneTest("hexBinary_bits_le_msbf_2") }
+ @Test def test_hexBinary_bits_le_lsbf_2() {
runner.runOneTest("hexBinary_bits_le_lsbf_2") }
+
@Test def test_dateTextNumberRep() { runner.runOneTest("dateTextNumberRep") }
@Test def test_datePattern02() { runner.runOneTest("datePattern02") }
diff --git a/tutorials/src/main/resources/bitorder.tutorial.tdml.xml
b/tutorials/src/main/resources/bitorder.tutorial.tdml.xml
index 7a1cbff2f..d6870e7a0 100644
--- a/tutorials/src/main/resources/bitorder.tutorial.tdml.xml
+++ b/tutorials/src/main/resources/bitorder.tutorial.tdml.xml
@@ -1714,7 +1714,7 @@ The schema corresponding to this table is given below.
This schema is derived fr
</document>
<tdml:infoset>
<tdml:dfdlInfoset xmlns="http://example.com";>
- <littleLeast>00925A</littleLeast>
+ <littleLeast>5A9200</littleLeast>
</tdml:dfdlInfoset>
</tdml:infoset>
</tdml:parserTestCase>
@@ -1744,7 +1744,7 @@ The schema corresponding to this table is given below.
This schema is derived fr
</document>
<tdml:infoset>
<tdml:dfdlInfoset xmlns="http://example.com";>
- <littleLeast>5A9200</littleLeast>
+ <littleLeast>00925A</littleLeast>
</tdml:dfdlInfoset>
</tdml:infoset>
</tdml:parserTestCase>
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
