This is an automated email from the ASF dual-hosted git repository.
slawrence pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-daffodil.git
The following commit(s) were added to refs/heads/master by this push:
new ba2e006 DAFFODIL-2293
ba2e006 is described below
commit ba2e006a9556f4f3cfde8ffe377f476519b4806b
Author: Kurt Baumann (E518309) <[email protected]>
AuthorDate: Fri Mar 27 07:32:07 2020 +0100
DAFFODIL-2293
---
.../charset/BitsCharsetNonByteSize.scala | 2 +-
.../charset/DaffodilCharsetProvider.scala | 2 +
.../processors/charset/ISO885918BitPacked.scala | 44 ++++++++++++
.../io/TestNonByteSizedCharsetDecoders8Bit.scala | 82 +++++++++++++++++++++
.../io/TestNonByteSizedCharsetEncoders8Bit.scala | 83 ++++++++++++++++++++++
.../representation_properties/encodings.tdml | 53 ++++++++++++++
.../representation_properties/TestRepProps2.scala | 4 +-
7 files changed, 268 insertions(+), 2 deletions(-)
diff --git
a/daffodil-io/src/main/scala/org/apache/daffodil/processors/charset/BitsCharsetNonByteSize.scala
b/daffodil-io/src/main/scala/org/apache/daffodil/processors/charset/BitsCharsetNonByteSize.scala
index bb18f32..14a8bd4 100644
---
a/daffodil-io/src/main/scala/org/apache/daffodil/processors/charset/BitsCharsetNonByteSize.scala
+++
b/daffodil-io/src/main/scala/org/apache/daffodil/processors/charset/BitsCharsetNonByteSize.scala
@@ -65,7 +65,7 @@ trait BitsCharsetNonByteSize extends BitsCharset {
def maxBitsPerChar() = 1 / maxCharsPerBit()
Assert.usage(decodeString.length == (1 << bitWidthOfACodeUnit))
- Assert.usage(bitWidthOfACodeUnit <= 7 && bitWidthOfACodeUnit >= 1)
+ Assert.usage(bitWidthOfACodeUnit <= 8 && bitWidthOfACodeUnit >= 1)
override def newDecoder(): BitsCharsetDecoder = new
BitsCharsetNonByteSizeDecoder(this)
diff --git
a/daffodil-io/src/main/scala/org/apache/daffodil/processors/charset/DaffodilCharsetProvider.scala
b/daffodil-io/src/main/scala/org/apache/daffodil/processors/charset/DaffodilCharsetProvider.scala
index 2968716..af3a505 100644
---
a/daffodil-io/src/main/scala/org/apache/daffodil/processors/charset/DaffodilCharsetProvider.scala
+++
b/daffodil-io/src/main/scala/org/apache/daffodil/processors/charset/DaffodilCharsetProvider.scala
@@ -45,6 +45,8 @@ object DaffodilCharsetProvider {
BitsCharsetUSASCII6BitPackedLSBF,
BitsCharsetUSASCII6BitPackedMSBF,
BitsCharsetUSASCII7BitPacked,
+ BitsCharsetISO885918BitPackedLSBF,
+ BitsCharsetISO885918BitPackedMSBF,
BitsCharsetUTF16BE,
BitsCharsetUTF16LE,
BitsCharsetUTF32BE,
diff --git
a/daffodil-io/src/main/scala/org/apache/daffodil/processors/charset/ISO885918BitPacked.scala
b/daffodil-io/src/main/scala/org/apache/daffodil/processors/charset/ISO885918BitPacked.scala
new file mode 100644
index 0000000..1e57b08
--- /dev/null
+++
b/daffodil-io/src/main/scala/org/apache/daffodil/processors/charset/ISO885918BitPacked.scala
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.daffodil.processors.charset
+
+import org.apache.daffodil.schema.annotation.props.gen.BitOrder
+
+/**
+ * X-DFDL-ISO-88591-8-BIT-PACKED-LSB-FIRST occupies only 8 bits with each
+ * code unit.
+ */
+object BitsCharsetISO885918BitPackedLSBF extends {
+ override val name = "X-DFDL-ISO-88591-8-BIT-PACKED-LSB-FIRST"
+ override val bitWidthOfACodeUnit = 8
+ override val decodeString = (0 to 255).map { _.toChar }.mkString
+ override val replacementCharCode = 0x3F
+ override val requiredBitOrder = BitOrder.LeastSignificantBitFirst
+} with BitsCharsetNonByteSize
+
+/**
+ * X-DFDL-ISO-88591-8-BIT-PACKED-MSB-FIRST occupies only 8 bits with each
+ * code unit.
+ */
+object BitsCharsetISO885918BitPackedMSBF extends {
+ override val name = "X-DFDL-ISO-88591-8-BIT-PACKED-MSB-FIRST"
+ override val bitWidthOfACodeUnit = 8
+ override val decodeString = (0 to 255).map { _.toChar }.mkString
+ override val replacementCharCode = 0x3F
+ override val requiredBitOrder = BitOrder.MostSignificantBitFirst
+} with BitsCharsetNonByteSize
diff --git
a/daffodil-io/src/test/scala/org/apache/daffodil/io/TestNonByteSizedCharsetDecoders8Bit.scala
b/daffodil-io/src/test/scala/org/apache/daffodil/io/TestNonByteSizedCharsetDecoders8Bit.scala
new file mode 100644
index 0000000..3b775e1
--- /dev/null
+++
b/daffodil-io/src/test/scala/org/apache/daffodil/io/TestNonByteSizedCharsetDecoders8Bit.scala
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.daffodil.io
+
+import java.nio.ByteBuffer
+import java.nio.CharBuffer
+
+import org.junit.Assert._
+import org.junit.Test
+
+import org.apache.daffodil.processors.charset.CharsetUtils
+import org.apache.daffodil.util.Misc
+import org.apache.daffodil.util.MaybeULong
+import org.apache.daffodil.schema.annotation.props.gen.ByteOrder
+import org.apache.daffodil.schema.annotation.props.gen.BitOrder
+
+class TestNonByteSizedCharsetDecoders8Bit {
+
+ val lsbfFinfo = FormatInfoForUnitTest()
+ lsbfFinfo.byteOrder = ByteOrder.BigEndian
+ lsbfFinfo.bitOrder = BitOrder.LeastSignificantBitFirst
+
+ val msbfFinfo = FormatInfoForUnitTest()
+ msbfFinfo.byteOrder = ByteOrder.BigEndian
+ msbfFinfo.bitOrder = BitOrder.MostSignificantBitFirst
+
+
+ @Test def test8BitMSBF_01(): Unit = {
+ val cs = CharsetUtils.getCharset("X-DFDL-ISO-88591-8-BIT-PACKED-MSB-FIRST")
+ val decoder = cs.newDecoder()
+ val cb = CharBuffer.allocate(8)
+ val bb = ByteBuffer.wrap(Misc.bits2Bytes("00110000 00110001 00110010
00110011 00110100 00110101 00110110 00110111"))
+ val dis = InputSourceDataInputStream(bb)
+ val res = decoder.decode(dis, msbfFinfo, cb)
+ assertEquals(8, res)
+ cb.flip()
+ val digits = cb.toString()
+ assertEquals("01234567", digits)
+ }
+
+ @Test def test8BitMSBF_02(): Unit = {
+ val cs = CharsetUtils.getCharset("X-DFDL-ISO-88591-8-BIT-PACKED-MSB-FIRST")
+ val decoder = cs.newDecoder()
+ val cb = CharBuffer.allocate(7) // not enough space for last digit
+ val bb = ByteBuffer.wrap(Misc.bits2Bytes("00110000 00110001 00110010
00110011 00110100 00110101 00110110 00110111"))
+ val dis = InputSourceDataInputStream(bb)
+ //dis.skip(4, msbfFinfo)
+ val res = decoder.decode(dis, msbfFinfo, cb)
+ assertEquals(7, res)
+ cb.flip()
+ val digits = cb.toString()
+ assertEquals("0123456", digits)
+ }
+
+ @Test def test8BitLSBF_01(): Unit = {
+ val cs = CharsetUtils.getCharset("X-DFDL-ISO-88591-8-BIT-PACKED-LSB-FIRST")
+ val decoder = cs.newDecoder()
+ val cb = CharBuffer.allocate(64)
+ val bb = ByteBuffer.wrap(Misc.bits2Bytes("00110000 00110001 00110010
00110011 00110100 00110101 00110110 00110111"))
+ val dis = InputSourceDataInputStream(bb)
+ val res = decoder.decode(dis, lsbfFinfo, cb)
+ assertEquals(8, res)
+ cb.flip()
+ val digits = cb.toString()
+ assertEquals("01234567", digits)
+ }
+}
diff --git
a/daffodil-io/src/test/scala/org/apache/daffodil/io/TestNonByteSizedCharsetEncoders8Bit.scala
b/daffodil-io/src/test/scala/org/apache/daffodil/io/TestNonByteSizedCharsetEncoders8Bit.scala
new file mode 100644
index 0000000..8bf7450
--- /dev/null
+++
b/daffodil-io/src/test/scala/org/apache/daffodil/io/TestNonByteSizedCharsetEncoders8Bit.scala
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.daffodil.io
+
+import java.nio.ByteBuffer
+import java.nio.CharBuffer
+
+import org.junit.Assert._
+import org.junit.Test
+
+import org.apache.daffodil.processors.charset.CharsetUtils
+import org.apache.daffodil.util.Misc
+
+class TestNonByteSizedCharsetEncoders8Bit {
+
+ @Test def test8BitMSBF_01(): Unit = {
+ val cs = CharsetUtils.getCharset("X-DFDL-ISO-88591-8-BIT-PACKED-MSB-FIRST")
+ val encoder = cs.newEncoder()
+ val cb = CharBuffer.wrap("01234567")
+ val expectedBytes = Misc.bits2Bytes("00110000 00110001 00110010 00110011
00110100 00110101 00110110 00110111").toList
+ val bb = ByteBuffer.allocate(8)
+ val res = encoder.encode(cb, bb, false)
+ assertTrue(res.isUnderflow())
+ bb.flip()
+ val actualBytes = bb.array().toList
+ assertEquals(expectedBytes, actualBytes)
+ }
+
+ @Test def test8BitMSBF_02(): Unit = {
+ val cs = CharsetUtils.getCharset("X-DFDL-ISO-88591-8-BIT-PACKED-MSB-FIRST")
+ val encoder = cs.newEncoder()
+ val cb = CharBuffer.wrap("012345677")
+ val expectedBytes = Misc.bits2Bytes("00110000 00110001 00110010 00110011
00110100 00110101 00110110 00110111").toList
+ val bb = ByteBuffer.allocate(8) // not enough space for last digit
+ val res = encoder.encode(cb, bb, false)
+ assertTrue(res.isOverflow())
+ bb.flip()
+ val actualBytes = bb.array().toList
+ assertEquals(expectedBytes, actualBytes)
+ }
+
+ @Test def test8BitLSBF_01(): Unit = {
+ val cs = CharsetUtils.getCharset("X-DFDL-ISO-88591-8-BIT-PACKED-LSB-FIRST")
+ val encoder = cs.newEncoder()
+ val cb = CharBuffer.wrap("01234567")
+ val expectedBytes = Misc.bits2Bytes("00110000 00110001 00110010 00110011
00110100 00110101 00110110 00110111").toList
+ val bb = ByteBuffer.allocate(8)
+ val res = encoder.encode(cb, bb, false)
+ assertTrue(res.isUnderflow())
+ bb.flip()
+ val actualBytes = bb.array().toList
+ assertEquals(expectedBytes, actualBytes)
+ }
+
+ @Test def test8BitLSBF_02(): Unit = {
+ val cs = CharsetUtils.getCharset("X-DFDL-ISO-88591-8-BIT-PACKED-LSB-FIRST")
+ val encoder = cs.newEncoder()
+ assertNotNull(encoder)
+ val cb = CharBuffer.wrap("012345677")
+ val expectedBytes = Misc.bits2Bytes("00110000 00110001 00110010 00110011
00110100 00110101 00110110 00110111").toList
+ val bb = ByteBuffer.allocate(8) // not enough space for last digit
+ val res = encoder.encode(cb, bb, false)
+ assertTrue(res.isOverflow())
+ bb.flip()
+ val actualBytes = bb.array().toList
+ assertEquals(expectedBytes, actualBytes)
+ }
+}
diff --git
a/daffodil-test/src/test/resources/org/apache/daffodil/section10/representation_properties/encodings.tdml
b/daffodil-test/src/test/resources/org/apache/daffodil/section10/representation_properties/encodings.tdml
index 93f009e..f9db6ec 100644
---
a/daffodil-test/src/test/resources/org/apache/daffodil/section10/representation_properties/encodings.tdml
+++
b/daffodil-test/src/test/resources/org/apache/daffodil/section10/representation_properties/encodings.tdml
@@ -28,6 +28,28 @@
<dfdl:format ref="tns:GeneralFormat" lengthKind="delimited"/>
+ <xs:simpleType name="bit" dfdl:lengthKind="explicit"
dfdl:lengthUnits="bits" dfdl:alignmentUnits="bits" dfdl:representation="binary"
>
+ <xs:restriction base="xs:unsignedInt" />
+ </xs:simpleType>
+
+ <xs:element name="isoMsb">
+ <xs:complexType>
+ <xs:sequence>
+ <xs:element name="a" type="ex:bit" dfdl:length="3" />
+ <xs:element name="b" type="xs:string"
dfdl:lengthKind="explicit" dfdl:length="8" dfdl:lengthUnits="bits"
dfdl:alignmentUnits="bits" dfdl:representation="text"
dfdl:encoding="X-DFDL-ISO-88591-8-BIT-PACKED-MSB-FIRST"/>
+ </xs:sequence>
+ </xs:complexType>
+ </xs:element>
+
+ <xs:element name="isoLsb">
+ <xs:complexType>
+ <xs:sequence>
+ <xs:element name="a" type="ex:bit" dfdl:length="3"
dfdl:bitOrder="leastSignificantBitFirst" dfdl:byteOrder="littleEndian" />
+ <xs:element name="b" type="xs:string"
dfdl:lengthKind="explicit" dfdl:length="8" dfdl:lengthUnits="bits"
dfdl:alignmentUnits="bits" dfdl:representation="text"
dfdl:bitOrder="leastSignificantBitFirst" dfdl:byteOrder="littleEndian"
dfdl:encoding="X-DFDL-ISO-88591-8-BIT-PACKED-LSB-FIRST"/>
+ </xs:sequence>
+ </xs:complexType>
+ </xs:element>
+
<xs:element name="eString" type="xs:string" dfdl:encoding="ebcdic-cp-us"/>
<xs:element name="bString" type="xs:string" dfdl:encoding="x-dfdl-bits-lsbf"
@@ -49,6 +71,37 @@
</tdml:defineSchema>
+
+ <tdml:parserTestCase name="iso88591msbbitsmisaligned" root="isoMsb"
model="enc1"
+ description="ISO-8859-1 testcase (MSB) Bits (misaligned)">
+ <tdml:document>
+ <tdml:documentPart type="bits">10100110111</tdml:documentPart>
+ </tdml:document>
+ <tdml:infoset>
+ <tdml:dfdlInfoset>
+ <tns:isoMsb>
+ <ex:a>5</ex:a>
+ <ex:b>7</ex:b>
+ </tns:isoMsb>
+ </tdml:dfdlInfoset>
+ </tdml:infoset>
+ </tdml:parserTestCase>
+
+ <tdml:parserTestCase name="iso88591lsbbitsmisaligned" root="isoLsb"
model="enc1"
+ description="ISO-8859-1 testcase (LSB) Bits (misaligned)">
+ <tdml:document>
+ <tdml:documentPart type="bits" bitOrder="LSBFirst"
byteOrder="RTL">001 10111101</tdml:documentPart>
+ </tdml:document>
+ <tdml:infoset>
+ <tdml:dfdlInfoset>
+ <tns:isoLsb>
+ <ex:a>5</ex:a>
+ <ex:b>7</ex:b>
+ </tns:isoLsb>
+ </tdml:dfdlInfoset>
+ </tdml:infoset>
+ </tdml:parserTestCase>
+
<tdml:parserTestCase name="ebcdic1" root="eString" model="enc1"
description="EBCDIC basic test">
<tdml:document>
diff --git
a/daffodil-test/src/test/scala/org/apache/daffodil/section10/representation_properties/TestRepProps2.scala
b/daffodil-test/src/test/scala/org/apache/daffodil/section10/representation_properties/TestRepProps2.scala
index c9bb236..d3a2aa5 100644
---
a/daffodil-test/src/test/scala/org/apache/daffodil/section10/representation_properties/TestRepProps2.scala
+++
b/daffodil-test/src/test/scala/org/apache/daffodil/section10/representation_properties/TestRepProps2.scala
@@ -32,7 +32,7 @@ object TestRepProps2 {
class TestRepProps2 {
import TestRepProps2._
-
+
@Test def test_ebcdic1() = { runner.runOneTest("ebcdic1") }
@Test def test_bits1() = { runner.runOneTest("bits1") }
@Test def test_bits1a() = { runner.runOneTest("bits1a") }
@@ -50,4 +50,6 @@ class TestRepProps2 {
@Test def test_sixBit1() = { runner.runOneTest("sixBit1") }
+ @Test def test_iso88591msbbitsmisaligned() = {
runner.runOneTest("iso88591msbbitsmisaligned") }
+ @Test def test_iso88591lsbbitsmisaligned() = {
runner.runOneTest("iso88591lsbbitsmisaligned") }
}
