tuxji commented on a change in pull request #643: URL: https://github.com/apache/daffodil/pull/643#discussion_r716734034
########## File path: daffodil-test/src/test/resources/org/apache/daffodil/layers/IPv4.dfdl.xsd ########## @@ -0,0 +1,199 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<!-- +Copyright (c) 2012-2021 Owl Cyber Defense. All rights reserved. + +Developed by: Owl Cyber Defense + http://www.owlcyberdefense.com + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal with +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimers. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimers in the + documentation and/or other materials provided with the distribution. + + 3. Neither the names of Owl Cyber Defense, nor the names of its contributors + may be used to endorse or promote products derived from this Software + without specific prior written permission. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE +SOFTWARE. +--> + +<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema"; + xmlns:dfdl="http://www.ogf.org/dfdl/dfdl-1.0/"; + xmlns:dfdlx="http://www.ogf.org/dfdl/dfdl-1.0/extensions"; + xmlns:fn="http://www.w3.org/2005/xpath-functions"; + xmlns:daf="urn:ogf:dfdl:2013:imp:daffodil.apache.org:2018:ext" + xmlns:chksum="urn:org.apache.daffodil.layers.IPv4Checksum" + xmlns:eth="urn:ethernet" + xmlns:tns="urn:ethernet" + targetNamespace="urn:ethernet"> Review comment: Why does this schema define its namespace as "urn:ethernet" when it is defined in a file with the name "IPv4.dfdl.xsd"? Also, Ethernet is a transmission link layer one layer below the internet layer and we can run IPv4 over other types of transmission links as well (e.g., wireless aka Wi-Fi). Would "urn:IPv4" be a more accurate namespace for this schema? ########## File path: daffodil-test/src/test/scala/org/apache/daffodil/layers/CheckDigit.scala ########## @@ -0,0 +1,205 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.daffodil.layers + +import org.apache.commons.io.IOUtils +import org.apache.daffodil.schema.annotation.props.gen.LayerLengthKind +import org.apache.daffodil.schema.annotation.props.gen.LayerLengthUnits +import org.apache.daffodil.util.Maybe +import org.apache.daffodil.processors.LayerLengthEv +import org.apache.daffodil.processors.LayerBoundaryMarkEv +import org.apache.daffodil.processors.LayerCharsetEv +import org.apache.daffodil.processors.parsers.PState +import org.apache.daffodil.processors.unparsers.UState + +import java.io._ +import org.apache.daffodil.dpath.NodeInfo.PrimType +import org.apache.daffodil.exceptions.Assert +import org.apache.daffodil.io.DataInputStream +import org.apache.daffodil.processors.CharsetEvBase +import org.apache.daffodil.processors.SequenceRuntimeData +import org.apache.daffodil.processors.SuspendableOperation +import org.apache.daffodil.processors.charset.BitsCharsetJava +import org.apache.daffodil.util.ByteBufferOutputStream +import org.apache.daffodil.xml.NS +import org.apache.daffodil.xml.RefQName + +import java.nio.ByteBuffer +import java.nio.charset.Charset + + +final class CheckDigitExplicit( + srd: SequenceRuntimeData, + charset: Charset, + layerLengthEv: LayerLengthEv) +extends LayerTransformer() { + Assert.invariant(charset.newDecoder().maxCharsPerByte() == 1) // is a SBCS charset + + private val checkDigitVRD = { + val varNamespace = NS("urn:org.apache.daffodil.layers.checkDigit") + val varQName = RefQName(Some("cd"), "checkDigit", varNamespace).toGlobalQName + val vrd = srd.variableMap.getVariableRuntimeData(varQName).getOrElse { + srd.SDE("Variable '%s' is not defined.", varQName.toExtendedSyntax) + } + srd.schemaDefinitionUnless(vrd.primType == PrimType.UnsignedInt, + "Variable '%s' is not of type 'xs:unsignedInt'.", varQName.toExtendedSyntax) + vrd + } + + private var limitingOutputStream : ByteBufferOutputStream = _ + private var originalOutputStream: OutputStream = _ + + protected def wrapLayerDecoder(jis: InputStream) = jis + + override def wrapLimitingStream(jis: java.io.InputStream, state: PState) = { + val layerLengthInBytes: Int = layerLengthEv.evaluate(state).toInt + val ba = new Array[Byte](layerLengthInBytes) + try + IOUtils.readFully(jis, ba) + catch { + case eof: EOFException => + throw DataInputStream.NotEnoughDataException(ba.length * 8) + } + // this stream will be used by the parse when it parses the + // schema contents of the layer. We have already consumed the bytes + // from the original input stream, and saved those in the byte array + // to enable the computation of the checkDigit. + val layerStream = new ByteArrayInputStream(ba) + val str = new String(ba, charset) // always does replacement on decode errors. + val checkDigit = computeCheckDigit(str) + state.setVariable(checkDigitVRD, checkDigit, srd) // assign to result variable + layerStream + } + + protected def wrapLayerEncoder(jos: OutputStream) = jos + + protected def wrapLimitingStream(jos: OutputStream, state: UState) = { + originalOutputStream = jos + val layerLengthInBytes: Int = layerLengthEv.evaluate(state).toInt + val ba = new Array[Byte](layerLengthInBytes) + val byteBuf = ByteBuffer.wrap(ba) // bigEndian byte order by default + limitingOutputStream = new ByteBufferOutputStream(byteBuf) + limitingOutputStream + } + + /** + * Shared by both parsing and unparsing. + * + * Ignores any non-digit character in the argument. + * + * The checkDigit is the total of all digits, viewed as a string, the last digit of that total. + */ + private def computeCheckDigit(s: String): Long = { Review comment: Oh, that's right, an uint32 won't fit in an Int so it needs to be returned as a Long (or passera.unsigned.UInt). ########## File path: daffodil-test/src/test/resources/org/apache/daffodil/layers/xsd/IPv4ChecksumLayer.dfdl.xsd ########## @@ -0,0 +1,51 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema"; + xmlns:dfdl="http://www.ogf.org/dfdl/dfdl-1.0/"; + xmlns:dfdlx="http://www.ogf.org/dfdl/dfdl-1.0/extensions"; + xmlns:fn="http://www.w3.org/2005/xpath-functions"; + xmlns:daf="urn:ogf:dfdl:2013:imp:daffodil.apache.org:2018:ext" + xmlns:tns="urn:org.apache.daffodil.layers.IPv4Checksum" + targetNamespace="urn:org.apache.daffodil.layers.IPv4Checksum"> + + <xs:annotation> + <xs:documentation> + Variable for the layer transform used for IPv4 checksum calculation. + + Per the checksum algorithm described in IETF RFC791. + + The data has a well-known fixed layer length. Hence, layerLengthKind is 'implicit'. + + If the data doesn't match this expected fixed length, issue PE/UE. + + This checksum is written into the result variable. + </xs:documentation> + <xs:appinfo source="http://www.ogf.org/dfdl/";> + <dfdl:defineVariable name="IPv4Checksum" type="xs:unsignedShort"/> + <dfdl:defineFormat name="IPv4ChecksumLayer"> + <dfdl:format dfdlx:layerTransform='{ "IPv4Checksum" }' dfdlx:layerLengthKind="implicit"/> + </dfdl:defineFormat> + </xs:appinfo> + </xs:annotation> + + <xs:simpleType name="IPv4Checksum" + dfdl:lengthKind="explicit" dfdl:length="16" dfdl:lengthUnits="bits"> + <xs:restriction base="xs:unsignedShort"/> Review comment: For curiosity, what happens if you define or import a simple type restricting `base="xs:unsignedShort"` with `dfdl:length="16"` and `dfdl:lengthUnits="bits"` in a text schema with `dfdl:representation="text"`? Does Daffodil automatically override the text representation when parsing/unparsing that type? ########## File path: daffodil-test/src/test/resources/org/apache/daffodil/layers/IPv4.dfdl.xsd ########## @@ -0,0 +1,199 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<!-- +Copyright (c) 2012-2021 Owl Cyber Defense. All rights reserved. + +Developed by: Owl Cyber Defense + http://www.owlcyberdefense.com + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal with +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimers. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimers in the + documentation and/or other materials provided with the distribution. + + 3. Neither the names of Owl Cyber Defense, nor the names of its contributors + may be used to endorse or promote products derived from this Software + without specific prior written permission. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE +SOFTWARE. +--> + +<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema"; + xmlns:dfdl="http://www.ogf.org/dfdl/dfdl-1.0/"; + xmlns:dfdlx="http://www.ogf.org/dfdl/dfdl-1.0/extensions"; + xmlns:fn="http://www.w3.org/2005/xpath-functions"; + xmlns:daf="urn:ogf:dfdl:2013:imp:daffodil.apache.org:2018:ext" + xmlns:chksum="urn:org.apache.daffodil.layers.IPv4Checksum" + xmlns:eth="urn:ethernet" + xmlns:tns="urn:ethernet" + targetNamespace="urn:ethernet"> + + <xs:include schemaLocation="org/apache/daffodil/xsd/DFDLGeneralFormat.dfdl.xsd"/> + + <xs:import namespace="urn:org.apache.daffodil.layers.IPv4Checksum" + schemaLocation="org/apache/daffodil/layers/xsd/IPv4ChecksumLayer.dfdl.xsd"/> + + <xs:annotation> + <xs:appinfo source="http://www.ogf.org/dfdl/";> + <dfdl:defineFormat name="basicBigEndianBinary"> + <dfdl:format + ref="tns:GeneralFormat" + representation="binary" + binaryNumberRep="binary" + byteOrder="bigEndian" + bitOrder="mostSignificantBitFirst" + alignmentUnits="bits" + alignment="1" + occursCountKind="implicit"/> Review comment: You will need to add `prefixIncludesPrefixLength="no"` here as well if you ever want to use prefixed lengths. For some reason, DFDLGeneralFormat.dfdl.xsd doesn't define any default value for `prefixIncludesPrefixLength`. FYI, you could remove every other property except `representation="binary"` and `alignmentUnits="bits"` (also add `lengthUnits="bits"` since it defaults to "bytes") and still end up with the same basic binary format since DFDLGeneralFormat.dfdl.xsd defines the same values for the rest of the properties. It's good for readability to know what values the other properties have, though. ########## File path: daffodil-test/src/test/resources/org/apache/daffodil/layers/IPv4.dfdl.xsd ########## @@ -0,0 +1,199 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<!-- +Copyright (c) 2012-2021 Owl Cyber Defense. All rights reserved. + +Developed by: Owl Cyber Defense + http://www.owlcyberdefense.com + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal with +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimers. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimers in the + documentation and/or other materials provided with the distribution. + + 3. Neither the names of Owl Cyber Defense, nor the names of its contributors + may be used to endorse or promote products derived from this Software + without specific prior written permission. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE +SOFTWARE. +--> + +<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema"; + xmlns:dfdl="http://www.ogf.org/dfdl/dfdl-1.0/"; + xmlns:dfdlx="http://www.ogf.org/dfdl/dfdl-1.0/extensions"; + xmlns:fn="http://www.w3.org/2005/xpath-functions"; + xmlns:daf="urn:ogf:dfdl:2013:imp:daffodil.apache.org:2018:ext" + xmlns:chksum="urn:org.apache.daffodil.layers.IPv4Checksum" + xmlns:eth="urn:ethernet" + xmlns:tns="urn:ethernet" + targetNamespace="urn:ethernet"> + + <xs:include schemaLocation="org/apache/daffodil/xsd/DFDLGeneralFormat.dfdl.xsd"/> + + <xs:import namespace="urn:org.apache.daffodil.layers.IPv4Checksum" + schemaLocation="org/apache/daffodil/layers/xsd/IPv4ChecksumLayer.dfdl.xsd"/> Review comment: Shouldn't both IP4v.dfdl.xsd and IPv4ChecksumLayer.dfdl.xsd have more similar namespaces? Both namespaces could be short like "urn:IPv4" and "urn:IPv4Checksum" or both could be long like "urn:org.apache.daffodil.layers.IPv4" and "urn:org.apache.daffodil.layers.IPv4Checksum". I think the long names would be a better choice since the long names will tell people where to find the schemas. ########## File path: daffodil-test/src/test/resources/org/apache/daffodil/layers/IPv4.dfdl.xsd ########## @@ -0,0 +1,199 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<!-- +Copyright (c) 2012-2021 Owl Cyber Defense. All rights reserved. + +Developed by: Owl Cyber Defense + http://www.owlcyberdefense.com + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal with +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimers. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimers in the + documentation and/or other materials provided with the distribution. + + 3. Neither the names of Owl Cyber Defense, nor the names of its contributors + may be used to endorse or promote products derived from this Software + without specific prior written permission. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE +SOFTWARE. +--> + +<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema"; + xmlns:dfdl="http://www.ogf.org/dfdl/dfdl-1.0/"; + xmlns:dfdlx="http://www.ogf.org/dfdl/dfdl-1.0/extensions"; + xmlns:fn="http://www.w3.org/2005/xpath-functions"; + xmlns:daf="urn:ogf:dfdl:2013:imp:daffodil.apache.org:2018:ext" + xmlns:chksum="urn:org.apache.daffodil.layers.IPv4Checksum" + xmlns:eth="urn:ethernet" + xmlns:tns="urn:ethernet" + targetNamespace="urn:ethernet"> + + <xs:include schemaLocation="org/apache/daffodil/xsd/DFDLGeneralFormat.dfdl.xsd"/> + + <xs:import namespace="urn:org.apache.daffodil.layers.IPv4Checksum" + schemaLocation="org/apache/daffodil/layers/xsd/IPv4ChecksumLayer.dfdl.xsd"/> + + <xs:annotation> + <xs:appinfo source="http://www.ogf.org/dfdl/";> + <dfdl:defineFormat name="basicBigEndianBinary"> + <dfdl:format + ref="tns:GeneralFormat" + representation="binary" + binaryNumberRep="binary" + byteOrder="bigEndian" + bitOrder="mostSignificantBitFirst" + alignmentUnits="bits" + alignment="1" + occursCountKind="implicit"/> + </dfdl:defineFormat> + <dfdl:format ref="tns:basicBigEndianBinary"/> + </xs:appinfo> + </xs:annotation> + + <xs:element name="IPv4Header" type="tns:IPv4Header"/> + + <xs:complexType name="IPv4Header"> + <!-- + Modified with proposed checksum computation via layer transform + --> + <xs:sequence> + <xs:annotation> + <xs:appinfo source="http://www.ogf.org/dfdl/";> + <!-- + The checksum field in an IPv4 actually lives in the middle of the + data that it is a checksum of. + + That is, some of the 16-bit words going into the checksum are before + the checksum itself, others are after. + + At parse time we must check that this computed checksum value, and the + actual value of the element in the infoset are equal. + + We have an option to place the computed checksum in another element + and then have a schematron validation rule or just a DFDL assert (recoverable) + check that the checksum element and computed checksum are equal. + + Below, that element is called "ComputedChecksum". + + When unparsing, the infoset contains a checksum, which remains in the infoset, but + it is overwritten (in the layer transform) by the recomputed checksum available for comparison purposes in + the ComputedChecksum variable. + --> + <dfdl:newVariableInstance ref="chksum:IPv4Checksum"/> + </xs:appinfo> + </xs:annotation> + + <xs:sequence dfdl:ref="chksum:IPv4ChecksumLayer"> + <xs:sequence> + <xs:element name="Version" type="tns:bit" dfdl:length="4"/> + <xs:element name="IHL" type="tns:bit" dfdl:length="4"/> + <xs:element name="DSCP" type="tns:bit" dfdl:length="6"/> + <xs:element name="ECN" type="tns:bit" dfdl:length="2"/> + <xs:element name="Length" type="tns:bit" dfdl:length="16" + dfdl:outputValueCalc='{ ../RealLength }'/> + <xs:element name="Identification" type="tns:bit" dfdl:length="16"/> + <xs:element name="Flags" type="tns:bit" dfdl:length="3"/> + <xs:element name="FragmentOffset" type="tns:bit" dfdl:length="13"/> + <xs:element name="TTL" type="tns:bit" dfdl:length="8"/> + <xs:element name="Protocol" type="tns:bit" dfdl:length="8"/> + <xs:element name="Checksum" type="chksum:IPv4Checksum"/> + <xs:element name="IPSrc" type="tns:hexByte" dfdl:length="4"/> + <xs:element name="IPDest" type="tns:hexByte" dfdl:length="4"/> + </xs:sequence> + </xs:sequence> + <!-- + This exists purely to make unparsing of the header suspend when trying to unparse + the Length element. + + We just want the tests to rule out deadlock in this case. + --> + <xs:element name="RealLength" type="tns:bit" + dfdl:inputValueCalc='{ ../Length }'/> + <!-- + We want the schema author to have all options on what to do if the + checksum is incorrect when parsing. Hence, we just put the computed value + into this element. + --> + <xs:element name="ComputedChecksum" type="chksum:IPv4Checksum" + dfdl:inputValueCalc='{ $chksum:IPv4Checksum }'/> + <!-- + One recommendation is to treat incorrect checksum values as a validation + error. This preserves the ability to use the schema forensically to examine + data with incorrect checksums. + + This uses a DFDL recoverable error assertion to report that the + checksum is incorrect. A recoverable error assert is not technically + a validation error, but behaves similarly in that it does not prevent the + parse from completing. It is essentially a warning about the data. + + A schematron check would accomplish much the same thing, with the + advantage that the error would be reported as an "official" validation error. + --> + <xs:sequence> + <xs:annotation> + <xs:appinfo source="http://www.ogf.org/dfdl/";> + <dfdl:assert + failureType="recoverableError" + message="Incorrect checksum." + test='{ Checksum eq ComputedChecksum }' + /> + </xs:appinfo> + </xs:annotation> + </xs:sequence> + </xs:sequence> + </xs:complexType> + + <xs:simpleType name="bit" + dfdl:lengthKind="explicit" + dfdl:lengthUnits="bits"> + <xs:restriction base="xs:unsignedInt"/> + </xs:simpleType> + + <xs:simpleType name="hexByte" dfdl:lengthKind="explicit" dfdl:lengthUnits="bytes"> + <xs:restriction base="xs:hexBinary"/> + </xs:simpleType> + + <xs:simpleType name="uint16" dfdl:lengthKind="explicit" dfdl:length="16"> Review comment: It's possible that your format still may be defining `dfdl:lengthUnits="bytes"` at this point which means your `uint16` and `uint32` types may be 16/32 bytes long, not 16/32 bits long. For completeness and safety, you should have `lengthUnits="bits"` in your basicBigEndianBinary definition above. Or consider removing all the types except `bits` and `hexByte` since those are the only types your schema is using right now anyway. ########## File path: daffodil-test/src/test/resources/org/apache/daffodil/layers/IPv4.dfdl.xsd ########## @@ -0,0 +1,199 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<!-- +Copyright (c) 2012-2021 Owl Cyber Defense. All rights reserved. + +Developed by: Owl Cyber Defense + http://www.owlcyberdefense.com + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal with +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimers. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimers in the + documentation and/or other materials provided with the distribution. + + 3. Neither the names of Owl Cyber Defense, nor the names of its contributors + may be used to endorse or promote products derived from this Software + without specific prior written permission. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE +SOFTWARE. +--> + +<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema"; + xmlns:dfdl="http://www.ogf.org/dfdl/dfdl-1.0/"; + xmlns:dfdlx="http://www.ogf.org/dfdl/dfdl-1.0/extensions"; + xmlns:fn="http://www.w3.org/2005/xpath-functions"; + xmlns:daf="urn:ogf:dfdl:2013:imp:daffodil.apache.org:2018:ext" + xmlns:chksum="urn:org.apache.daffodil.layers.IPv4Checksum" + xmlns:eth="urn:ethernet" + xmlns:tns="urn:ethernet" + targetNamespace="urn:ethernet"> + + <xs:include schemaLocation="org/apache/daffodil/xsd/DFDLGeneralFormat.dfdl.xsd"/> + + <xs:import namespace="urn:org.apache.daffodil.layers.IPv4Checksum" + schemaLocation="org/apache/daffodil/layers/xsd/IPv4ChecksumLayer.dfdl.xsd"/> + + <xs:annotation> + <xs:appinfo source="http://www.ogf.org/dfdl/";> + <dfdl:defineFormat name="basicBigEndianBinary"> + <dfdl:format + ref="tns:GeneralFormat" + representation="binary" + binaryNumberRep="binary" + byteOrder="bigEndian" + bitOrder="mostSignificantBitFirst" + alignmentUnits="bits" + alignment="1" + occursCountKind="implicit"/> + </dfdl:defineFormat> + <dfdl:format ref="tns:basicBigEndianBinary"/> + </xs:appinfo> + </xs:annotation> + + <xs:element name="IPv4Header" type="tns:IPv4Header"/> + + <xs:complexType name="IPv4Header"> + <!-- + Modified with proposed checksum computation via layer transform + --> + <xs:sequence> + <xs:annotation> + <xs:appinfo source="http://www.ogf.org/dfdl/";> + <!-- + The checksum field in an IPv4 actually lives in the middle of the + data that it is a checksum of. + + That is, some of the 16-bit words going into the checksum are before + the checksum itself, others are after. + + At parse time we must check that this computed checksum value, and the + actual value of the element in the infoset are equal. + + We have an option to place the computed checksum in another element + and then have a schematron validation rule or just a DFDL assert (recoverable) + check that the checksum element and computed checksum are equal. + + Below, that element is called "ComputedChecksum". + + When unparsing, the infoset contains a checksum, which remains in the infoset, but + it is overwritten (in the layer transform) by the recomputed checksum available for comparison purposes in + the ComputedChecksum variable. + --> + <dfdl:newVariableInstance ref="chksum:IPv4Checksum"/> + </xs:appinfo> + </xs:annotation> + + <xs:sequence dfdl:ref="chksum:IPv4ChecksumLayer"> + <xs:sequence> + <xs:element name="Version" type="tns:bit" dfdl:length="4"/> + <xs:element name="IHL" type="tns:bit" dfdl:length="4"/> + <xs:element name="DSCP" type="tns:bit" dfdl:length="6"/> + <xs:element name="ECN" type="tns:bit" dfdl:length="2"/> + <xs:element name="Length" type="tns:bit" dfdl:length="16" + dfdl:outputValueCalc='{ ../RealLength }'/> + <xs:element name="Identification" type="tns:bit" dfdl:length="16"/> + <xs:element name="Flags" type="tns:bit" dfdl:length="3"/> + <xs:element name="FragmentOffset" type="tns:bit" dfdl:length="13"/> + <xs:element name="TTL" type="tns:bit" dfdl:length="8"/> + <xs:element name="Protocol" type="tns:bit" dfdl:length="8"/> + <xs:element name="Checksum" type="chksum:IPv4Checksum"/> + <xs:element name="IPSrc" type="tns:hexByte" dfdl:length="4"/> + <xs:element name="IPDest" type="tns:hexByte" dfdl:length="4"/> + </xs:sequence> + </xs:sequence> + <!-- + This exists purely to make unparsing of the header suspend when trying to unparse + the Length element. + + We just want the tests to rule out deadlock in this case. + --> + <xs:element name="RealLength" type="tns:bit" + dfdl:inputValueCalc='{ ../Length }'/> + <!-- + We want the schema author to have all options on what to do if the + checksum is incorrect when parsing. Hence, we just put the computed value + into this element. + --> + <xs:element name="ComputedChecksum" type="chksum:IPv4Checksum" + dfdl:inputValueCalc='{ $chksum:IPv4Checksum }'/> + <!-- + One recommendation is to treat incorrect checksum values as a validation + error. This preserves the ability to use the schema forensically to examine + data with incorrect checksums. + + This uses a DFDL recoverable error assertion to report that the + checksum is incorrect. A recoverable error assert is not technically + a validation error, but behaves similarly in that it does not prevent the + parse from completing. It is essentially a warning about the data. + + A schematron check would accomplish much the same thing, with the + advantage that the error would be reported as an "official" validation error. + --> + <xs:sequence> + <xs:annotation> + <xs:appinfo source="http://www.ogf.org/dfdl/";> + <dfdl:assert + failureType="recoverableError" + message="Incorrect checksum." + test='{ Checksum eq ComputedChecksum }' + /> + </xs:appinfo> + </xs:annotation> + </xs:sequence> + </xs:sequence> + </xs:complexType> + + <xs:simpleType name="bit" + dfdl:lengthKind="explicit" + dfdl:lengthUnits="bits"> + <xs:restriction base="xs:unsignedInt"/> + </xs:simpleType> + + <xs:simpleType name="hexByte" dfdl:lengthKind="explicit" dfdl:lengthUnits="bytes"> + <xs:restriction base="xs:hexBinary"/> + </xs:simpleType> + + <xs:simpleType name="uint16" dfdl:lengthKind="explicit" dfdl:length="16"> + <xs:restriction base="xs:unsignedInt"/> + </xs:simpleType> + + <xs:simpleType name="uint32" dfdl:lengthKind="explicit" dfdl:length="32"> + <xs:restriction base="xs:unsignedInt"/> + </xs:simpleType> + + <xs:simpleType name="int32" dfdl:lengthKind="explicit" dfdl:length="4" dfdl:lengthUnits="bytes"> Review comment: I would expect to read 32 bits, not 4 bytes, in an `int32` definition, but you've made this definition (almost) explicit enough to be portable to any schema. Actually, the funny thing is that the base type `xs:int` is also 32 bits long so the only restrictions you really need are `dfdl:lengthKind="explicit"` and `dfdl:representation="binary"`. ########## File path: daffodil-test/src/test/resources/org/apache/daffodil/layers/IPv4.tdml ########## @@ -0,0 +1,202 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<tdml:testSuite + xmlns:tdml="http://www.ibm.com/xmlns/dfdl/testData"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xmlns:dfdl="http://www.ogf.org/dfdl/dfdl-1.0/"; + xmlns:xs="http://www.w3.org/2001/XMLSchema"; + xmlns:fn="http://www.w3.org/2005/xpath-functions"; + xmlns:dfdlx="http://www.ogf.org/dfdl/dfdl-1.0/extensions"; + xmlns:eth="urn:ethernet" + xmlns:chksum="urn:com.owlcyberdefense.IPv4Checksum" Review comment: I saw `"urn:org.apache.daffodil.layers.IPv4Checksum"` in the previous file. It would be nice if both files were consistent with each other. Also note you may want to replace `urn:ethernet` too. ########## File path: daffodil-test/src/test/scala/org/apache/daffodil/layers/IPv4Checksum.scala ########## @@ -0,0 +1,239 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.daffodil.layers + +import org.apache.commons.io.IOUtils +import org.apache.daffodil.schema.annotation.props.gen.LayerLengthKind +import org.apache.daffodil.schema.annotation.props.gen.LayerLengthUnits +import org.apache.daffodil.util.Maybe +import org.apache.daffodil.processors.LayerLengthEv +import org.apache.daffodil.processors.LayerBoundaryMarkEv +import org.apache.daffodil.processors.LayerCharsetEv +import org.apache.daffodil.processors.parsers.PState +import org.apache.daffodil.processors.unparsers.UState + +import java.io._ +import org.apache.daffodil.dpath.NodeInfo.PrimType +import org.apache.daffodil.exceptions.Assert +import org.apache.daffodil.io.DataInputStream +import org.apache.daffodil.processors.SequenceRuntimeData +import org.apache.daffodil.processors.SuspendableOperation +import org.apache.daffodil.util.ByteBufferOutputStream +import org.apache.daffodil.util.Maybe.One +import org.apache.daffodil.xml.NS +import org.apache.daffodil.xml.RefQName +import passera.unsigned.UShort + +import java.nio.ByteBuffer +import java.nio.ShortBuffer + +/** + * The layer transform computes the checksum of the header data. + * per IETF RFC 791. + * + * The data has a well-known fixed length, so layerLengthKind is always 'implicit'. + */ +final class IPv4Checksum(srd: SequenceRuntimeData) +extends LayerTransformer() { + + private def hdrLengthInBytes = 20 + + private def chksumShortIndex = 5 + + private val finalChecksumVRD = { + val varNamespace = NS("urn:org.apache.daffodil.layers.IPv4Checksum") + val finalChecksumVarQName = RefQName(Some("chksum"), "IPv4Checksum", varNamespace).toGlobalQName + val vrd = srd.variableMap.getVariableRuntimeData(finalChecksumVarQName).getOrElse { + srd.SDE("Variable '%s' is not defined.", finalChecksumVarQName.toExtendedSyntax) + } + srd.schemaDefinitionUnless(vrd.primType == PrimType.UnsignedShort, + "Variable '%s' is not of type 'xs:unsignedShort'.", finalChecksumVarQName.toExtendedSyntax) + vrd + } + + /** + * The header data will be captured here. All the limiting streams and + * the byte/short buffers are all aliases into this same object. + */ + private val hdrByteArr = new Array[Byte](hdrLengthInBytes) + + /** + * Stream from which the header will be parsed. + */ + private val limitingInputStream = new ByteArrayInputStream(hdrByteArr) + + + /** + * ByteBuffer view of the header bytes. + */ + private val byteBuf = ByteBuffer.wrap(hdrByteArr) // bigEndian byte order by default + private val shortBuf = byteBuf.asShortBuffer() + + /** + * Stream to which the header will be unparsed. + */ + private val limitingOutputStream = new ByteBufferOutputStream(byteBuf) + + /** + * Assigned by wrapLimitingStream for parsing to capture the original source + * of the header bytes for parsing. + */ + private var optOriginalInputStream: Maybe[InputStream] = Maybe.Nope + + /** + * Assigned by wrapLimitingStream for unparsing to capture the original + * output stream to which the bytes are ultimately written. + */ + private var optOriginalOutputStream: Maybe[OutputStream] = Maybe.Nope + + protected def wrapLayerDecoder(jis: InputStream) = jis + + protected def wrapLimitingStream(jis: InputStream, state: PState) = { + optOriginalInputStream = One(jis) + limitingInputStream + } + + protected def wrapLayerEncoder(jos: OutputStream) = jos + + protected def wrapLimitingStream(jos: OutputStream, state: UState) = { + optOriginalOutputStream = One(jos) + limitingOutputStream + } + + /** + * Shared by both parsing and unparsing. Contains the checksum algorithm. + */ + private def computeChecksum(shortBuf: ShortBuffer): Int = { + var i = 0 + var chksum: Int = 0 + val nShorts = hdrLengthInBytes / 2 + while (i < nShorts) { + if (i == chksumShortIndex) { + // for the checksum calculation treat the incoming checksum field of the data as 0 + // so we just don't do an addition here. + } else { + chksum += UShort(shortBuf.get(i)).toInt + } + Assert.invariant(chksum >= 0) + i += 1 + } + // + // now combine the carry bits in the most significant 16 bits into the lower 16 bits. + // + val checksumLow = chksum & 0xFFFF + val checksumHigh = chksum >>> 16 + val checksumTotal: Int = checksumLow + checksumHigh + Assert.invariant(checksumTotal <= 0xFFFF && checksumTotal >= 0) + val checksumTotalShort = UShort(checksumTotal.toShort) + val checksum = checksumTotalShort.toInt + // + // take ones complement to get the final checksum + // + val finalChecksum: Int = (~checksum) & 0xFFFF + finalChecksum + } + + override def startLayerForParse(s: PState): Unit = { + // + // For parsing, all the hard work happens here, allowing the layered input stream to + // just deliver the bytes + // + Assert.invariant(s.bitPos0b % 8 == 0) // we are byte aligned. + Assert.invariant(optOriginalInputStream.isDefined) + val bitLen = hdrLengthInBytes * 8 + + try + IOUtils.readFully(optOriginalInputStream.get, hdrByteArr) + catch { + case eof: EOFException => + throw DataInputStream.NotEnoughDataException(bitLen) + } + val checksumInDataStream = shortBuf.get() + val checksum = computeChecksum(shortBuf) + s.setVariable(finalChecksumVRD, checksum, srd) // assign to result variable. + } + + final class SuspendableChecksumLayerOperation() + extends SuspendableOperation { + override def rd = srd + + /** + * Test succeeds if all the data required has been written to the layer + */ + protected def test(ustate: UState) = { + limitingOutputStream.size() == hdrLengthInBytes + } + + /** + * Computes checksum and overwrites that part of the layer data + * with the new checksum, writes the layer data out, + * and assigns the output variable. + */ + protected def continuation(ustate: UState): Unit = { + Assert.invariant(optOriginalOutputStream.isDefined) + val finalChecksum = computeChecksum(shortBuf) + ustate.setVariable(finalChecksumVRD, finalChecksum, srd) // assign to the result variable. + // + // clobber the byte buffer bytes corresponding to the checksum with + // the recomputed value + // + shortBuf.put(chksumShortIndex, finalChecksum.toShort) + // + // write out the layer data (which has recomputed checksum in it. + optOriginalOutputStream.get.write(hdrByteArr) + optOriginalOutputStream.get.close() + } + } + + private lazy val suspendableOperation = new SuspendableChecksumLayerOperation() + + override def endLayerForUnparse(s: UState): Unit = { + suspendableOperation.run(s) + } + +} + + object IPv4Checksum + extends LayerTransformerFactory("IPv4Checksum") { + + override def newInstance( + maybeLayerCharsetEv: Maybe[LayerCharsetEv], + maybeLayerLengthKind: Maybe[LayerLengthKind], + maybeLayerLengthEv: Maybe[LayerLengthEv], + maybeLayerLengthUnits: Maybe[LayerLengthUnits], + maybeLayerBoundaryMarkEv: Maybe[LayerBoundaryMarkEv], + srd: SequenceRuntimeData) = { + srd.schemaDefinitionUnless(maybeLayerLengthKind.isEmpty || + maybeLayerLengthKind.get == LayerLengthKind.Implicit, + "Must have dfdlx:layerLengthKind undefined or 'implicit', but was '%s'.", + maybeLayerLengthKind.get.toString) + // + // ToDo: We can't issue SDW because we don't have a context object + // for that sort of compilation time warning. We *should* have that. + // That requires a layer factory method called at schema compile time with the + // Sequence term (not just term runtime data) as an argument. + // + // We would like to warn if maybeLayerLengthUnits is defined here and is not bytes + // We would like to warn if maybeLayerBoundaryMarkEv is defined (since it is unused by this layer) + // We would like to warn if maybeLayerCharsetEv is defined (since it is unused by this layer) + // We would like to warn if maybeLayerLengthEv is defined (since it is unused by this layer) Review comment: May be worth keeping these comments (or acting on some of them). ########## File path: daffodil-test/src/test/scala/org/apache/daffodil/layers/IPv4Checksum.scala ########## @@ -0,0 +1,239 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.daffodil.layers + +import org.apache.commons.io.IOUtils +import org.apache.daffodil.schema.annotation.props.gen.LayerLengthKind +import org.apache.daffodil.schema.annotation.props.gen.LayerLengthUnits +import org.apache.daffodil.util.Maybe +import org.apache.daffodil.processors.LayerLengthEv +import org.apache.daffodil.processors.LayerBoundaryMarkEv +import org.apache.daffodil.processors.LayerCharsetEv +import org.apache.daffodil.processors.parsers.PState +import org.apache.daffodil.processors.unparsers.UState + +import java.io._ +import org.apache.daffodil.dpath.NodeInfo.PrimType +import org.apache.daffodil.exceptions.Assert +import org.apache.daffodil.io.DataInputStream +import org.apache.daffodil.processors.SequenceRuntimeData +import org.apache.daffodil.processors.SuspendableOperation +import org.apache.daffodil.util.ByteBufferOutputStream +import org.apache.daffodil.util.Maybe.One +import org.apache.daffodil.xml.NS +import org.apache.daffodil.xml.RefQName +import passera.unsigned.UShort + +import java.nio.ByteBuffer +import java.nio.ShortBuffer + +/** + * The layer transform computes the checksum of the header data. + * per IETF RFC 791. + * + * The data has a well-known fixed length, so layerLengthKind is always 'implicit'. + */ +final class IPv4Checksum(srd: SequenceRuntimeData) +extends LayerTransformer() { + + private def hdrLengthInBytes = 20 + + private def chksumShortIndex = 5 + + private val finalChecksumVRD = { + val varNamespace = NS("urn:org.apache.daffodil.layers.IPv4Checksum") + val finalChecksumVarQName = RefQName(Some("chksum"), "IPv4Checksum", varNamespace).toGlobalQName + val vrd = srd.variableMap.getVariableRuntimeData(finalChecksumVarQName).getOrElse { + srd.SDE("Variable '%s' is not defined.", finalChecksumVarQName.toExtendedSyntax) + } + srd.schemaDefinitionUnless(vrd.primType == PrimType.UnsignedShort, + "Variable '%s' is not of type 'xs:unsignedShort'.", finalChecksumVarQName.toExtendedSyntax) + vrd + } + + /** + * The header data will be captured here. All the limiting streams and + * the byte/short buffers are all aliases into this same object. + */ + private val hdrByteArr = new Array[Byte](hdrLengthInBytes) + + /** + * Stream from which the header will be parsed. + */ + private val limitingInputStream = new ByteArrayInputStream(hdrByteArr) + + + /** + * ByteBuffer view of the header bytes. + */ + private val byteBuf = ByteBuffer.wrap(hdrByteArr) // bigEndian byte order by default + private val shortBuf = byteBuf.asShortBuffer() + + /** + * Stream to which the header will be unparsed. + */ + private val limitingOutputStream = new ByteBufferOutputStream(byteBuf) + + /** + * Assigned by wrapLimitingStream for parsing to capture the original source + * of the header bytes for parsing. + */ + private var optOriginalInputStream: Maybe[InputStream] = Maybe.Nope + + /** + * Assigned by wrapLimitingStream for unparsing to capture the original + * output stream to which the bytes are ultimately written. + */ + private var optOriginalOutputStream: Maybe[OutputStream] = Maybe.Nope + + protected def wrapLayerDecoder(jis: InputStream) = jis + + protected def wrapLimitingStream(jis: InputStream, state: PState) = { + optOriginalInputStream = One(jis) + limitingInputStream + } + + protected def wrapLayerEncoder(jos: OutputStream) = jos + + protected def wrapLimitingStream(jos: OutputStream, state: UState) = { + optOriginalOutputStream = One(jos) + limitingOutputStream + } + + /** + * Shared by both parsing and unparsing. Contains the checksum algorithm. + */ + private def computeChecksum(shortBuf: ShortBuffer): Int = { + var i = 0 + var chksum: Int = 0 + val nShorts = hdrLengthInBytes / 2 + while (i < nShorts) { + if (i == chksumShortIndex) { + // for the checksum calculation treat the incoming checksum field of the data as 0 + // so we just don't do an addition here. + } else { + chksum += UShort(shortBuf.get(i)).toInt + } + Assert.invariant(chksum >= 0) + i += 1 + } + // + // now combine the carry bits in the most significant 16 bits into the lower 16 bits. + // + val checksumLow = chksum & 0xFFFF + val checksumHigh = chksum >>> 16 + val checksumTotal: Int = checksumLow + checksumHigh + Assert.invariant(checksumTotal <= 0xFFFF && checksumTotal >= 0) + val checksumTotalShort = UShort(checksumTotal.toShort) + val checksum = checksumTotalShort.toInt + // + // take ones complement to get the final checksum + // + val finalChecksum: Int = (~checksum) & 0xFFFF + finalChecksum + } + + override def startLayerForParse(s: PState): Unit = { + // + // For parsing, all the hard work happens here, allowing the layered input stream to + // just deliver the bytes + // + Assert.invariant(s.bitPos0b % 8 == 0) // we are byte aligned. + Assert.invariant(optOriginalInputStream.isDefined) + val bitLen = hdrLengthInBytes * 8 + + try + IOUtils.readFully(optOriginalInputStream.get, hdrByteArr) + catch { + case eof: EOFException => + throw DataInputStream.NotEnoughDataException(bitLen) + } + val checksumInDataStream = shortBuf.get() + val checksum = computeChecksum(shortBuf) + s.setVariable(finalChecksumVRD, checksum, srd) // assign to result variable. + } + + final class SuspendableChecksumLayerOperation() + extends SuspendableOperation { + override def rd = srd + + /** + * Test succeeds if all the data required has been written to the layer + */ + protected def test(ustate: UState) = { + limitingOutputStream.size() == hdrLengthInBytes + } + + /** + * Computes checksum and overwrites that part of the layer data + * with the new checksum, writes the layer data out, + * and assigns the output variable. + */ + protected def continuation(ustate: UState): Unit = { + Assert.invariant(optOriginalOutputStream.isDefined) + val finalChecksum = computeChecksum(shortBuf) + ustate.setVariable(finalChecksumVRD, finalChecksum, srd) // assign to the result variable. + // + // clobber the byte buffer bytes corresponding to the checksum with + // the recomputed value + // + shortBuf.put(chksumShortIndex, finalChecksum.toShort) + // + // write out the layer data (which has recomputed checksum in it. + optOriginalOutputStream.get.write(hdrByteArr) + optOriginalOutputStream.get.close() + } + } + + private lazy val suspendableOperation = new SuspendableChecksumLayerOperation() + + override def endLayerForUnparse(s: UState): Unit = { + suspendableOperation.run(s) + } + +} + + object IPv4Checksum + extends LayerTransformerFactory("IPv4Checksum") { + + override def newInstance( + maybeLayerCharsetEv: Maybe[LayerCharsetEv], + maybeLayerLengthKind: Maybe[LayerLengthKind], + maybeLayerLengthEv: Maybe[LayerLengthEv], + maybeLayerLengthUnits: Maybe[LayerLengthUnits], + maybeLayerBoundaryMarkEv: Maybe[LayerBoundaryMarkEv], + srd: SequenceRuntimeData) = { + srd.schemaDefinitionUnless(maybeLayerLengthKind.isEmpty || + maybeLayerLengthKind.get == LayerLengthKind.Implicit, + "Must have dfdlx:layerLengthKind undefined or 'implicit', but was '%s'.", + maybeLayerLengthKind.get.toString) + // + // ToDo: We can't issue SDW because we don't have a context object + // for that sort of compilation time warning. We *should* have that. + // That requires a layer factory method called at schema compile time with the + // Sequence term (not just term runtime data) as an argument. Review comment: Is this comment out of date? I see SequenceRuntimeData being passed above, so it looks like you've already made that change. If so, please delete the above comment. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
