mbeckerle commented on code in PR #1170: URL: https://github.com/apache/daffodil/pull/1170#discussion_r1506787093
########## daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/layers/api/LayerCompileInfo.java: ########## @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.daffodil.runtime1.layers.api; + +import org.apache.daffodil.lib.exceptions.SchemaFileLocation; + +/** + * Provides contextual information for the layer's own static checking. + * + * This mostly is about getting access to DFDL variables. I + * It also allows reporting schema definition errors for any + * reason. + */ +public interface LayerCompileInfo { + + String layerName(); + + SchemaFileLocation schemaFileLocation(); + + /** + * Obtains the LayerVariable object that can be used to access the variable at runtime. + * Review Comment: I agree with the goal. Get rid of check() entirely (or most layers wouldn't need one). A layer has a namespace, all variables in that namespace are made available for reading and writing by the layerRuntime. The layer expresses which variables are input, which are output, and what the expected types for them are. Nothing is saved by the check() routine for re-use. ########## daffodil-runtime1-layers/src/main/scala/org/apache/daffodil/layers/runtime1/ByteSwapTransformer.scala: ########## @@ -22,53 +22,27 @@ import java.io.OutputStream import java.util.ArrayDeque import java.util.Deque -import org.apache.daffodil.io.ExplicitLengthLimitingStream import org.apache.daffodil.lib.exceptions.Assert -import org.apache.daffodil.lib.schema.annotation.props.gen.LayerLengthKind -import org.apache.daffodil.runtime1.layers._ -import org.apache.daffodil.runtime1.processors.ParseOrUnparseState - -final class FourByteSwapLayerCompiler extends LayerCompiler("fourbyteswap") { - - override def compileLayer(layerCompileInfo: LayerCompileInfo): LayerTransformerFactory = { - - layerCompileInfo.optLayerLengthKind match { - case Some(LayerLengthKind.Explicit) => // ok - case None => - layerCompileInfo.SDE( - "The property dfdlx:layerLengthKind must be defined and have value 'explicit'.", - ) - case Some(other) => - layerCompileInfo.SDE( - "The property dfdlx:layerLengthKind must be 'explicit' but was '$other'.", - ) - } +import org.apache.daffodil.runtime1.layers.api.Layer +import org.apache.daffodil.runtime1.layers.api.LayerRuntime - layerCompileInfo.SDEUnless( - layerCompileInfo.optLayerLengthOptConstantValue.isDefined, - "The property dfdlx:layerLength must be defined.", - ) +final class TwoByteSwapLayer extends ByteSwap("twobyteswap", 2) +final class FourByteSwapLayer extends ByteSwap("fourbyteswap", 4) Review Comment: No, I don't want to test it for "3" or "5" or "1000". ########## daffodil-io/src/main/resources/META-INF/services/org.apache.daffodil.io.processors.charset.BitsCharsetDefinition: ########## @@ -24,7 +24,6 @@ org.apache.daffodil.io.processors.charset.BitsCharset5BitPackedLSBFDefinition org.apache.daffodil.io.processors.charset.BitsCharset6BitDFI264DUI001Definition org.apache.daffodil.io.processors.charset.BitsCharset6BitDFI311DUI002Definition org.apache.daffodil.io.processors.charset.BitsCharset6BitICAOAircraftIDDefinition -org.apache.daffodil.io.processors.charset.BitsCharsetAISPayloadArmoringDefinition Review Comment: Moved it down into the definition of the AIS payload armoring stuff - it's special purpose for just that. All that stuff is in daffodil-test. Really it should get pulled out of daffodil and an AIS schema project started. ########## daffodil-runtime1-layers/src/main/scala/org/apache/daffodil/layers/runtime1/FixedLengthLayer.scala: ########## @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.daffodil.layers.runtime1 + +import java.io.ByteArrayInputStream +import java.io.ByteArrayOutputStream +import java.io.InputStream +import java.io.OutputStream +import java.nio.ByteBuffer + +import org.apache.daffodil.lib.exceptions.Assert +import org.apache.daffodil.runtime1.api.DFDLPrimType +import org.apache.daffodil.runtime1.layers.api.Layer +import org.apache.daffodil.runtime1.layers.api.LayerCompileInfo +import org.apache.daffodil.runtime1.layers.api.LayerRuntime +import org.apache.daffodil.runtime1.layers.api.LayerVariable + +import org.apache.commons.io.IOUtils + +/** + * Suitable only for checksums computed over small sections of data, not large data streams or whole files. + * + * The entire region of data the checksum is being computed over, will be pulled into a byte buffer in memory. + */ +final class FixedLengthLayer extends Layer("fixedLength") { + + var layerLengthVar: LayerVariable = _ + + private def init(lci: LayerCompileInfo): Unit = { + if (layerLengthVar == null) { + layerLengthVar = lci.layerVariable("urn:org.apache.daffodil.layers.fixedLength", name()) + if (layerLengthVar.dfdlPrimType != DFDLPrimType.UnsignedInt) + lci.schemaDefinitionError( + "Layer variable 'fixedLength' is not an 'xs:unsignedInt'.", + ) + } + } + + override def check(lci: LayerCompileInfo): Unit = init(lci) + + override def wrapLayerDecoder(jis: InputStream, lr: LayerRuntime): InputStream = { + init(lr) + new FixedLengthInputStream(this, jis, lr) + } + + override def wrapLayerEncoder(jos: OutputStream, lr: LayerRuntime): OutputStream = { + init(lr) + new FixedLengthOutputStream(this, jos, lr) + } +} + +class FixedLengthInputStream( + layer: FixedLengthLayer, + jis: InputStream, + lr: LayerRuntime, +) extends InputStream { + + private val layerLength = { + val res = lr.getLong(layer.layerLengthVar) + Assert.invariant(res > 0) + res + } + + private lazy val bais = { + val ba = new Array[Byte](layerLength.toInt) + val nRead = IOUtils.read(jis, ba) Review Comment: Adding processing errors if the data is too short or too long for both the read/write cases. ########## daffodil-lib/src/main/scala/org/apache/daffodil/lib/schema/annotation/props/ByHandMixins.scala: ########## @@ -394,8 +394,8 @@ object BinaryBooleanTrueRepType { if (i < 0) element.schemaDefinitionError( - "For property 'binaryBooleanFalseRep', value must be an empty string or a non-negative integer. Found: %d", - i, + "For property 'binaryBooleanFalseRep', value must be an empty string or a non-negative integer. Found: %s", + i.toString, Review Comment: Correct. It doesn't autobox the int. Strange but true. ########## daffodil-propgen/src/main/resources/org/apache/daffodil/xsd/dfdlx.xsd: ########## @@ -123,106 +123,10 @@ <xs:attributeGroup name="ExtLayeringAGQualified"> <xs:attribute form="qualified" name="layerTransform" type="dfdlx:LayerTransformType" /> - <xs:attribute form="qualified" name="layerEncoding" type="dfdl:EncodingEnum_Or_DFDLExpression" /> - <xs:attribute form="qualified" name="layerLengthKind" type="dfdlx:LayerLengthKindEnum" /> - <xs:attribute form="qualified" name="layerLength" type="dfdl:DFDLNonNegativeInteger_Or_DFDLExpression" /> - <xs:attribute form="qualified" name="layerLengthUnits" type="dfdlx:LayerLengthUnitsEnum" /> - <xs:attribute form="qualified" name="layerBoundaryMark" type="dfdl:ListOfDFDLStringLiteral_Or_DFDLExpression" /> </xs:attributeGroup> - <xs:simpleType name="LayerTransformType"> - <xs:union> - <xs:simpleType> - <xs:restriction base="dfdlx:LayerTransformEnum" /> - </xs:simpleType> - <xs:simpleType> - <xs:restriction base="xs:NCName" /><!-- for dynamically loaded layers --> - </xs:simpleType> - </xs:union> - </xs:simpleType> - - <xs:simpleType name="LayerTransformEnum"> - <xs:restriction base="xs:token"> - <xs:enumeration value="fourbyteswap"> - <xs:annotation> - <xs:documentation> - Swap bytes in 32 bit words. - </xs:documentation> - </xs:annotation> - </xs:enumeration> - <xs:enumeration value="base64_MIME"> - <xs:annotation> - <xs:documentation> - IETF RFC 2045 - - Max line length 76 characters. - </xs:documentation> - </xs:annotation> - </xs:enumeration> - <xs:enumeration value="lineFolded_IMF"> - <xs:annotation> - <xs:documentation><![CDATA[ - IETF RFC 2822 Internet Message Format (IMF) - - Each line of characters MUST be no more than - 998 characters, and SHOULD be no more than 78 characters, excluding - the CRLF. - - Though structured field bodies are defined in such a way that - folding can take place between many of the lexical tokens (and even - within some of the lexical tokens), folding SHOULD be limited to - placing the CRLF at higher-level syntactic breaks. For instance, if - a field body is defined as comma-separated values, it is recommended - that folding occur after the comma separating the structured items in - preference to other places where the field could be folded, even if - it is allowed elsewhere. - - Unfolding is accomplished by simply removing any CRLF - that is immediately followed by WSP. - ]]> - </xs:documentation> - </xs:annotation> - </xs:enumeration> - <xs:enumeration value="lineFolded_iCalendar"> - <xs:annotation> - <xs:documentation><![CDATA[ - IETF RFC 5545 Internet Calendaring and Scheduling (iCalendar) - - Lines of text SHOULD NOT be longer than 75 octets, excluding the line - break. Long content lines SHOULD be split into a multiple line - representations using a line "folding" technique. That is, a long - line can be split between any two characters by inserting a CRLF - immediately followed by a single linear white-space character (i.e., - SPACE or HTAB). Any sequence of CRLF followed immediately by a - single linear white-space character is ignored (i.e., removed) when - processing the content type. - ]]> - </xs:documentation> - </xs:annotation> - </xs:enumeration> - <xs:enumeration value="gzip"> - <xs:annotation> - <xs:documentation> - GZIP per https://www.ietf.org/rfc/rfc1952.txt - </xs:documentation> - </xs:annotation> - </xs:enumeration> - </xs:restriction> - </xs:simpleType> - - <xs:simpleType name="LayerLengthUnitsEnum"> - <xs:restriction base="xs:string"> - <xs:enumeration value="bytes" /> - </xs:restriction> - </xs:simpleType> - - <xs:simpleType name="LayerLengthKindEnum"> - <xs:restriction base="xs:string"> - <xs:enumeration value="explicit" /> - <xs:enumeration value="boundaryMark" /> - <xs:enumeration value="implicit" /> - </xs:restriction> + <xs:restriction base="xs:NCName" /> Review Comment: Good suggestion. This stuff is out of hand with deep nesting due to the requirement that a layer have exactly one child (which we can also lift) ########## daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/layers/LayerRuntimeImpl.scala: ########## @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.daffodil.runtime1.layers + +import java.nio.charset.Charset + +import org.apache.daffodil.lib.exceptions.SchemaFileLocation +import org.apache.daffodil.lib.util.Maybe +import org.apache.daffodil.runtime1.infoset.DataValue +import org.apache.daffodil.runtime1.layers.api.LayerRuntime +import org.apache.daffodil.runtime1.layers.api.LayerVariable +import org.apache.daffodil.runtime1.processors.ParseOrUnparseState +import org.apache.daffodil.runtime1.processors.SequenceRuntimeData +import org.apache.daffodil.runtime1.processors.VariableRuntimeData +import org.apache.daffodil.runtime1.processors.parsers.PState +import org.apache.daffodil.runtime1.processors.parsers.ParseError +import org.apache.daffodil.runtime1.processors.unparsers.UState +import org.apache.daffodil.runtime1.processors.unparsers.UnparseError + +class LayerRuntimeImpl(val state: ParseOrUnparseState, val srd: SequenceRuntimeData) + extends LayerRuntime { + + override def processingError(cause: Throwable): Nothing = + processingError(Maybe(cause), Maybe.Nope) + + override def processingError(msg: String): Nothing = + processingError(Maybe.Nope, Maybe(msg)) + + private def processingError(mCause: Maybe[Throwable], mMsg: Maybe[String]): Nothing = { + val diagnostic = state match { + case ps: PState => + new ParseError( + rd = Maybe(srd.schemaFileLocation), + loc = Maybe(state.currentLocation), + causedBy = mCause, + kind = mMsg, + ) + case us: UState => + new UnparseError( + rd = Maybe(srd.schemaFileLocation), + loc = Maybe(state.currentLocation), + causedBy = mCause, + kind = mMsg, + ) + } + throw diagnostic + } + + override def runtimeSchemaDefinitionError(msg: String, args: AnyRef*): Nothing = + state.SDE(msg, args: _*) + + override def runtimeSchemaDefinitionError(cause: Throwable): Nothing = + state.SDE(cause) + + override def getString(variable: LayerVariable): String = + state + .getVariable(variable.asInstanceOf[VariableRuntimeData], srd) + .getString + + override def setString(variable: LayerVariable, s: String): Unit = + state.setVariable( + variable.asInstanceOf[VariableRuntimeData], + DataValue.toDataValue(s), + srd, + ) + + override def getInt(variable: LayerVariable): Int = + state + .getVariable(variable.asInstanceOf[VariableRuntimeData], srd) + .getInt + + override def setInt(variable: LayerVariable, v: Int): Unit = + state.setVariable( + variable.asInstanceOf[VariableRuntimeData], + DataValue.toDataValue(v), + srd, + ) + + override def schemaDefinitionError(msg: String, args: AnyRef*): Unit = + runtimeSchemaDefinitionError(msg, args) + + override def getCharset(layerEncoding: String): Charset = Charset.forName(layerEncoding) Review Comment: Right. We could have a getBitsCharset() if that was needed. For now,just removing. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
