stevedlawrence commented on code in PR #1176:
URL: https://github.com/apache/daffodil/pull/1176#discussion_r1511326669
##########
daffodil-cli/src/test/resources/org/apache/daffodil/layers/xsd/buggyLayer.dfdl.xsd:
##########
@@ -34,8 +34,7 @@
<dfdl:format ref="buggy:GeneralFormat" />
<dfdl:defineFormat name="buggyFormat">
- <dfdl:format dfdlx:layerTransform="buggy"
dfdlx:layerLengthKind="explicit" dfdlx:layerLengthUnits="bytes"
- dfdlx:layerEncoding="ascii" />
+ <dfdl:format dfdlx:layerTransform="buggy:buggy" />
Review Comment:
Nice :+1: I like that layer names are qnames now. Feels more consistent
with the rest of DFDL. And it makes it clear that which namespaces variables
are associated with which layer.
##########
daffodil-core/src/main/scala/org/apache/daffodil/core/layers/LayerCompiler.scala:
##########
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.daffodil.core.layers
+
+import org.apache.daffodil.core.dsom.SequenceGroupTermBase
+import org.apache.daffodil.lib.exceptions.Assert
+import org.apache.daffodil.lib.xml.RefQName
+import org.apache.daffodil.runtime1.layers.LayerFactory
+import org.apache.daffodil.runtime1.layers.LayerRegistry
+import org.apache.daffodil.runtime1.layers.LayerRuntimeData
+import org.apache.daffodil.runtime1.layers.api.Layer
+
+object LayerCompiler {
+
+ /**
+ * Compiles the layer.
+ *
+ * This is mostly checking for errant use of DFDL properties that
+ * can't be used on layers.
+ *
+ * The real compilation - creating runtime data structures based
+ * on constructor signatures and DFDL variables that are in the
+ * layer's namespace, that is called here, but it is part of
+ * the daffodil runtime because that step has to be carried out
+ * *also* at runtime to verify that the dynamically loaded layer
+ * classes are compatible with the variables and their definitions.
+ *
+ * Constructs a LayerFactory which is the serializable runtime data structure
+ * used by the LayerParser, and LayerUnparser at runtime.
+ */
+ def compileLayer(sq: SequenceGroupTermBase): LayerFactory = {
+ val lc = new LayerCompiler(sq)
+ val res = lc.compile()
+ res
+ }
+}
+
+/**
+ *
+ * @param sq
+ */
+private class LayerCompiler private (sq: SequenceGroupTermBase) {
+
+ Assert.usage(sq.isLayered)
+
+ private def srd = sq.sequenceRuntimeData
+ private def lrd = sq.optionLayerRuntimeData.get
+
+ private def layerQName: RefQName = lrd.layerQName
+ private def layerName = layerQName.local
+ private def layerNamespace = layerQName.namespace
+
+ private val layeredSequenceAllowedProps = Seq(
+ "ref",
Review Comment:
Why is `ref` allowed on a layer `xs:sequence`?
##########
daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/layers/LayerFactory.scala:
##########
@@ -0,0 +1,326 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.daffodil.runtime1.layers
+
+import java.lang.reflect.Constructor
+import java.lang.reflect.Method
+import scala.collection.immutable.ListSet
+import scala.collection.mutable
+
+import org.apache.daffodil.lib.exceptions.Assert
+import org.apache.daffodil.runtime1.dpath.NodeInfo.PrimType
+import org.apache.daffodil.runtime1.infoset.DataValue
+import org.apache.daffodil.runtime1.layers.api.Layer
+import org.apache.daffodil.runtime1.layers.api.LayerException
+import org.apache.daffodil.runtime1.processors.VariableRuntimeData
+
+object LayerFactory {
+
+ /** cache that maps spiName of layer to the LayerVarsRuntime */
+ private lazy val alreadyCheckedLayers =
+ new mutable.LinkedHashMap[String, LayerVarsRuntime]()
+
+ /**
+ * Computes the things that can be computed once-only to make calling the
constructor
+ * and passing the parameter variables faster at runtime. This also does all
the
+ * checking that the constructor has an argument for each variable of
properly matching type, and
+ * has a getter for each result variable, again returning the proper type.
+ *
+ * This is called at schema compile time to ensure the layer code is
properly defined and matches
+ * the layer variable definitions.
+ *
+ * It is called again at runtime after the Layer class is loaded by the SPI
+ * to ensure that the loaded layer class constructor signature at least
matches the layer
+ * variables defined in the schema.
+ * @param lrd
+ * @param protoLayer the layer instance allocated by the SPI loader
(zero-arg constructed)
+ * @return
+ */
+ def computeLayerVarsRuntime(lrd: LayerRuntimeData, protoLayer: Layer):
LayerVarsRuntime = {
+ val optLayerVarsRuntime = alreadyCheckedLayers.get(protoLayer.name())
+ optLayerVarsRuntime.getOrElse {
+ // we know there is a default zero arg constructor
+ // find another constructor and check that there is an argument
+ // corresponding to each of the layer variables.
+ val c = protoLayer.getClass
+ val allConstructors = c.getConstructors.toSeq
+
+ val constructor: Constructor[_] =
+ if (allConstructors.length == 1) {
+ // There is only the default constructor.
+ // That's ok if there are no variables for the layer, which we check
later.
+ allConstructors.head
+ } else if (allConstructors.length == 2) {
+ allConstructors.filter(_.getParameterCount > 0).head
+ } else {
+ def tooManyConstructorsMsg: String = {
+ s"""Layer class $c has multiple non-default constructors. It
should have a default (no args)
+ | constructor and a single additional constructor with
arguments for
+ | each of the layer's parameter variables.""".stripMargin
+ }
+ lrd.context.SDE(tooManyConstructorsMsg)
+ }
+
+ if (lrd.vmap.isEmpty && allConstructors.length == 1) {
+ // there are no vars, we're done
+ new LayerVarsRuntime(constructor, Nil, Nil)
+ } else {
+ // there is a constructor with args that are supposed to correspond to
bound vars
+ val params = constructor.getParameters.toSeq
+ val nParams = params.length
+ val nVars = lrd.vmap.size
+
+ val paramTypes = constructor.getParameterTypes.toSeq
+ val paramVRDs = params.map { p =>
+ lrd.vmap.getOrElse(
+ p.getName,
+ lrd.context.SDE(s"No layer DFDL variable named '$p.getName' was
found."),
Review Comment:
I might be useful to include the namespace in the error message just to make
it clear that the variable must be in the same namespace as layer. Feels like
that could be a common mistake to forget about namespaces and try to expect to
access a variable in another namespace.
##########
daffodil-runtime1-layers/src/main/scala/org/apache/daffodil/layers/runtime1/Base64MimeLayer.scala:
##########
@@ -14,27 +14,21 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.daffodil.io
-import java.nio.charset.Charset
+package org.apache.daffodil.layers.runtime1
-class LayerBoundaryMarkInsertingJavaOutputStream(
- jos: java.io.OutputStream,
- boundaryMark: String,
- charset: Charset,
-) extends java.io.FilterOutputStream(jos) {
+import java.io.InputStream
+import java.io.OutputStream
- private var closed = false
+import org.apache.daffodil.runtime1.layers.api.Layer
+import org.apache.daffodil.runtime1.layers.api.LayerRuntime
- private val boundaryMarkBytes = boundaryMark.getBytes(charset)
+final class Base64MimeLayer
+ extends Layer("base64_MIME", "urn:org.apache.daffodil.layers.base64_MIME") {
- override def close(): Unit = {
- if (!closed) {
- jos.write(boundaryMarkBytes)
- jos.flush()
- jos.close()
- closed = true
- }
- }
+ override def wrapLayerOutput(jos: OutputStream, lrd: LayerRuntime):
OutputStream =
+ java.util.Base64.getMimeEncoder().wrap(jos)
+ override def wrapLayerInput(jis: InputStream, lrd: LayerRuntime):
InputStream =
+ java.util.Base64.getMimeDecoder.wrap(jis)
Review Comment:
Awesome! I think this shows how much of an improvement this API is. A layer
to support base64 can be done in a handful of lines of actual code--granted it
depends on a library to do the actual implementation, but it shows how little
boilerplate there is
##########
daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/layers/api/Layer.java:
##########
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.daffodil.runtime1.layers.api;
+
+import org.apache.daffodil.lib.xml.QName;
+import org.apache.daffodil.runtime1.layers.LayerUtils;
+
+import java.io.FilterInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+/**
+ * This is the primary API class for writing layers.
+ * <p/>
+ * All layers are derived from this class, and must have no-args default
constructors.
+ * <p/>
+ * Derived classes will be dynamically loaded by Java's SPI system.
+ * The names of concrete classes derived from Layer are listed in a
resources/M.services file
+ * so that they can be found and dynamically loaded.
+ * <p/>
+ * The SPI creates an instance the class of which is used as a factory to
create the
+ * instances actually used by the Daffodil runtime. Compilation of the static
information about
+ * the layer occurs only once and is then shared by all runtime instances.
+ * <p/>
+ * Instances of derived layer classes can be stateful. They are private to
threads, and each time a layer
+ * is encountered during parse/unparse, an instance is created for that
situation.
+ * <p/>
+ * Layer instances should not share mutable state (such as via singleton
objects)
+ * <p/>
+ * All the static information about the layer is provided in the arguments.
+ * <p/>
+ * The rest of the Layer class implements the
+ * layer decode/encode logic, which is done as part of deriving one's Layer
class from the
+ * Layer base class.
+ * <p/>
+ * About variables: Layer logic may read and write variables. Variables being
read are parameters to
+ * the layer algorithm. Variables being written are outputs (such as
checksums) from the layer algorithm.
+ * Variables being written must be undefined, since variables in DFDL are
single-assignment.
+ * Variables being read must be defined before being read by the layer, and
this is true for both
+ * parsing and unparsing. When unparsing, variables being read cannot be
forward-referencing to parts
+ * of the DFDL infoset that have not yet been unparsed.
+ * <p/>
+ */
+public abstract class Layer {
+
+ protected final String layerLocalName;
+ protected final String layerNamespace;
+
+ /**
+ * Constructs a new Layer object with the given layer name and namespace.
+ *
+ * @param layerLocalName the local NCName of the layer. Must be usable
as a Java identifier.
+ * @param layerNamespace the namespace of the layer. Must obey URI syntax.
+ * @throws IllegalArgumentException if arguments are null or do not obey
required syntax.
+ */
+ public Layer(String layerLocalName, String layerNamespace) {
Review Comment:
Suggest the parameters just be called `localName` and `namespace`? I imagine
it's clear is the name/namespace of the layer?
##########
daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/layers/api/Layer.java:
##########
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.daffodil.runtime1.layers.api;
+
+import org.apache.daffodil.lib.xml.QName;
+import org.apache.daffodil.runtime1.layers.LayerUtils;
+
+import java.io.FilterInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+/**
+ * This is the primary API class for writing layers.
+ * <p/>
+ * All layers are derived from this class, and must have no-args default
constructors.
+ * <p/>
+ * Derived classes will be dynamically loaded by Java's SPI system.
+ * The names of concrete classes derived from Layer are listed in a
resources/M.services file
+ * so that they can be found and dynamically loaded.
+ * <p/>
+ * The SPI creates an instance the class of which is used as a factory to
create the
+ * instances actually used by the Daffodil runtime. Compilation of the static
information about
+ * the layer occurs only once and is then shared by all runtime instances.
+ * <p/>
+ * Instances of derived layer classes can be stateful. They are private to
threads, and each time a layer
+ * is encountered during parse/unparse, an instance is created for that
situation.
+ * <p/>
+ * Layer instances should not share mutable state (such as via singleton
objects)
+ * <p/>
+ * All the static information about the layer is provided in the arguments.
+ * <p/>
+ * The rest of the Layer class implements the
+ * layer decode/encode logic, which is done as part of deriving one's Layer
class from the
+ * Layer base class.
+ * <p/>
+ * About variables: Layer logic may read and write variables. Variables being
read are parameters to
+ * the layer algorithm. Variables being written are outputs (such as
checksums) from the layer algorithm.
+ * Variables being written must be undefined, since variables in DFDL are
single-assignment.
+ * Variables being read must be defined before being read by the layer, and
this is true for both
+ * parsing and unparsing. When unparsing, variables being read cannot be
forward-referencing to parts
+ * of the DFDL infoset that have not yet been unparsed.
+ * <p/>
+ */
+public abstract class Layer {
+
+ protected final String layerLocalName;
+ protected final String layerNamespace;
Review Comment:
Should these members be private? Otherwise a layer implementation could set
them to something that isn't JavaID/URI compatible?
##########
daffodil-runtime1-layers/src/main/resources/org/apache/daffodil/layers/xsd/boundaryMarkLayer.dfdl.xsd:
##########
@@ -0,0 +1,39 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<schema
+ xmlns:xs="http://www.w3.org/2001/XMLSchema";
+ xmlns="http://www.w3.org/2001/XMLSchema";
+ xmlns:dfdl="http://www.ogf.org/dfdl/dfdl-1.0/";
+ xmlns:dfdlx="http://www.ogf.org/dfdl/dfdl-1.0/extensions";
+ xmlns:bm="urn:org.apache.daffodil.layers.boundaryMark"
+ targetNamespace="urn:org.apache.daffodil.layers.boundaryMark">
+
+ <annotation>
+ <appinfo source="http://www.ogf.org/dfdl/";>
+
+ <dfdl:defineVariable name="boundaryMark" type="xs:string"/>
+ <dfdl:defineVariable name="layerEncoding" type="xs:string"/>
Review Comment:
Do these layer dfdl.xsd files want to be the user visible documentation
about the layer? So it could include a description about what the layer does,
how it works, what effects the variables have, etc.?
##########
daffodil-runtime1-layers/src/main/scala/org/apache/daffodil/layers/runtime1/FixedLengthLayer.scala:
##########
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.daffodil.layers.runtime1
+
+import java.io.ByteArrayInputStream
+import java.io.ByteArrayOutputStream
+import java.io.InputStream
+import java.io.OutputStream
+import java.lang.{ Long => JLong }
+import java.nio.ByteBuffer
+
+import org.apache.daffodil.lib.exceptions.Assert
+import org.apache.daffodil.runtime1.layers.api.Layer
+import org.apache.daffodil.runtime1.layers.api.LayerRuntime
+
+import org.apache.commons.io.IOUtils
+
+/**
+ * Suitable only for small sections of data, not large data streams or whole
files.
+ * See the maxFixedLength value defined herein for the maximum.
+ *
+ * The entire fixed length region of the data will be pulled into a byte
buffer in memory.
+ *
+ * TODO: Someday, enhance to make this streaming.
+ *
+ * One DFDL Variable is a parameter
+ * - fixedLength - an unsignedInt giving the fixed length of this layer.
+ * This length is enforced on both parsing and unparsing the layer.
+ * There are no output/result DFDL variables from this layer.
+ */
+final class FixedLengthLayer(var fixedLength: JLong)
Review Comment:
I just noticed this, do input variable constructors need to have `var` types
to work with reflection?
##########
daffodil-runtime1-layers/src/main/scala/org/apache/daffodil/layers/runtime1/FixedLengthLayer.scala:
##########
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.daffodil.layers.runtime1
+
+import java.io.ByteArrayInputStream
+import java.io.ByteArrayOutputStream
+import java.io.InputStream
+import java.io.OutputStream
+import java.lang.{ Long => JLong }
+import java.nio.ByteBuffer
+
+import org.apache.daffodil.lib.exceptions.Assert
+import org.apache.daffodil.runtime1.layers.api.Layer
+import org.apache.daffodil.runtime1.layers.api.LayerRuntime
+
+import org.apache.commons.io.IOUtils
+
+/**
+ * Suitable only for small sections of data, not large data streams or whole
files.
+ * See the maxFixedLength value defined herein for the maximum.
+ *
+ * The entire fixed length region of the data will be pulled into a byte
buffer in memory.
+ *
+ * TODO: Someday, enhance to make this streaming.
+ *
+ * One DFDL Variable is a parameter
+ * - fixedLength - an unsignedInt giving the fixed length of this layer.
+ * This length is enforced on both parsing and unparsing the layer.
+ * There are no output/result DFDL variables from this layer.
+ */
+final class FixedLengthLayer(var fixedLength: JLong)
+ extends Layer("fixedLength", "urn:org.apache.daffodil.layers.fixedLength") {
+
+ Assert.invariant(fixedLength > 0)
+
+ /** Required for SPI class loading */
+ def this() = this(1)
+
+ private def maxFixedLength = Short.MaxValue
+
+ override def wrapLayerInput(jis: InputStream, lr: LayerRuntime): InputStream
= {
+
+ if (fixedLength > maxFixedLength)
+ lr.processingError(
+ s"fixedLength value of $fixedLength is above the maximum of
$maxFixedLength.",
+ )
+
+ new FixedLengthInputStream(fixedLength.toInt, jis, lr)
+ }
+
+ override def wrapLayerOutput(jos: OutputStream, lr: LayerRuntime):
OutputStream = {
+
+ if (fixedLength > maxFixedLength)
Review Comment:
Having to duplicate input variable validation doesn't feel great. I guess
users could implement their own `check` function like we used to have and call
it from both wrap functions. Other option would be LayerRuntimeData as the
first parameter of the input variable constructor and implementations could
perform checks in the constructor? This would then look something like
```scala
final class FixedLengthLayer(var lr: LayerRuntime, var fixedLength: JLong)
extends Layer("fixedLength", "urn:org.apache.daffodil.layers.fixedLength")
{
Assert.invariant(fixedLength > 0)
if (lr != null) {
if (fixedLength > maxFixedLength) lr.processingError(....)
}
/** Required for SPI class loading */
def this() = this(null, 1)
...
}
```
So some silly null stuff we have to do.
But the equivalent Java implementation seems reasonable:
```java
class FixedLengthLayer extends Layer {
public FixedLengthLayer() {
super("fixedLength", "urn:org.apache.daffodil.layers.fixedLength");
}
private Long fixedLength;
public FixdLengthLayer(LayerRuntime lr, Long fixedLength) {
if (fixedLength > maxFixedLength) lr.processingError(...);
this.fixedLength = fixedLength;
}
...
```
Maybe a second constructor, input variables are passed to a separate
`setVariables` function, this has the benefit that construction can never fail?
And it also means there's only a single constructor and an optional
`setVariables` function if variables are needed? Avoids the whole `def this() =
this("dummy")` kind of thing for scala implementations. But does require `var`s
to store the state of the variables to be used in the wrap functions though.
##########
daffodil-runtime1-layers/src/main/scala/org/apache/daffodil/layers/runtime1/FixedLengthLayer.scala:
##########
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.daffodil.layers.runtime1
+
+import java.io.ByteArrayInputStream
+import java.io.ByteArrayOutputStream
+import java.io.InputStream
+import java.io.OutputStream
+import java.lang.{ Long => JLong }
+import java.nio.ByteBuffer
+
+import org.apache.daffodil.lib.exceptions.Assert
+import org.apache.daffodil.runtime1.layers.api.Layer
+import org.apache.daffodil.runtime1.layers.api.LayerRuntime
+
+import org.apache.commons.io.IOUtils
+
+/**
+ * Suitable only for small sections of data, not large data streams or whole
files.
+ * See the maxFixedLength value defined herein for the maximum.
+ *
+ * The entire fixed length region of the data will be pulled into a byte
buffer in memory.
+ *
+ * TODO: Someday, enhance to make this streaming.
+ *
+ * One DFDL Variable is a parameter
+ * - fixedLength - an unsignedInt giving the fixed length of this layer.
+ * This length is enforced on both parsing and unparsing the layer.
+ * There are no output/result DFDL variables from this layer.
+ */
+final class FixedLengthLayer(var fixedLength: JLong)
+ extends Layer("fixedLength", "urn:org.apache.daffodil.layers.fixedLength") {
+
+ Assert.invariant(fixedLength > 0)
+
+ /** Required for SPI class loading */
+ def this() = this(1)
+
+ private def maxFixedLength = Short.MaxValue
+
+ override def wrapLayerInput(jis: InputStream, lr: LayerRuntime): InputStream
= {
+
+ if (fixedLength > maxFixedLength)
+ lr.processingError(
+ s"fixedLength value of $fixedLength is above the maximum of
$maxFixedLength.",
+ )
+
+ new FixedLengthInputStream(fixedLength.toInt, jis, lr)
+ }
+
+ override def wrapLayerOutput(jos: OutputStream, lr: LayerRuntime):
OutputStream = {
+
+ if (fixedLength > maxFixedLength)
+ lr.processingError(
+ s"fixedLength value of $fixedLength is above the maximum of
$maxFixedLength.",
+ )
+
+ new FixedLengthOutputStream(fixedLength.toInt, jos, lr)
+ }
+}
+
+class FixedLengthInputStream(
+ layerLength: Int,
+ jis: InputStream,
+ lr: LayerRuntime,
+) extends InputStream {
+
+ private lazy val bais = {
+ val ba = new Array[Byte](layerLength)
+ val nRead = IOUtils.read(jis, ba)
+ if (nRead < layerLength)
+ lr.processingError(
+ s"Insufficient data for fixed-length layer. Needed $layerLength bytes,
but only $nRead were available.",
+ )
+ val buf = ByteBuffer.wrap(ba)
+ new ByteArrayInputStream(ba)
+ }
+
+ override def read(): Int = bais.read()
+}
+
+class FixedLengthOutputStream(
+ layerLength: Int,
+ jos: OutputStream,
+ lr: LayerRuntime,
+) extends OutputStream {
+
+ private lazy val baos = new ByteArrayOutputStream(layerLength.toInt)
+
+ private var count: Long = 0
+
+ override def write(b: Int): Unit = {
+ baos.write(b)
+ count += 1
+ if (count == layerLength) {
+ // we can auto-close it in this case
+ close()
+ } else if (count > layerLength) {
+ //
+ // This could happen if the layer logically unparses as one of two
choice branches where they
+ // are supposed to be all the same length, but one is in fact longer
than expected by the bufLen.
+ lr.processingError(
+ new IndexOutOfBoundsException(
+ s"Written data amount exceeded fixed layer length of $layerLength.",
+ ),
+ )
+ } else {
+ // Assert.invariant(count < layerLength)
+ // ok. We're still accumulating data
+ }
+ }
+
+ private var isOpen: Boolean = true
+
+ override def close(): Unit = {
+ if (isOpen) { // allow multiple closes
+ isOpen = false
+ val ba = baos.toByteArray
+ val baLen = ba.length
+ if (baLen != layerLength)
+ lr.processingError(
+ s"Insufficient output data for fixed-length layer. Needed
$layerLength bytes, but only $baLen were unparsed.",
+ )
+ jos.write(ba)
+
+ // TODO: Consider if this close should happen in the framework instead
of here.
Review Comment:
Is it possible to allow both? Ideally our framework could ensure everything
is closed so implementations are forced to worry about it (we've seen a case
where an implementation forgot it and had subtle issues). But if users do want
to ensure things are closed we should allow it.
##########
daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/dpath/NodeInfo.scala:
##########
@@ -932,6 +936,33 @@ object NodeInfo extends Enum {
DFDLTimeConversion.fromXMLString(s)
}
}
+
+ def toJavaType(dfdlType: DFDLPrimType): Class[_] = {
+ dfdlType match {
+ case DFDLPrimType.String => classOf[JString]
+ case DFDLPrimType.Int => classOf[JInt]
+ case DFDLPrimType.Byte => classOf[JByte]
+ case DFDLPrimType.Short => classOf[JShort]
+ case DFDLPrimType.Long => classOf[JLong]
+ case DFDLPrimType.Integer => classOf[JBigInt]
+ case DFDLPrimType.Decimal => classOf[JBigDecimal]
+ case DFDLPrimType.UnsignedInt => classOf[JLong]
+ case DFDLPrimType.UnsignedByte => classOf[JShort]
+ case DFDLPrimType.UnsignedShort => classOf[JInt]
+ case DFDLPrimType.UnsignedLong => classOf[JBigInt]
+ case DFDLPrimType.NonNegativeInteger => classOf[JBigInt]
+ case DFDLPrimType.Double => classOf[JDouble]
+ case DFDLPrimType.Float => classOf[JFloat]
+ case DFDLPrimType.HexBinary => classOf[Array[Byte]]
+ case DFDLPrimType.AnyURI => classOf[java.net.URI]
+ case DFDLPrimType.Boolean => classOf[JBoolean]
+ case DFDLPrimType.DateTime => classOf[ICUCalendar]
+ case DFDLPrimType.Date => classOf[ICUCalendar]
+ case DFDLPrimType.Time => classOf[ICUCalendar]
Review Comment:
Is this correct? For some reason I thought we used a custom class that
wrapped ICU calendar for our infoset/variable representations?
##########
daffodil-runtime1-layers/src/main/resources/org/apache/daffodil/layers/xsd/base64_MIMELayer.dfdl.xsd:
##########
@@ -0,0 +1,37 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<schema
+ xmlns:xs="http://www.w3.org/2001/XMLSchema";
+ xmlns="http://www.w3.org/2001/XMLSchema";
+ xmlns:dfdl="http://www.ogf.org/dfdl/dfdl-1.0/";
+ xmlns:dfdlx="http://www.ogf.org/dfdl/dfdl-1.0/extensions";
+ xmlns:b64="urn:org.apache.daffodil.layers.base64_MIME"
+ targetNamespace="urn:org.apache.daffodil.layers.base64_MIME">
+
+ <!--
+ This layer does not define any variables for parameters or results
+ -->
+ <annotation>
+ <appinfo source="http://www.ogf.org/dfdl/";>
+ <dfdl:defineFormat name="base64_MIME">
+ <dfdl:format dfdlx:layerTransform="b64:base64_MIME"/>
+ </dfdl:defineFormat>
+ </appinfo>
+ </annotation>
Review Comment:
Is the idea with this defined format so you could do either
```xml
<sequence dfdlx:layer="b64:base64_MIME">
```
or
```xml
<sequence dfdl:ref="b64:base64_MIME">
```
Is there a benefit of `dfdl:ref` over `dfdlx:layer`? Seems they are
virtually the same, but using `dfdl:ref` seems less obviously a layer to me. I
guess it allows us to change the layer name without having to change the ref if
we wanted?
##########
daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/layers/LayerFactory.scala:
##########
@@ -0,0 +1,326 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.daffodil.runtime1.layers
+
+import java.lang.reflect.Constructor
+import java.lang.reflect.Method
+import scala.collection.immutable.ListSet
+import scala.collection.mutable
+
+import org.apache.daffodil.lib.exceptions.Assert
+import org.apache.daffodil.runtime1.dpath.NodeInfo.PrimType
+import org.apache.daffodil.runtime1.infoset.DataValue
+import org.apache.daffodil.runtime1.layers.api.Layer
+import org.apache.daffodil.runtime1.layers.api.LayerException
+import org.apache.daffodil.runtime1.processors.VariableRuntimeData
+
+object LayerFactory {
+
+ /** cache that maps spiName of layer to the LayerVarsRuntime */
+ private lazy val alreadyCheckedLayers =
+ new mutable.LinkedHashMap[String, LayerVarsRuntime]()
+
+ /**
+ * Computes the things that can be computed once-only to make calling the
constructor
+ * and passing the parameter variables faster at runtime. This also does all
the
+ * checking that the constructor has an argument for each variable of
properly matching type, and
+ * has a getter for each result variable, again returning the proper type.
+ *
+ * This is called at schema compile time to ensure the layer code is
properly defined and matches
+ * the layer variable definitions.
+ *
+ * It is called again at runtime after the Layer class is loaded by the SPI
+ * to ensure that the loaded layer class constructor signature at least
matches the layer
+ * variables defined in the schema.
+ * @param lrd
+ * @param protoLayer the layer instance allocated by the SPI loader
(zero-arg constructed)
+ * @return
+ */
+ def computeLayerVarsRuntime(lrd: LayerRuntimeData, protoLayer: Layer):
LayerVarsRuntime = {
+ val optLayerVarsRuntime = alreadyCheckedLayers.get(protoLayer.name())
+ optLayerVarsRuntime.getOrElse {
+ // we know there is a default zero arg constructor
+ // find another constructor and check that there is an argument
+ // corresponding to each of the layer variables.
+ val c = protoLayer.getClass
+ val allConstructors = c.getConstructors.toSeq
+
+ val constructor: Constructor[_] =
+ if (allConstructors.length == 1) {
+ // There is only the default constructor.
+ // That's ok if there are no variables for the layer, which we check
later.
+ allConstructors.head
+ } else if (allConstructors.length == 2) {
+ allConstructors.filter(_.getParameterCount > 0).head
+ } else {
+ def tooManyConstructorsMsg: String = {
+ s"""Layer class $c has multiple non-default constructors. It
should have a default (no args)
+ | constructor and a single additional constructor with
arguments for
+ | each of the layer's parameter variables.""".stripMargin
+ }
+ lrd.context.SDE(tooManyConstructorsMsg)
+ }
+
+ if (lrd.vmap.isEmpty && allConstructors.length == 1) {
+ // there are no vars, we're done
+ new LayerVarsRuntime(constructor, Nil, Nil)
+ } else {
+ // there is a constructor with args that are supposed to correspond to
bound vars
+ val params = constructor.getParameters.toSeq
+ val nParams = params.length
+ val nVars = lrd.vmap.size
+
+ val paramTypes = constructor.getParameterTypes.toSeq
+ val paramVRDs = params.map { p =>
+ lrd.vmap.getOrElse(
+ p.getName,
+ lrd.context.SDE(s"No layer DFDL variable named '$p.getName' was
found."),
+ )
+ }.toSeq
+
+ // Now we deal with the result getters and the corresponding vars
+ //
+ val allLayerVRDs = ListSet(lrd.vmap.toSeq.map(_._2): _*)
+ val returnVRDs = allLayerVRDs -- paramVRDs // set subtraction
Review Comment:
Ah, intersing, so output variables are those that do not have an associated
parameter in the constructor? Makes sense, might be worth adding a comment
about that some where. I hadn't thought that's how it would work by it makes
sense.
##########
daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/layers/api/ChecksumLayer.java:
##########
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.daffodil.runtime1.layers.api;
+
+import org.apache.daffodil.runtime1.layers.ChecksumLayerBase;
+
+import java.nio.ByteBuffer;
+
+/**
+ * Suitable only for checksums computed over small sections of data, not large
data streams or whole files.
+ *
+ * The entire region of data the checksum is being computed over, will be
pulled into a byte buffer in memory.
+ */
+public abstract class ChecksumLayer extends ChecksumLayerBase {
+
+ public ChecksumLayer(String layerName, String layerNamespace, int
checksumLayerLength) {
+ super(layerName, layerNamespace, checksumLayerLength);
+ }
+
+ /**
+ * Override to compute the checksum of a buffer of data. The value computed
is assigned to the first
+ * DFDL variable defined by the layer in the LayerInfo object.
+ *
+ * @param layerRuntime layer context for the computation
+ * @param isUnparse true if the direction is unparsing. Used because in some
cases the computed checksum must
+ * be written into the byte buffer in a specific location.
+ * @param byteBuffer the bytes over which the checksum is to be computed.
This can be modified, (for example so as
+ * to embed the computed checksum in the middle of the
data somewhere) and the resulting
+ * bytes become the data that is written when unparsing.
+ * If the bytes in this buffer are modified by the compute
method, those modified bytes are what
+ * the parsing will parse from, and the unparsing will
output.
+ * @return the checksum value as an Int (32-bit signed integer)
+ */
+ public abstract int compute(
+ LayerRuntime layerRuntime,
+ boolean isUnparse,
+ ByteBuffer byteBuffer
+ );
+}
Review Comment:
Do implementations need to so something like this to make the checksum
result available?
```java
public int getDFDLResultVariable_myChecksumVariable() {
return this.getChecksum();
}
```
Assuming the variable `myChecksumVariable` is defined with type `xs:int`?
Shoudl we add documentation about that?
##########
daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/layers/LayerFactory.scala:
##########
@@ -0,0 +1,326 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.daffodil.runtime1.layers
+
+import java.lang.reflect.Constructor
+import java.lang.reflect.Method
+import scala.collection.immutable.ListSet
+import scala.collection.mutable
+
+import org.apache.daffodil.lib.exceptions.Assert
+import org.apache.daffodil.runtime1.dpath.NodeInfo.PrimType
+import org.apache.daffodil.runtime1.infoset.DataValue
+import org.apache.daffodil.runtime1.layers.api.Layer
+import org.apache.daffodil.runtime1.layers.api.LayerException
+import org.apache.daffodil.runtime1.processors.VariableRuntimeData
+
+object LayerFactory {
+
+ /** cache that maps spiName of layer to the LayerVarsRuntime */
+ private lazy val alreadyCheckedLayers =
+ new mutable.LinkedHashMap[String, LayerVarsRuntime]()
+
+ /**
+ * Computes the things that can be computed once-only to make calling the
constructor
+ * and passing the parameter variables faster at runtime. This also does all
the
+ * checking that the constructor has an argument for each variable of
properly matching type, and
+ * has a getter for each result variable, again returning the proper type.
+ *
+ * This is called at schema compile time to ensure the layer code is
properly defined and matches
+ * the layer variable definitions.
+ *
+ * It is called again at runtime after the Layer class is loaded by the SPI
+ * to ensure that the loaded layer class constructor signature at least
matches the layer
+ * variables defined in the schema.
+ * @param lrd
+ * @param protoLayer the layer instance allocated by the SPI loader
(zero-arg constructed)
+ * @return
+ */
+ def computeLayerVarsRuntime(lrd: LayerRuntimeData, protoLayer: Layer):
LayerVarsRuntime = {
+ val optLayerVarsRuntime = alreadyCheckedLayers.get(protoLayer.name())
+ optLayerVarsRuntime.getOrElse {
+ // we know there is a default zero arg constructor
+ // find another constructor and check that there is an argument
+ // corresponding to each of the layer variables.
+ val c = protoLayer.getClass
+ val allConstructors = c.getConstructors.toSeq
+
+ val constructor: Constructor[_] =
+ if (allConstructors.length == 1) {
+ // There is only the default constructor.
+ // That's ok if there are no variables for the layer, which we check
later.
+ allConstructors.head
+ } else if (allConstructors.length == 2) {
+ allConstructors.filter(_.getParameterCount > 0).head
+ } else {
+ def tooManyConstructorsMsg: String = {
+ s"""Layer class $c has multiple non-default constructors. It
should have a default (no args)
+ | constructor and a single additional constructor with
arguments for
+ | each of the layer's parameter variables.""".stripMargin
+ }
+ lrd.context.SDE(tooManyConstructorsMsg)
+ }
+
+ if (lrd.vmap.isEmpty && allConstructors.length == 1) {
+ // there are no vars, we're done
+ new LayerVarsRuntime(constructor, Nil, Nil)
+ } else {
+ // there is a constructor with args that are supposed to correspond to
bound vars
+ val params = constructor.getParameters.toSeq
+ val nParams = params.length
+ val nVars = lrd.vmap.size
+
+ val paramTypes = constructor.getParameterTypes.toSeq
+ val paramVRDs = params.map { p =>
+ lrd.vmap.getOrElse(
+ p.getName,
+ lrd.context.SDE(s"No layer DFDL variable named '$p.getName' was
found."),
+ )
+ }.toSeq
+
+ // Now we deal with the result getters and the corresponding vars
+ //
+ val allLayerVRDs = ListSet(lrd.vmap.toSeq.map(_._2): _*)
+ val returnVRDs = allLayerVRDs -- paramVRDs // set subtraction
+ val allMethods = c.getMethods
+ val allVarResultGetters =
+ ListSet(allMethods.filter { m =>
+ val nom = m.getName
+ nom.startsWith(varResultPrefix)
+ }.toSeq: _*)
+ // each returnVRD needs to have a corresponding getter method
+ val returnVRDNames = returnVRDs.map(_.globalQName.local)
+ val resultGettersNames =
allVarResultGetters.map(_.getName.replace(varResultPrefix, ""))
+ val nResultGetters = resultGettersNames.size
+ def javaConstructorArgs =
+ paramVRDs
+ .map {
+ case vrd => {
+ s"type: ${PrimType.toJavaTypeString(vrd.primType.dfdlType)}
name: ${vrd.globalQName.local}"
+ }
+ }
+ .mkString(", ")
+ def badConstructorMsg: String = {
+ s"""Layer class $c does not have a constructor with arguments for
each of the layer's variables.
+ | It should have a constructor with these arguments in any order,
such as
+ | ($javaConstructorArgs)""".stripMargin
+ }
+
+ lrd.context.schemaDefinitionUnless(nParams + nResultGetters == nVars,
badConstructorMsg)
+
+ // at this point the number of vars and number of constructor args
match
+
+ val typePairs = (paramVRDs.zip(paramTypes)).toSeq
+ typePairs.foreach { case (vrd, pt) =>
+ val vrdClass = PrimType.toJavaType(vrd.primType.dfdlType)
+ lrd.context.schemaDefinitionUnless(
+ vrdClass == pt,
+ s"""Layer constructor argument ${vrd.globalQName.local} and the
corresponding
+ |Layer DFDL variable have differing types: ${pt.getName}
+ | and ${vrdClass.getName} respectively.""".stripMargin,
+ )
+ }
+
+ val returnVRDsWithoutGetters = returnVRDNames -- resultGettersNames
+ val resultGettersWithoutVRDs = resultGettersNames -- returnVRDNames
+ lrd.context.schemaDefinitionUnless(
+ returnVRDsWithoutGetters.isEmpty,
+ s"""The layer variables ${returnVRDsWithoutGetters.mkString(
+ ",",
+ )} have no corresponding getters.""",
+ )
+ lrd.context.schemaDefinitionUnless(
+ resultGettersWithoutVRDs.isEmpty, {
+ val getterFullNames = returnVRDsWithoutGetters.map { vname =>
+ this.varResultPrefix + vname
+ }
+ s"""The getters ${getterFullNames.mkString(
+ ",",
+ )} have no corresponding layer variables."""
+ },
+ )
+ // at this point we know each variable that was not a parameter of the
constructor
+ // has a getter with matching name.
+ val resultVarPairs = resultGettersNames.map { rgn: String =>
+ val getter: Method =
+ allVarResultGetters.find { g: Method => g.getName.endsWith(rgn)
}.getOrElse {
+ Assert.invariantFailed("no getter for getter name.")
+ }
+ val vrd = returnVRDs.find { vrd => vrd.globalQName.local == rgn
}.getOrElse {
+ Assert.invariantFailed("no vrd for getter name.")
+ }
+ (vrd, getter)
+ }
+ resultVarPairs.foreach { case (vrd, getter) =>
+ val vrdClass = PrimType.toJavaType(vrd.primType.dfdlType)
+ val gt = getter.getReturnType
+ lrd.context.schemaDefinitionUnless(
+ vrdClass == gt,
+ s"""Layer return variable ${vrd.globalQName.local} and the
corresponding
+ |Layer getter have differing types: ${vrdClass.getName}
+ | and ${gt.getName} respectively.""".stripMargin,
+ )
+ }
+ val lrv = new LayerVarsRuntime(constructor, paramVRDs,
resultVarPairs.toSeq)
+ alreadyCheckedLayers.put(lrd.spiName, lrv)
+ lrv
+ }
+ }
+ }
+
+ private def varResultPrefix = "getDFDLResultVariable_"
+
+ type ParameterVarsInfo = Seq[Seq[(String, Class[_])]]
+ type ResultVarsInfo = Seq[Method]
+
+ def analyzeClass(obj: Any): (ParameterVarsInfo, ResultVarsInfo) = {
Review Comment:
Is this old? Looks like it's only used in a test?
##########
daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/layers/LayerFactory.scala:
##########
@@ -0,0 +1,326 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.daffodil.runtime1.layers
+
+import java.lang.reflect.Constructor
+import java.lang.reflect.Method
+import scala.collection.immutable.ListSet
+import scala.collection.mutable
+
+import org.apache.daffodil.lib.exceptions.Assert
+import org.apache.daffodil.runtime1.dpath.NodeInfo.PrimType
+import org.apache.daffodil.runtime1.infoset.DataValue
+import org.apache.daffodil.runtime1.layers.api.Layer
+import org.apache.daffodil.runtime1.layers.api.LayerException
+import org.apache.daffodil.runtime1.processors.VariableRuntimeData
+
+object LayerFactory {
+
+ /** cache that maps spiName of layer to the LayerVarsRuntime */
+ private lazy val alreadyCheckedLayers =
+ new mutable.LinkedHashMap[String, LayerVarsRuntime]()
+
+ /**
+ * Computes the things that can be computed once-only to make calling the
constructor
+ * and passing the parameter variables faster at runtime. This also does all
the
+ * checking that the constructor has an argument for each variable of
properly matching type, and
+ * has a getter for each result variable, again returning the proper type.
+ *
+ * This is called at schema compile time to ensure the layer code is
properly defined and matches
+ * the layer variable definitions.
+ *
+ * It is called again at runtime after the Layer class is loaded by the SPI
+ * to ensure that the loaded layer class constructor signature at least
matches the layer
+ * variables defined in the schema.
+ * @param lrd
+ * @param protoLayer the layer instance allocated by the SPI loader
(zero-arg constructed)
+ * @return
+ */
+ def computeLayerVarsRuntime(lrd: LayerRuntimeData, protoLayer: Layer):
LayerVarsRuntime = {
+ val optLayerVarsRuntime = alreadyCheckedLayers.get(protoLayer.name())
+ optLayerVarsRuntime.getOrElse {
+ // we know there is a default zero arg constructor
+ // find another constructor and check that there is an argument
+ // corresponding to each of the layer variables.
+ val c = protoLayer.getClass
+ val allConstructors = c.getConstructors.toSeq
+
+ val constructor: Constructor[_] =
+ if (allConstructors.length == 1) {
+ // There is only the default constructor.
+ // That's ok if there are no variables for the layer, which we check
later.
+ allConstructors.head
+ } else if (allConstructors.length == 2) {
+ allConstructors.filter(_.getParameterCount > 0).head
+ } else {
+ def tooManyConstructorsMsg: String = {
+ s"""Layer class $c has multiple non-default constructors. It
should have a default (no args)
+ | constructor and a single additional constructor with
arguments for
+ | each of the layer's parameter variables.""".stripMargin
+ }
+ lrd.context.SDE(tooManyConstructorsMsg)
+ }
+
+ if (lrd.vmap.isEmpty && allConstructors.length == 1) {
+ // there are no vars, we're done
+ new LayerVarsRuntime(constructor, Nil, Nil)
+ } else {
+ // there is a constructor with args that are supposed to correspond to
bound vars
+ val params = constructor.getParameters.toSeq
+ val nParams = params.length
+ val nVars = lrd.vmap.size
+
+ val paramTypes = constructor.getParameterTypes.toSeq
+ val paramVRDs = params.map { p =>
+ lrd.vmap.getOrElse(
+ p.getName,
+ lrd.context.SDE(s"No layer DFDL variable named '$p.getName' was
found."),
+ )
+ }.toSeq
+
+ // Now we deal with the result getters and the corresponding vars
+ //
+ val allLayerVRDs = ListSet(lrd.vmap.toSeq.map(_._2): _*)
+ val returnVRDs = allLayerVRDs -- paramVRDs // set subtraction
+ val allMethods = c.getMethods
+ val allVarResultGetters =
+ ListSet(allMethods.filter { m =>
+ val nom = m.getName
+ nom.startsWith(varResultPrefix)
+ }.toSeq: _*)
+ // each returnVRD needs to have a corresponding getter method
+ val returnVRDNames = returnVRDs.map(_.globalQName.local)
+ val resultGettersNames =
allVarResultGetters.map(_.getName.replace(varResultPrefix, ""))
+ val nResultGetters = resultGettersNames.size
+ def javaConstructorArgs =
+ paramVRDs
+ .map {
+ case vrd => {
+ s"type: ${PrimType.toJavaTypeString(vrd.primType.dfdlType)}
name: ${vrd.globalQName.local}"
+ }
+ }
+ .mkString(", ")
+ def badConstructorMsg: String = {
+ s"""Layer class $c does not have a constructor with arguments for
each of the layer's variables.
+ | It should have a constructor with these arguments in any order,
such as
+ | ($javaConstructorArgs)""".stripMargin
+ }
+
+ lrd.context.schemaDefinitionUnless(nParams + nResultGetters == nVars,
badConstructorMsg)
+
+ // at this point the number of vars and number of constructor args
match
+
+ val typePairs = (paramVRDs.zip(paramTypes)).toSeq
+ typePairs.foreach { case (vrd, pt) =>
+ val vrdClass = PrimType.toJavaType(vrd.primType.dfdlType)
+ lrd.context.schemaDefinitionUnless(
+ vrdClass == pt,
+ s"""Layer constructor argument ${vrd.globalQName.local} and the
corresponding
+ |Layer DFDL variable have differing types: ${pt.getName}
+ | and ${vrdClass.getName} respectively.""".stripMargin,
+ )
+ }
+
+ val returnVRDsWithoutGetters = returnVRDNames -- resultGettersNames
+ val resultGettersWithoutVRDs = resultGettersNames -- returnVRDNames
+ lrd.context.schemaDefinitionUnless(
+ returnVRDsWithoutGetters.isEmpty,
+ s"""The layer variables ${returnVRDsWithoutGetters.mkString(
+ ",",
+ )} have no corresponding getters.""",
+ )
+ lrd.context.schemaDefinitionUnless(
+ resultGettersWithoutVRDs.isEmpty, {
+ val getterFullNames = returnVRDsWithoutGetters.map { vname =>
+ this.varResultPrefix + vname
+ }
+ s"""The getters ${getterFullNames.mkString(
+ ",",
+ )} have no corresponding layer variables."""
+ },
+ )
+ // at this point we know each variable that was not a parameter of the
constructor
+ // has a getter with matching name.
+ val resultVarPairs = resultGettersNames.map { rgn: String =>
+ val getter: Method =
+ allVarResultGetters.find { g: Method => g.getName.endsWith(rgn)
}.getOrElse {
+ Assert.invariantFailed("no getter for getter name.")
+ }
+ val vrd = returnVRDs.find { vrd => vrd.globalQName.local == rgn
}.getOrElse {
+ Assert.invariantFailed("no vrd for getter name.")
+ }
+ (vrd, getter)
+ }
+ resultVarPairs.foreach { case (vrd, getter) =>
+ val vrdClass = PrimType.toJavaType(vrd.primType.dfdlType)
+ val gt = getter.getReturnType
+ lrd.context.schemaDefinitionUnless(
+ vrdClass == gt,
+ s"""Layer return variable ${vrd.globalQName.local} and the
corresponding
+ |Layer getter have differing types: ${vrdClass.getName}
+ | and ${gt.getName} respectively.""".stripMargin,
+ )
+ }
+ val lrv = new LayerVarsRuntime(constructor, paramVRDs,
resultVarPairs.toSeq)
+ alreadyCheckedLayers.put(lrd.spiName, lrv)
+ lrv
+ }
+ }
+ }
+
+ private def varResultPrefix = "getDFDLResultVariable_"
+
+ type ParameterVarsInfo = Seq[Seq[(String, Class[_])]]
+ type ResultVarsInfo = Seq[Method]
+
+ def analyzeClass(obj: Any): (ParameterVarsInfo, ResultVarsInfo) = {
+ val c = obj.getClass
+ val cs = c.getConstructors.toSeq
+ val constructorInfo = cs.map { c =>
+ val parms = c.getParameters.toSeq
+ parms.map { p =>
+ val pn: String = p.getName
+ val pt = p.getType
+ (pn, pt)
+ }
+ }
+ val getterInfo: Array[Method] = c.getMethods.filter {
+ _.getName.startsWith(varResultPrefix)
+ }
+ (constructorInfo, getterInfo)
+ }
+
+}
+
+/**
+ * Factory for a layer
+ *
+ * This is the serialized object which is saved as part of a processor.
+ * It constructs the layer at runtime when newInstance() is called.
+ *
+ * This allows the layer instance itself to be stateful and not serializable.
+ */
+class LayerFactory(val layerRuntimeData: LayerRuntimeData) extends
Serializable {
+ import LayerFactory._
+
+ /**
+ * Call at runtime to create a layer object. This layer object can be
stateful
+ * and non-thread safe.
+ *
+ * Called by the LayeredSequenceParser or LayeredSequenceUnparser to
allocate the
+ * layer, and the result is used to carry out the layering mechanism.
+ *
+ * @param lri the layer runtime info which includes both static and runtime
+ * state-based information for the parse or unparse
+ * @return the Layer properly initialized/constructed for this layer
+ */
+ def newInstance(lri: LayerRuntimeImpl): LayerDriver = {
+ val optCache = alreadyCheckedLayers.get(lri.layerRuntimeData.spiName)
+ val layerVarsRuntime: LayerVarsRuntime = optCache.getOrElse {
+ val optLayerInstance =
LayerRegistry.findLayer(lri.layerRuntimeData.spiName)
+ val spiLayerInstance: Layer = optLayerInstance.getOrElse {
+ lri.runtimeSchemaDefinitionError(
+ new LayerException(
+ lri,
+ s"Unable to load class for layerTransform
'${lri.layerRuntimeData.layerQName.toQNameString}'.",
+ ),
+ )
+ }
+ // Since layer implementations are dynamically loaded, we must re-verify
that
+ // the layer implementation matches the DFDL schema's layer variable
definitions.
+ // This prevents using a layer class file that doesn't match the schema,
at
+ // least as far as the number and type of the DFDL variables it consumes
and writes goes.
+ // However, we want to do this exactly once, not every time this method
is called,
+ // So there is a cache of instances that have already been through these
checks,
+ // at runtime.
+ // We compute this data structure once only at the time the SPI Loads
the layer
+ // class.
+ // In addition, this process of verifying the DFDL variables used by the
layer
+ // pre-computes some data structures that facilitate fast run-time
processing
+
+ val lvr = computeLayerVarsRuntime(lri.layerRuntimeData, spiLayerInstance)
Review Comment:
Is it possible to have the `LayerRegistry` store the LVR information? So
when it loads a layer, it immediately computes the LVR and stores it in the
registry along with the Layer instance? So the registry acts as this cache and
it's checked immediately when a layer instance is created? I'm not sure when
the registry is loaded, but it would be good to trigger that on schema reload
to make sure everything is checked if the classpath changed.
So this just becomes something like:
```scala
val (layerSPI, lvr) = LayerRegistry.findLayer(...)
```
In fact, do we even need the Layer instance from the registery? Doesn't lvr
have everything needed to create an actual new instance?
##########
project/Dependencies.scala:
##########
@@ -25,6 +25,7 @@ object Dependencies {
"com.lihaoyi" %% "os-lib" % "0.9.3", // for writing/compiling C source
files
"org.scala-lang.modules" %% "scala-xml" % "2.2.0",
"org.scala-lang.modules" %% "scala-parser-combinators" % "2.3.0",
+ "org.scala-lang.modules" %% "scala-java8-compat" % "1.0.2",
Review Comment:
What is this for? This has a NOTICE file so we'll need to update our NOTICE
files.
##########
daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/layers/LayerDriver.scala:
##########
@@ -0,0 +1,227 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.daffodil.runtime1.layers
+
+import java.io.FilterInputStream
+import java.io.FilterOutputStream
+import java.io.InputStream
+import java.io.OutputStream
+
+import org.apache.daffodil.io.DataInputStream.Mark
+import org.apache.daffodil.io.DataOutputStream
+import org.apache.daffodil.io.DirectOrBufferedDataOutputStream
+import org.apache.daffodil.io.FormatInfo
+import org.apache.daffodil.io.InputSourceDataInputStream
+import org.apache.daffodil.lib.exceptions.Assert
+import org.apache.daffodil.lib.schema.annotation.props.gen.BitOrder
+import org.apache.daffodil.lib.util.Maybe
+import org.apache.daffodil.lib.util.Maybe.Nope
+import org.apache.daffodil.lib.util.Maybe.One
+import org.apache.daffodil.lib.util.Misc
+import org.apache.daffodil.runtime1.layers.api.Layer
+import org.apache.daffodil.runtime1.processors.unparsers.UState
+
+import passera.unsigned.ULong
+
+/**
+ * Driving mechanism that incorporates a layer at runtime to transform the
data stream.
+ *
+ * A layer driver is created at runtime as part of a single parse/unparse call.
+ * Hence, they can be stateful without causing thread-safety issues.
+ */
+class LayerDriver(
+ layerRuntimeData: LayerRuntimeData,
+ layer: Layer,
+ layerVarsRuntime: LayerVarsRuntime,
+) {
+
+ private def wrapJavaInputStream(
+ s: InputSourceDataInputStream,
+ layerRuntimeImpl: LayerRuntimeImpl,
+ ): InputStream =
+ new JavaIOInputStream(s, layerRuntimeImpl.finfo)
+
+ private def wrapJavaOutputStream(
+ s: DataOutputStream,
+ layerRuntimeImpl: LayerRuntimeImpl,
+ ): OutputStream =
+ new JavaIOOutputStream(s, layer, layerRuntimeImpl, layerVarsRuntime)
+
+ /**
+ * Override these if we ever have transformers that don't have these
+ * requirements.
+ */
+ val mandatoryLayerAlignmentInBits: Int = 8
+
+ final def addInputLayer(
+ s: InputSourceDataInputStream,
+ layerRuntimeImpl: LayerRuntimeImpl,
+ ): InputSourceDataInputStream = {
+ val jis = wrapJavaInputStream(s, layerRuntimeImpl)
+ val decodedInputStream =
+ new AssuredCloseInputStream(layer.wrapLayerInput(jis, layerRuntimeImpl),
jis)
+ val newDIS = InputSourceDataInputStream(decodedInputStream)
+ newDIS.cst.setPriorBitOrder(
+ BitOrder.MostSignificantBitFirst,
+ ) // must initialize priorBitOrder
+ newDIS.setDebugging(s.areDebugging)
+ newDIS
+ }
+
+ /**
+ * Parsing works as a tree traversal, so when the parser unwinds from
+ * parsing the layer we can invoke this to handle cleanups, and
+ * finalization issues like assigning the result variables
+ */
+ final def removeInputLayer(
+ s: InputSourceDataInputStream,
+ layerRuntimeImpl: LayerRuntimeImpl,
+ ): Unit = {
+ layerVarsRuntime.callGettersToPopulateResultVars(layer, layerRuntimeImpl)
Review Comment:
I was a little confused that the `LayerDriver` handles getting output
variables but the `LayerFactory` handles setting input variables. I guess it
makes sense since the Factory creates the `Layer` and so it needs the input
variables, but I wonder if it would be more obvious if the driver handled both?
Maybe this is an argument for a special setDFDLVariables function that is
called after construction to set all variables? Then the factory get just get
the lvr and create a Driver, and the driver handles both setting and getting
variables?
##########
daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/layers/LayerFactory.scala:
##########
@@ -0,0 +1,326 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.daffodil.runtime1.layers
+
+import java.lang.reflect.Constructor
+import java.lang.reflect.Method
+import scala.collection.immutable.ListSet
+import scala.collection.mutable
+
+import org.apache.daffodil.lib.exceptions.Assert
+import org.apache.daffodil.runtime1.dpath.NodeInfo.PrimType
+import org.apache.daffodil.runtime1.infoset.DataValue
+import org.apache.daffodil.runtime1.layers.api.Layer
+import org.apache.daffodil.runtime1.layers.api.LayerException
+import org.apache.daffodil.runtime1.processors.VariableRuntimeData
+
+object LayerFactory {
+
+ /** cache that maps spiName of layer to the LayerVarsRuntime */
+ private lazy val alreadyCheckedLayers =
+ new mutable.LinkedHashMap[String, LayerVarsRuntime]()
+
+ /**
+ * Computes the things that can be computed once-only to make calling the
constructor
+ * and passing the parameter variables faster at runtime. This also does all
the
+ * checking that the constructor has an argument for each variable of
properly matching type, and
+ * has a getter for each result variable, again returning the proper type.
+ *
+ * This is called at schema compile time to ensure the layer code is
properly defined and matches
+ * the layer variable definitions.
+ *
+ * It is called again at runtime after the Layer class is loaded by the SPI
+ * to ensure that the loaded layer class constructor signature at least
matches the layer
+ * variables defined in the schema.
+ * @param lrd
+ * @param protoLayer the layer instance allocated by the SPI loader
(zero-arg constructed)
+ * @return
+ */
+ def computeLayerVarsRuntime(lrd: LayerRuntimeData, protoLayer: Layer):
LayerVarsRuntime = {
+ val optLayerVarsRuntime = alreadyCheckedLayers.get(protoLayer.name())
+ optLayerVarsRuntime.getOrElse {
+ // we know there is a default zero arg constructor
+ // find another constructor and check that there is an argument
+ // corresponding to each of the layer variables.
+ val c = protoLayer.getClass
+ val allConstructors = c.getConstructors.toSeq
+
+ val constructor: Constructor[_] =
+ if (allConstructors.length == 1) {
+ // There is only the default constructor.
+ // That's ok if there are no variables for the layer, which we check
later.
+ allConstructors.head
+ } else if (allConstructors.length == 2) {
+ allConstructors.filter(_.getParameterCount > 0).head
+ } else {
+ def tooManyConstructorsMsg: String = {
+ s"""Layer class $c has multiple non-default constructors. It
should have a default (no args)
+ | constructor and a single additional constructor with
arguments for
+ | each of the layer's parameter variables.""".stripMargin
+ }
+ lrd.context.SDE(tooManyConstructorsMsg)
+ }
+
+ if (lrd.vmap.isEmpty && allConstructors.length == 1) {
+ // there are no vars, we're done
+ new LayerVarsRuntime(constructor, Nil, Nil)
+ } else {
+ // there is a constructor with args that are supposed to correspond to
bound vars
+ val params = constructor.getParameters.toSeq
+ val nParams = params.length
+ val nVars = lrd.vmap.size
+
+ val paramTypes = constructor.getParameterTypes.toSeq
+ val paramVRDs = params.map { p =>
+ lrd.vmap.getOrElse(
+ p.getName,
+ lrd.context.SDE(s"No layer DFDL variable named '$p.getName' was
found."),
+ )
+ }.toSeq
+
+ // Now we deal with the result getters and the corresponding vars
+ //
+ val allLayerVRDs = ListSet(lrd.vmap.toSeq.map(_._2): _*)
+ val returnVRDs = allLayerVRDs -- paramVRDs // set subtraction
+ val allMethods = c.getMethods
+ val allVarResultGetters =
+ ListSet(allMethods.filter { m =>
+ val nom = m.getName
+ nom.startsWith(varResultPrefix)
+ }.toSeq: _*)
+ // each returnVRD needs to have a corresponding getter method
+ val returnVRDNames = returnVRDs.map(_.globalQName.local)
+ val resultGettersNames =
allVarResultGetters.map(_.getName.replace(varResultPrefix, ""))
+ val nResultGetters = resultGettersNames.size
+ def javaConstructorArgs =
+ paramVRDs
+ .map {
+ case vrd => {
+ s"type: ${PrimType.toJavaTypeString(vrd.primType.dfdlType)}
name: ${vrd.globalQName.local}"
+ }
+ }
+ .mkString(", ")
+ def badConstructorMsg: String = {
+ s"""Layer class $c does not have a constructor with arguments for
each of the layer's variables.
+ | It should have a constructor with these arguments in any order,
such as
+ | ($javaConstructorArgs)""".stripMargin
+ }
+
+ lrd.context.schemaDefinitionUnless(nParams + nResultGetters == nVars,
badConstructorMsg)
+
+ // at this point the number of vars and number of constructor args
match
+
+ val typePairs = (paramVRDs.zip(paramTypes)).toSeq
+ typePairs.foreach { case (vrd, pt) =>
+ val vrdClass = PrimType.toJavaType(vrd.primType.dfdlType)
+ lrd.context.schemaDefinitionUnless(
+ vrdClass == pt,
+ s"""Layer constructor argument ${vrd.globalQName.local} and the
corresponding
+ |Layer DFDL variable have differing types: ${pt.getName}
+ | and ${vrdClass.getName} respectively.""".stripMargin,
+ )
+ }
+
+ val returnVRDsWithoutGetters = returnVRDNames -- resultGettersNames
+ val resultGettersWithoutVRDs = resultGettersNames -- returnVRDNames
+ lrd.context.schemaDefinitionUnless(
+ returnVRDsWithoutGetters.isEmpty,
+ s"""The layer variables ${returnVRDsWithoutGetters.mkString(
+ ",",
+ )} have no corresponding getters.""",
+ )
+ lrd.context.schemaDefinitionUnless(
+ resultGettersWithoutVRDs.isEmpty, {
+ val getterFullNames = returnVRDsWithoutGetters.map { vname =>
+ this.varResultPrefix + vname
+ }
+ s"""The getters ${getterFullNames.mkString(
+ ",",
+ )} have no corresponding layer variables."""
+ },
+ )
+ // at this point we know each variable that was not a parameter of the
constructor
+ // has a getter with matching name.
+ val resultVarPairs = resultGettersNames.map { rgn: String =>
+ val getter: Method =
+ allVarResultGetters.find { g: Method => g.getName.endsWith(rgn)
}.getOrElse {
+ Assert.invariantFailed("no getter for getter name.")
+ }
+ val vrd = returnVRDs.find { vrd => vrd.globalQName.local == rgn
}.getOrElse {
+ Assert.invariantFailed("no vrd for getter name.")
+ }
+ (vrd, getter)
+ }
+ resultVarPairs.foreach { case (vrd, getter) =>
+ val vrdClass = PrimType.toJavaType(vrd.primType.dfdlType)
+ val gt = getter.getReturnType
+ lrd.context.schemaDefinitionUnless(
+ vrdClass == gt,
+ s"""Layer return variable ${vrd.globalQName.local} and the
corresponding
+ |Layer getter have differing types: ${vrdClass.getName}
+ | and ${gt.getName} respectively.""".stripMargin,
+ )
Review Comment:
I think we need to make sure the getter has no parameters.
##########
daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/layers/LayerFactory.scala:
##########
@@ -0,0 +1,326 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.daffodil.runtime1.layers
+
+import java.lang.reflect.Constructor
+import java.lang.reflect.Method
+import scala.collection.immutable.ListSet
+import scala.collection.mutable
+
+import org.apache.daffodil.lib.exceptions.Assert
+import org.apache.daffodil.runtime1.dpath.NodeInfo.PrimType
+import org.apache.daffodil.runtime1.infoset.DataValue
+import org.apache.daffodil.runtime1.layers.api.Layer
+import org.apache.daffodil.runtime1.layers.api.LayerException
+import org.apache.daffodil.runtime1.processors.VariableRuntimeData
+
+object LayerFactory {
+
+ /** cache that maps spiName of layer to the LayerVarsRuntime */
+ private lazy val alreadyCheckedLayers =
+ new mutable.LinkedHashMap[String, LayerVarsRuntime]()
+
+ /**
+ * Computes the things that can be computed once-only to make calling the
constructor
+ * and passing the parameter variables faster at runtime. This also does all
the
+ * checking that the constructor has an argument for each variable of
properly matching type, and
+ * has a getter for each result variable, again returning the proper type.
+ *
+ * This is called at schema compile time to ensure the layer code is
properly defined and matches
+ * the layer variable definitions.
+ *
+ * It is called again at runtime after the Layer class is loaded by the SPI
+ * to ensure that the loaded layer class constructor signature at least
matches the layer
+ * variables defined in the schema.
+ * @param lrd
+ * @param protoLayer the layer instance allocated by the SPI loader
(zero-arg constructed)
+ * @return
+ */
+ def computeLayerVarsRuntime(lrd: LayerRuntimeData, protoLayer: Layer):
LayerVarsRuntime = {
+ val optLayerVarsRuntime = alreadyCheckedLayers.get(protoLayer.name())
+ optLayerVarsRuntime.getOrElse {
+ // we know there is a default zero arg constructor
+ // find another constructor and check that there is an argument
+ // corresponding to each of the layer variables.
+ val c = protoLayer.getClass
+ val allConstructors = c.getConstructors.toSeq
+
+ val constructor: Constructor[_] =
+ if (allConstructors.length == 1) {
+ // There is only the default constructor.
+ // That's ok if there are no variables for the layer, which we check
later.
+ allConstructors.head
+ } else if (allConstructors.length == 2) {
+ allConstructors.filter(_.getParameterCount > 0).head
+ } else {
+ def tooManyConstructorsMsg: String = {
+ s"""Layer class $c has multiple non-default constructors. It
should have a default (no args)
+ | constructor and a single additional constructor with
arguments for
+ | each of the layer's parameter variables.""".stripMargin
+ }
+ lrd.context.SDE(tooManyConstructorsMsg)
+ }
+
+ if (lrd.vmap.isEmpty && allConstructors.length == 1) {
+ // there are no vars, we're done
+ new LayerVarsRuntime(constructor, Nil, Nil)
+ } else {
+ // there is a constructor with args that are supposed to correspond to
bound vars
+ val params = constructor.getParameters.toSeq
+ val nParams = params.length
+ val nVars = lrd.vmap.size
+
+ val paramTypes = constructor.getParameterTypes.toSeq
+ val paramVRDs = params.map { p =>
+ lrd.vmap.getOrElse(
+ p.getName,
+ lrd.context.SDE(s"No layer DFDL variable named '$p.getName' was
found."),
+ )
+ }.toSeq
+
+ // Now we deal with the result getters and the corresponding vars
+ //
+ val allLayerVRDs = ListSet(lrd.vmap.toSeq.map(_._2): _*)
+ val returnVRDs = allLayerVRDs -- paramVRDs // set subtraction
+ val allMethods = c.getMethods
+ val allVarResultGetters =
+ ListSet(allMethods.filter { m =>
+ val nom = m.getName
+ nom.startsWith(varResultPrefix)
+ }.toSeq: _*)
+ // each returnVRD needs to have a corresponding getter method
+ val returnVRDNames = returnVRDs.map(_.globalQName.local)
+ val resultGettersNames =
allVarResultGetters.map(_.getName.replace(varResultPrefix, ""))
+ val nResultGetters = resultGettersNames.size
+ def javaConstructorArgs =
+ paramVRDs
+ .map {
+ case vrd => {
+ s"type: ${PrimType.toJavaTypeString(vrd.primType.dfdlType)}
name: ${vrd.globalQName.local}"
+ }
+ }
+ .mkString(", ")
+ def badConstructorMsg: String = {
+ s"""Layer class $c does not have a constructor with arguments for
each of the layer's variables.
+ | It should have a constructor with these arguments in any order,
such as
+ | ($javaConstructorArgs)""".stripMargin
+ }
+
+ lrd.context.schemaDefinitionUnless(nParams + nResultGetters == nVars,
badConstructorMsg)
+
+ // at this point the number of vars and number of constructor args
match
+
+ val typePairs = (paramVRDs.zip(paramTypes)).toSeq
+ typePairs.foreach { case (vrd, pt) =>
+ val vrdClass = PrimType.toJavaType(vrd.primType.dfdlType)
+ lrd.context.schemaDefinitionUnless(
+ vrdClass == pt,
+ s"""Layer constructor argument ${vrd.globalQName.local} and the
corresponding
+ |Layer DFDL variable have differing types: ${pt.getName}
+ | and ${vrdClass.getName} respectively.""".stripMargin,
+ )
+ }
+
+ val returnVRDsWithoutGetters = returnVRDNames -- resultGettersNames
+ val resultGettersWithoutVRDs = resultGettersNames -- returnVRDNames
+ lrd.context.schemaDefinitionUnless(
+ returnVRDsWithoutGetters.isEmpty,
+ s"""The layer variables ${returnVRDsWithoutGetters.mkString(
+ ",",
+ )} have no corresponding getters.""",
+ )
+ lrd.context.schemaDefinitionUnless(
+ resultGettersWithoutVRDs.isEmpty, {
+ val getterFullNames = returnVRDsWithoutGetters.map { vname =>
+ this.varResultPrefix + vname
+ }
+ s"""The getters ${getterFullNames.mkString(
+ ",",
+ )} have no corresponding layer variables."""
+ },
+ )
+ // at this point we know each variable that was not a parameter of the
constructor
+ // has a getter with matching name.
+ val resultVarPairs = resultGettersNames.map { rgn: String =>
+ val getter: Method =
+ allVarResultGetters.find { g: Method => g.getName.endsWith(rgn)
}.getOrElse {
Review Comment:
I think using `endsWith` could potentially find the wrong method if one
variable is a suffix of the other? For example, if we have two result getters:
```
getDFDLResultVariable_length
getDFDLResultVariable_maxlength
```
Finding the "length" getter could potentially return the maxlength getter.
Instead of endsWith, we should do `g.getName == varResultPrefix + rgn`.
##########
daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/LayeredSequenceParser.scala:
##########
@@ -17,56 +17,48 @@
package org.apache.daffodil.runtime1.processors.parsers
-import org.apache.daffodil.runtime1.layers.LayerExecutionException
-import org.apache.daffodil.runtime1.layers.LayerNotEnoughDataException
-import org.apache.daffodil.runtime1.layers.LayerRuntimeInfo
-import org.apache.daffodil.runtime1.layers.LayerTransformerFactory
+import org.apache.daffodil.runtime1.layers.LayerFactory
+import org.apache.daffodil.runtime1.layers.LayerRuntimeImpl
+import org.apache.daffodil.runtime1.layers.api.LayerUnexpectedException
import org.apache.daffodil.runtime1.processors.SequenceRuntimeData
class LayeredSequenceParser(
rd: SequenceRuntimeData,
- layerTransformerFactory: LayerTransformerFactory,
- layerRuntimeInfo: LayerRuntimeInfo,
+ layerFactory: LayerFactory,
bodyParser: SequenceChildParser,
) extends OrderedUnseparatedSequenceParser(rd, Vector(bodyParser)) {
override def nom = "LayeredSequence"
- override lazy val runtimeDependencies =
- layerRuntimeInfo.evaluatables.toVector
-
override def parse(state: PState): Unit = {
- val layerTransformer =
layerTransformerFactory.newInstance(layerRuntimeInfo)
+ // TODO: Separate the creation of layer into layerParse and layer
+ // unparse. Right now they're blended onto one object.
+ // It should be possible to define only a layer parser if a schema (and
its required layers)
+ // are intended to be used only to parse data.
+
+ val layerRuntimeImpl = new LayerRuntimeImpl(state, rd.layerRuntimeData)
+ val layerDriver = layerFactory.newInstance(layerRuntimeImpl)
val savedDIS = state.dataInputStream
- val isAligned =
savedDIS.align(layerTransformer.mandatoryLayerAlignmentInBits, state)
+ val isAligned = savedDIS.align(layerDriver.mandatoryLayerAlignmentInBits,
state)
if (!isAligned)
PE(
state,
"Unable to align to the mandatory layer alignment of %s(bits)",
- layerTransformer.mandatoryLayerAlignmentInBits,
+ layerDriver.mandatoryLayerAlignmentInBits,
)
try {
- val newDIS = layerTransformer.addLayer(savedDIS, state)
+ val newDIS = layerDriver.addInputLayer(savedDIS, layerRuntimeImpl)
state.dataInputStream = newDIS
- layerTransformer.startLayerForParse(state)
super.parse(state)
- layerTransformer.removeLayer(newDIS)
+ layerDriver.removeInputLayer(newDIS, layerRuntimeImpl)
} catch {
- case le: LayerNotEnoughDataException =>
- PENotEnoughBits(
- state,
- le.schemaFileLocation,
- le.dataLocation,
- le.nBytesRequired * 8,
- )
+ case pe: ParseError =>
+ state.setFailed(pe)
case e: Exception =>
Review Comment:
Could a runtime SDE be caught here? Should that be treated differently than
LayerUnexpectedException?
##########
daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/processors/VariableMap1.scala:
##########
@@ -278,8 +280,8 @@ class VariableMap private (vrds: Seq[VariableRuntimeData],
vTable: Array[Seq[Var
}
def cloneForSuspension(): VariableMap = {
- val newTable = new Array[Seq[VariableInstance]](vTable.size)
- Array.copy(vTable, 0, newTable, 0, vTable.size)
+ val newTable = new Array[Seq[VariableInstance]](vTable.length)
+ Array.copy(vTable, 0, newTable, 0, vTable.length)
new VariableMap(vrds, newTable)
Review Comment:
Is here a different between size vs length?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
