This is an automated email from the ASF dual-hosted git repository.
mbeckerle pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/daffodil.git
The following commit(s) were added to refs/heads/main by this push:
new dd9e836 Added test to confirm delimiter behavior.
dd9e836 is described below
commit dd9e836ba88c1598e8562d986ae6d63152693b64
Author: Michael Beckerle <[email protected]>
AuthorDate: Mon Jan 31 08:59:04 2022 -0500
Added test to confirm delimiter behavior.
DAFFODIL-2639
---
.../runtime1/TestDelimiterFinalBacktracking.scala | 89 ++++++++++++++++++
.../scala/org/apache/daffodil/util/TestUtils.scala | 103 ++++++++++++++++++++-
2 files changed, 187 insertions(+), 5 deletions(-)
diff --git
a/daffodil-core/src/test/scala/org/apache/daffodil/runtime1/TestDelimiterFinalBacktracking.scala
b/daffodil-core/src/test/scala/org/apache/daffodil/runtime1/TestDelimiterFinalBacktracking.scala
new file mode 100644
index 0000000..b6508fb
--- /dev/null
+++
b/daffodil-core/src/test/scala/org/apache/daffodil/runtime1/TestDelimiterFinalBacktracking.scala
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.daffodil.runtime1
+
+import org.apache.daffodil.util.SchemaUtils
+import org.apache.daffodil.util.StreamParser
+import org.junit.Test
+import org.apache.daffodil.xml.XMLUtils
+import org.junit.Assert.assertEquals
+
+class TestDelimiterFinalBacktracking {
+
+ val xsd = XMLUtils.XSD_NAMESPACE
+ val dfdl = XMLUtils.dfdlAppinfoSource // XMLUtils.DFDL_NAMESPACE
+ val xsi = XMLUtils.XSI_NAMESPACE
+ val example = XMLUtils.EXAMPLE_NAMESPACE
+
+ val schema1 =
+ <xs:element name="r" dfdl:lengthKind="implicit">
+ <xs:complexType>
+ <xs:sequence>
+ <xs:element name="a" type="xs:string"
+ dfdl:lengthKind="delimited"
+ dfdl:terminator="long longer"
+ dfdl:ignoreCase="no" />
+ </xs:sequence>
+ </xs:complexType>
+ </xs:element>
+
+ /**
+ * This test shows that even if Daffodil speculates forward to see if a
longer
+ * delimiter is present, but it isn't, it resets the
InputSourceDataInputStream
+ * to the position after the last character of the delimiter that was found.
+ *
+ * Daffodil has to speculate to see if the delimiter is "longer", not just
"long", so
+ * it has to read the "e" and "R" characters to know that the first
delimiter is just "long".
+ *
+ * If the input stream was a TCP network socket, then Daffodil would need to
read the two
+ * bytes ("eR") past the first "long", and so would block waiting for those
bytes to become available.
+ * But it does not consume them from its own InputSourceDataInputStream,
which is what this test
+ * shows. Rather, daffodil looks at them, but backtracks them so they are
not consumed.
+ */
+ @Test def testParseTerminatorBacktrack1(): Unit = {
+ val testSchema = SchemaUtils.dfdlTestSchema(
+ <xs:include
schemaLocation="org/apache/daffodil/xsd/DFDLGeneralFormat.dfdl.xsd"/>,
+ <dfdl:format ref="tns:GeneralFormat"/>,
+ schema1,
+ elementFormDefault = "unqualified")
+
+ //
+ // Baseline behavior
+ //
+ {
+ val data = "AlongerBlong" // we should get "A" then "B"
+ val res = StreamParser.doStreamTest(testSchema, data)
+ val firstBytePos1b = ((res(0).bitPos1b - 1) / 8).toInt
+ assertEquals(7, firstBytePos1b)
+ assertEquals("Alonger", data.substring(0, firstBytePos1b))
+ assertEquals("A", (res(0).message \\ "a").text)
+ assertEquals("B", (res(1).message \\ "a").text)
+ }
+ //
+ // Now the actual test
+ //
+ val data = "AlongeRBlong" // we should get "A" then "eRB"
+ val res = StreamParser.doStreamTest(testSchema, data)
+ val firstBytePos1b = ((res(0).bitPos1b - 1) / 8).toInt
+ assertEquals(5, firstBytePos1b)
+ assertEquals("Along", data.substring(0, firstBytePos1b))
+ assertEquals("A", (res(0).message \\ "a").text)
+ assertEquals("eRB", (res(1).message \\ "a").text)
+ }
+
+}
diff --git
a/daffodil-core/src/test/scala/org/apache/daffodil/util/TestUtils.scala
b/daffodil-core/src/test/scala/org/apache/daffodil/util/TestUtils.scala
index c4f5343..ad48517 100644
--- a/daffodil-core/src/test/scala/org/apache/daffodil/util/TestUtils.scala
+++ b/daffodil-core/src/test/scala/org/apache/daffodil/util/TestUtils.scala
@@ -23,12 +23,9 @@ import java.nio.channels.Channels
import java.nio.channels.ReadableByteChannel
import java.nio.channels.WritableByteChannel
import scala.util.Try
-import scala.xml._
import org.apache.commons.io.output.NullOutputStream
import org.junit.Assert.assertEquals
import org.apache.daffodil.Implicits._
-
-import java.io.InputStream
import org.apache.daffodil.api.DFDL
import org.apache.daffodil.api._
import org.apache.daffodil.compiler.Compiler
@@ -39,13 +36,21 @@ import org.apache.daffodil.grammar.VariableMapFactory
import org.apache.daffodil.infoset.InfosetInputter
import org.apache.daffodil.infoset.InfosetOutputter
import org.apache.daffodil.infoset.ScalaXMLInfosetInputter
-import org.apache.daffodil.infoset.ScalaXMLInfosetOutputter
-import org.apache.daffodil.io.InputSourceDataInputStream
import org.apache.daffodil.processors.DataProcessor
import org.apache.daffodil.processors.VariableMap
import org.apache.daffodil.xml.XMLUtils
import org.apache.daffodil.xml._
+import java.io.ByteArrayInputStream
+import org.apache.daffodil.infoset.ScalaXMLInfosetOutputter
+import org.apache.daffodil.io.InputSourceDataInputStream
+
+import java.io.InputStream
+import scala.collection.mutable.ArrayBuffer
+import scala.xml._
+
+
+
object INoWarnU2 { ImplicitsSuppressUnusedImportWarning() }
/*
@@ -371,3 +376,91 @@ class Fakes private () {
lazy val fakeDP = new FakeDataProcessor
}
+
+/**
+ * Testing class for streaming message parse behavior
+ */
+object StreamParser {
+ case class CompileFailure (diags: Seq[Diagnostic]) extends Exception("DFDL
Schema Compile Failure"){
+ override def getMessage() = diags.map{ _.toString }.mkString(",\n")
+ }
+
+ /**
+ * Result object for parse calls. Just a tuple.
+ */
+ case class Result (message: Node, // document that is the current parse
result, or null
+ diags: Seq[Diagnostic], // diagnostics.
+ isProcessingError: Boolean,
+ isValidationError: Boolean,
+ bitPos1b: Long) {
+
+ def toXML: Node = {
+ <Result>
+ { message }
+ { if (!diags.isEmpty) {
+ <diagnostics>
+ { diags map { diag => <diagnostic>{ diag.toString} </diagnostic> } }
+ </diagnostics>
+ }
+ else Null
+ }
+ </Result> %
+ (if (isProcessingError) new
UnprefixedAttribute("isProcessingError",isProcessingError.toString, Null) else
Null) %
+ (if (isValidationError) new UnprefixedAttribute("isValidationError",
isValidationError.toString, Null) else Null) %
+ new UnprefixedAttribute("bitPos1b", bitPos1b.toString, Null)
+ }
+ }
+
+
+ def doStreamTest(schema: Node, data: String): Seq[Result] = {
+ val mp = new StreamParser(schema)
+ val is: InputStream = new ByteArrayInputStream(data.getBytes("ascii"))
+ mp.setInputStream(is)
+ var r: StreamParser.Result = null
+ val results = new ArrayBuffer[Result]
+ val resStream = Stream.continually( mp.parse ).takeWhile( r =>
!r.isProcessingError)
+ resStream.toSeq
+ }
+}
+
+class StreamParser(val schema: Node) {
+
+ val outputter = new ScalaXMLInfosetOutputter()
+ var dis: InputSourceDataInputStream = _
+ var dp: DFDL.DataProcessor = _
+ //
+ // First compile the DFDL Schema
+ val c = Compiler()
+ val pf = c.compileNode(schema)
+ val pfDiags = pf.getDiagnostics
+ if (pf.isError) throw new StreamParser.CompileFailure(pfDiags)
+ dp = pf.onPath("/")
+ .withValidationMode(ValidationMode.Full)
+ // .withDebuggerRunner(new TraceDebuggerRunner()) // DAFFODIL-2624 - cannot
trace in streaming SAPI
+ // .withDebugging(true)
+ val dpDiags = dp.getDiagnostics
+ if (dp.isError) throw new StreamParser.CompileFailure(dpDiags)
+ val compilationWarnings = if (!pfDiags.isEmpty) pfDiags else dpDiags //
dpDiags might be empty. That's ok.
+
+ def setInputStream(inputStream: InputStream): Unit = {
+ dis = InputSourceDataInputStream(inputStream)
+ }
+
+ /**
+ * Called to pull messages from the data stream.
+ *
+ * @return a Result object containing the results of the parse including
diagnostic information.
+ */
+ def parse = {
+ if (dis == null) throw new IllegalStateException("Input stream must be
provided by setInputStream() call.")
+ val res: DFDL.ParseResult = dp.parse(dis, outputter)
+ val procErr = res.isProcessingError
+ val validationErr = res.isValidationError
+ val diags = res.getDiagnostics
+ val doc = if (!procErr) outputter.getResult else null
+ val bitPos1b = res.resultState.currentLocation.bitPos1b
+ val r = new StreamParser.Result(doc, diags, procErr, validationErr,
bitPos1b)
+ outputter.reset()
+ r
+ }
+}
