This is an automated email from the ASF dual-hosted git repository.
slawrence pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-daffodil.git
The following commit(s) were added to refs/heads/master by this push:
new 0eba8b8 Correctly detect zero length status for "dead"
DataOutputStreams
0eba8b8 is described below
commit 0eba8b874c6b31ecefd3e30880f184500cf69332
Author: Steve Lawrence <[email protected]>
AuthorDate: Mon Nov 16 08:18:05 2020 -0500
Correctly detect zero length status for "dead" DataOutputStreams
When unparsing postfix separators, we create split regions to determine
if a data region is zero length or not, which is used to help decided
whether or not to output a postfix separator. If these regions are
"finished" without anything ever writing to them, then we know the
region is zero length. But if one of these regions is finished and
eventually becomes "dead" (i.e. its following DOS becomes direct), it is
possible for the zero length status to have never been set. This leaves
the zero length status in an "unknown" state which can lead to locked
suspensions.
This change fixes this so if a DOS is either finished *or dead* and the
zero length status has never been set, then we can assume that region
had zero length, thus allowing suspensions to resolve.
DAFFODIL-2431
---
.../daffodil/io/DataOutputStreamImplMixin.scala | 13 +-
.../org/apache/daffodil/usertests/test-csv.tdml | 150 +++++++++++++++++++++
.../org/apache/daffodil/usertests/TestCSV.scala | 43 ++++++
3 files changed, 200 insertions(+), 6 deletions(-)
diff --git
a/daffodil-io/src/main/scala/org/apache/daffodil/io/DataOutputStreamImplMixin.scala
b/daffodil-io/src/main/scala/org/apache/daffodil/io/DataOutputStreamImplMixin.scala
index 84949a0..2ae1298 100644
---
a/daffodil-io/src/main/scala/org/apache/daffodil/io/DataOutputStreamImplMixin.scala
+++
b/daffodil-io/src/main/scala/org/apache/daffodil/io/DataOutputStreamImplMixin.scala
@@ -84,13 +84,14 @@ trait DataOutputStreamImplMixin extends
DataStreamCommonState
case Zero => // ok
case NonZero => // ok
case Unknown => {
- if (this.isFinished) {
+ if (this.isFinished || this.isDead) {
+ // this DOS was finished or this DOS is dead (i.e. was finished,
+ // became direct and then the following DOS become direct). In either
+ // case, we know that nothing has written to this DOS or the
+ // zlStatus_ would have been set by setNonZeroLength() call, which is
+ // required to be called by everything that writes.
//
- // nothing has written to this, or the
- // zlStatus would have been set by setNonZeroLength() call,
- // which is required to be called by everything that writes.
- //
- // So regardless of whether this stream has been merged/collapsed,
etc.
+ // So regardless of whether this stream has been
merged/collapsed/dead, etc.
// we know that no actual writes occurred to this DOS, so it is zero
length.
// And now that it is finsihed, that can never change.
zlStatus_ = Zero
diff --git
a/daffodil-test/src/test/resources/org/apache/daffodil/usertests/test-csv.tdml
b/daffodil-test/src/test/resources/org/apache/daffodil/usertests/test-csv.tdml
new file mode 100644
index 0000000..5091880
--- /dev/null
+++
b/daffodil-test/src/test/resources/org/apache/daffodil/usertests/test-csv.tdml
@@ -0,0 +1,150 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<tdml:testSuite suiteName="user-csv"
+ xmlns:tdml="http://www.ibm.com/xmlns/dfdl/testData";
+ xmlns:tns="http://example.com";
+ xmlns:dfdl="http://www.ogf.org/dfdl/dfdl-1.0/";
+ xmlns:dfdlx="http://www.ogf.org/dfdl/dfdl-1.0/extensions";
+ xmlns:xs="http://www.w3.org/2001/XMLSchema";
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
+ defaultRoundTrip="true">
+
+ <tdml:defineSchema name="csv">
+
+ <xs:include
schemaLocation="org/apache/daffodil/xsd/DFDLGeneralFormat.dfdl.xsd"/>
+
+ <dfdl:format ref="tns:GeneralFormat"
+ lengthKind="delimited"
+ occursCountKind="implicit"/>
+
+ <xs:element name="csv">
+ <xs:complexType>
+ <xs:sequence dfdl:separator="%NL;" dfdl:separatorPosition="postfix">
+ <xs:element name="record" maxOccurs="unbounded">
+ <xs:complexType>
+ <xs:sequence dfdl:separator=",">
+ <xs:element name="item" type="xs:string"
maxOccurs="unbounded"/>
+ </xs:sequence>
+ </xs:complexType>
+ </xs:element>
+ </xs:sequence>
+ </xs:complexType>
+ </xs:element>
+
+ </tdml:defineSchema>
+
+ <tdml:parserTestCase name="csv_01" model="csv" roundTrip="false">
+ <tdml:document>
+ <tdml:documentPart type="text"
replaceDFDLEntities="true"><![CDATA[1,2,3%LF;%LF;4,5,6%LF;]]></tdml:documentPart>
+ </tdml:document>
+ <tdml:infoset>
+ <tdml:dfdlInfoset>
+ <tns:csv xmlns:tns="http://example.com";>
+ <tns:record>
+ <tns:item>1</tns:item>
+ <tns:item>2</tns:item>
+ <tns:item>3</tns:item>
+ </tns:record>
+ <tns:record>
+ <tns:item></tns:item>
+ </tns:record>
+ <tns:record>
+ <tns:item>4</tns:item>
+ <tns:item>5</tns:item>
+ <tns:item>6</tns:item>
+ </tns:record>
+ </tns:csv>
+ </tdml:dfdlInfoset>
+ </tdml:infoset>
+ </tdml:parserTestCase>
+
+ <tdml:unparserTestCase name="csv_02" model="csv" roundTrip="false">
+ <tdml:document>
+ <tdml:documentPart type="text"
replaceDFDLEntities="true"><![CDATA[1,2,3%LF;4,5,6%LF;]]></tdml:documentPart>
+ </tdml:document>
+ <tdml:infoset>
+ <tdml:dfdlInfoset>
+ <tns:csv xmlns:tns="http://example.com";>
+ <tns:record>
+ <tns:item>1</tns:item>
+ <tns:item>2</tns:item>
+ <tns:item>3</tns:item>
+ </tns:record>
+ <tns:record>
+ <tns:item></tns:item>
+ </tns:record>
+ <tns:record>
+ <tns:item>4</tns:item>
+ <tns:item>5</tns:item>
+ <tns:item>6</tns:item>
+ </tns:record>
+ </tns:csv>
+ </tdml:dfdlInfoset>
+ </tdml:infoset>
+ </tdml:unparserTestCase>
+
+ <tdml:parserTestCase name="csv_03" model="csv" roundTrip="false">
+ <tdml:document>
+ <tdml:documentPart type="text"
replaceDFDLEntities="true"><![CDATA[,,%LF;,,%LF;,,1,2,3%LF;]]></tdml:documentPart>
+ </tdml:document>
+ <tdml:infoset>
+ <tdml:dfdlInfoset>
+ <tns:csv xmlns:tns="http://example.com";>
+ <tns:record>
+ <tns:item></tns:item>
+ </tns:record>
+ <tns:record>
+ <tns:item></tns:item>
+ </tns:record>
+ <tns:record>
+ <tns:item></tns:item>
+ <tns:item>1</tns:item>
+ <tns:item>2</tns:item>
+ <tns:item>3</tns:item>
+ </tns:record>
+ </tns:csv>
+ </tdml:dfdlInfoset>
+ </tdml:infoset>
+ </tdml:parserTestCase>
+
+ <tdml:unparserTestCase name="csv_04" model="csv" roundTrip="false">
+ <tdml:document>
+ <tdml:documentPart type="text"
replaceDFDLEntities="true"><![CDATA[,1,2,3%LF;]]></tdml:documentPart>
+ </tdml:document>
+ <tdml:infoset>
+ <tdml:dfdlInfoset>
+ <tns:csv xmlns:tns="http://example.com";>
+ <tns:record>
+ <tns:item></tns:item>
+ </tns:record>
+ <tns:record>
+ <tns:item></tns:item>
+ </tns:record>
+ <tns:record>
+ <tns:item></tns:item>
+ <tns:item>1</tns:item>
+ <tns:item>2</tns:item>
+ <tns:item>3</tns:item>
+ </tns:record>
+ </tns:csv>
+ </tdml:dfdlInfoset>
+ </tdml:infoset>
+ </tdml:unparserTestCase>
+
+</tdml:testSuite>
diff --git
a/daffodil-test/src/test/scala/org/apache/daffodil/usertests/TestCSV.scala
b/daffodil-test/src/test/scala/org/apache/daffodil/usertests/TestCSV.scala
new file mode 100644
index 0000000..c2f9251
--- /dev/null
+++ b/daffodil-test/src/test/scala/org/apache/daffodil/usertests/TestCSV.scala
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.daffodil.usertests
+
+import org.junit.Test
+import org.apache.daffodil.tdml.Runner
+import org.junit.AfterClass
+
+object TestCSV {
+ val testDir = "/org/apache/daffodil/usertests/"
+ val runner = Runner(testDir, "test-csv.tdml")
+
+ @AfterClass def shutDown: Unit = {
+ runner.reset
+ }
+
+}
+
+class TestCSV {
+
+ import TestCSV._
+
+ @Test def test_csv_01(): Unit = { runner.runOneTest("csv_01") }
+ @Test def test_csv_02(): Unit = { runner.runOneTest("csv_02") }
+ @Test def test_csv_03(): Unit = { runner.runOneTest("csv_03") }
+ @Test def test_csv_04(): Unit = { runner.runOneTest("csv_04") }
+
+}
