This is an automated email from the ASF dual-hosted git repository.
slawrence pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/daffodil.git
The following commit(s) were added to refs/heads/main by this push:
new 3ae1ef479 Allow changing blob attributes when full/custom validation
is enabled
3ae1ef479 is described below
commit 3ae1ef479f0530322ab20f0111b70c47fa5430c6
Author: Steve Lawrence <[email protected]>
AuthorDate: Fri Aug 4 11:09:03 2023 -0400
Allow changing blob attributes when full/custom validation is enabled
When full or custom validation is enabled, Daffodil creates a new
TeeInfosetOutputter which proxies infoset events to the user provided
InfosetOutputter and another one used for validation. This means that
calls to getBlobDirectory, getBlobSuffix, setBlobPath, etc. go to the
TeeInfosetOutputter, and ignore any values that a user may have set on
their provided InfosetOutputter. This essentially means you cannot
change the directory, suffix, or prefix of blobs if full or custom
validation is enabled. It also means that if calling getBlobPaths on the
user provided InfosetOutputter never returns anything, even if blobs
were created.
To fix this, if full or custom validation is enabled, we copy the blob
attributes to the TeeInfosetOutputter used for parsing. We also ensure
that at the end of parsing, we call setBlobPaths on the InfosetOutputter
the user passed in instead of the TeeInfosetOutputter.
DAFFODIL-2837
---
.../org/apache/daffodil/example/TestJavaAPI.java | 38 ++++++++++++++++++++++
.../{blob_backtracking.dfdl.xsd => blob.dfdl.xsd} | 20 ++++--------
.../runtime1/processors/DataProcessor.scala | 14 ++++++--
.../src/test/resources/test/sapi/blob.dfdl.xsd | 20 ++++--------
.../org/apache/daffodil/example/TestScalaAPI.scala | 35 ++++++++++++++++++++
5 files changed, 96 insertions(+), 31 deletions(-)
diff --git
a/daffodil-japi/src/test/java/org/apache/daffodil/example/TestJavaAPI.java
b/daffodil-japi/src/test/java/org/apache/daffodil/example/TestJavaAPI.java
index 9f4be4a7b..1ad39bd5c 100644
--- a/daffodil-japi/src/test/java/org/apache/daffodil/example/TestJavaAPI.java
+++ b/daffodil-japi/src/test/java/org/apache/daffodil/example/TestJavaAPI.java
@@ -35,7 +35,9 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
+import java.util.Iterator;
import java.util.List;
+import scala.collection.JavaConverters;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
@@ -1308,4 +1310,40 @@ public class TestJavaAPI {
assertFalse(err);
assertEquals(expect, value);
}
+
+ @Test
+ public void testJavaAPIBlob1() throws IOException, ClassNotFoundException,
InvalidUsageException {
+ org.apache.daffodil.japi.Compiler c = Daffodil.compiler();
+ java.io.File schemaFile = getResource("/test/japi/blob.dfdl.xsd");
+ ProcessorFactory pf = c.compileFile(schemaFile);
+ DataProcessor dp = pf.onPath("/");
+ dp = dp.withValidationMode(ValidationMode.Full);
+
+ byte[] data = new byte[] { 0x00, 0x00, 0x00, 0x04, 0x01, 0x02, 0x03,
0x04 };
+ ByteArrayInputStream bis = new ByteArrayInputStream(data);
+ InputSourceDataInputStream input = new
InputSourceDataInputStream(data);
+
+ Path blobRoot = Paths.get(System.getProperty("java.io.tmpdir"),
"daffodil", "japi");
+ Files.createDirectories(blobRoot);
+ Path blobDir = Files.createTempDirectory(blobRoot, "blob-");
+
+ ByteArrayOutputStream bos = new ByteArrayOutputStream();
+ XMLTextInfosetOutputter output = new XMLTextInfosetOutputter(bos,
true);
+ output.setBlobAttributes(blobDir, "pre-", ".suf");
+
+ ParseResult res = dp.parse(input, output);
+ List<Path> blobPaths =
JavaConverters.seqAsJavaList(output.getBlobPaths());
+
+ try {
+ assertFalse(res.isError());
+ assertTrue(blobPaths.size() == 1);
+ assertTrue(blobPaths.get(0).toString().contains("blob-"));
+ assertTrue(blobPaths.get(0).toString().contains("pre-"));
+ assertTrue(blobPaths.get(0).toString().contains(".suf"));
+ } finally {
+ Iterator<Path> pathIter = blobPaths.iterator();
+ while (pathIter.hasNext()) Files.delete(pathIter.next());
+ Files.delete(blobDir);
+ }
+ }
}
diff --git
a/daffodil-japi/src/test/resources/test/japi/blob_backtracking.dfdl.xsd
b/daffodil-japi/src/test/resources/test/japi/blob.dfdl.xsd
similarity index 72%
copy from daffodil-japi/src/test/resources/test/japi/blob_backtracking.dfdl.xsd
copy to daffodil-japi/src/test/resources/test/japi/blob.dfdl.xsd
index 4bc3c2f2e..842b378ef 100644
--- a/daffodil-japi/src/test/resources/test/japi/blob_backtracking.dfdl.xsd
+++ b/daffodil-japi/src/test/resources/test/japi/blob.dfdl.xsd
@@ -27,25 +27,17 @@
<annotation>
<appinfo source="http://www.ogf.org/dfdl/";>
<dfdl:format ref="tns:GeneralFormat"
- representation="binary"
- encodingErrorPolicy="replace" />
+ representation="binary" />
</appinfo>
</annotation>
<xs:element name="root">
<xs:complexType>
- <xs:choice>
- <xs:sequence>
- <xs:element name="b" type="xs:anyURI"
- dfdl:lengthKind="explicit" dfdl:length="2098176008"
- dfdlx:objectKind="bytes" />
- <xs:element name="a" type="xs:string"
- dfdl:lengthKind="explicit" dfdl:length="1" />
- </xs:sequence>
- <xs:element name="oneGigBlob" type="xs:anyURI"
- dfdl:lengthKind="explicit" dfdl:length="2098176008"
- dfdlx:objectKind="bytes" />
- </xs:choice>
+ <xs:sequence>
+ <xs:element name="length" type="xs:int" />
+ <xs:element name="blob" type="xs:anyURI" dfdlx:objectKind="bytes"
+ dfdl:lengthKind="explicit" dfdl:length="{ ../length }" />
+ </xs:sequence>
</xs:complexType>
</xs:element>
diff --git
a/daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/processors/DataProcessor.scala
b/daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/processors/DataProcessor.scala
index efedbdb8b..633b50054 100644
---
a/daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/processors/DataProcessor.scala
+++
b/daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/processors/DataProcessor.scala
@@ -356,6 +356,13 @@ class DataProcessor(
val bos = new java.io.ByteArrayOutputStream()
val xmlOutputter = new XMLTextInfosetOutputter(bos, false)
val teeOutputter = new TeeInfosetOutputter(output, xmlOutputter)
+ // copy the blob attributes from the users outputter to the tee
infoset outputter
+ // since Daffodil will now use that to get blob attributes
+ teeOutputter.setBlobAttributes(
+ output.getBlobDirectory(),
+ output.getBlobPrefix(),
+ output.getBlobSuffix(),
+ )
(teeOutputter, One(bos))
case _ =>
(output, Nope)
@@ -387,15 +394,16 @@ class DataProcessor(
}
res
}
- state.output.setBlobPaths(state.blobPaths)
+ // copy the blob paths we created to the users infoset outputter
+ output.setBlobPaths(state.blobPaths)
new ParseResult(state, vr)
} else {
// failed, so delete all blobs that were created
state.blobPaths.foreach { path =>
Files.delete(path)
}
- // ensure the blob paths are empty in case of outputter reuse
- state.output.setBlobPaths(Seq.empty)
+ // ensure the blob paths on the users infoset outputter are empty in
case of reuse
+ output.setBlobPaths(Seq.empty)
new ParseResult(state, None)
}
val s = state
diff --git
a/daffodil-japi/src/test/resources/test/japi/blob_backtracking.dfdl.xsd
b/daffodil-sapi/src/test/resources/test/sapi/blob.dfdl.xsd
similarity index 72%
rename from
daffodil-japi/src/test/resources/test/japi/blob_backtracking.dfdl.xsd
rename to daffodil-sapi/src/test/resources/test/sapi/blob.dfdl.xsd
index 4bc3c2f2e..842b378ef 100644
--- a/daffodil-japi/src/test/resources/test/japi/blob_backtracking.dfdl.xsd
+++ b/daffodil-sapi/src/test/resources/test/sapi/blob.dfdl.xsd
@@ -27,25 +27,17 @@
<annotation>
<appinfo source="http://www.ogf.org/dfdl/";>
<dfdl:format ref="tns:GeneralFormat"
- representation="binary"
- encodingErrorPolicy="replace" />
+ representation="binary" />
</appinfo>
</annotation>
<xs:element name="root">
<xs:complexType>
- <xs:choice>
- <xs:sequence>
- <xs:element name="b" type="xs:anyURI"
- dfdl:lengthKind="explicit" dfdl:length="2098176008"
- dfdlx:objectKind="bytes" />
- <xs:element name="a" type="xs:string"
- dfdl:lengthKind="explicit" dfdl:length="1" />
- </xs:sequence>
- <xs:element name="oneGigBlob" type="xs:anyURI"
- dfdl:lengthKind="explicit" dfdl:length="2098176008"
- dfdlx:objectKind="bytes" />
- </xs:choice>
+ <xs:sequence>
+ <xs:element name="length" type="xs:int" />
+ <xs:element name="blob" type="xs:anyURI" dfdlx:objectKind="bytes"
+ dfdl:lengthKind="explicit" dfdl:length="{ ../length }" />
+ </xs:sequence>
</xs:complexType>
</xs:element>
diff --git
a/daffodil-sapi/src/test/scala/org/apache/daffodil/example/TestScalaAPI.scala
b/daffodil-sapi/src/test/scala/org/apache/daffodil/example/TestScalaAPI.scala
index 5e9a2092f..14b31031b 100644
---
a/daffodil-sapi/src/test/scala/org/apache/daffodil/example/TestScalaAPI.scala
+++
b/daffodil-sapi/src/test/scala/org/apache/daffodil/example/TestScalaAPI.scala
@@ -25,6 +25,7 @@ import java.io.ObjectOutputStream
import java.nio.ByteBuffer
import java.nio.channels.Channels
import java.nio.charset.StandardCharsets
+import java.nio.file.Files
import java.nio.file.Paths
import javax.xml.XMLConstants
@@ -1290,4 +1291,38 @@ class TestScalaAPI {
assertFalse(err)
assertEquals(expect, value)
}
+
+ @Test
+ def testScalaAPIBlob1(): Unit = {
+ val c = Daffodil.compiler()
+ val schemaFile = getResource("/test/sapi/blob.dfdl.xsd")
+ val pf = c.compileFile(schemaFile)
+ val dp = pf.onPath("/").withValidationMode(ValidationMode.Full)
+
+ val data = Array[Byte](0x00, 0x00, 0x00, 0x04, 0x01, 0x02, 0x03, 0x04)
+ val bis = new ByteArrayInputStream(data)
+ val input = new InputSourceDataInputStream(data)
+
+ val blobRoot = Paths.get(System.getProperty("java.io.tmpdir"), "daffodil",
"sapi")
+ Files.createDirectories(blobRoot)
+ val blobDir = Files.createTempDirectory(blobRoot, "blob-")
+
+ val bos = new ByteArrayOutputStream()
+ val output = new XMLTextInfosetOutputter(bos, true)
+ output.setBlobAttributes(blobDir, "pre-", ".suf")
+
+ val res = dp.parse(input, output)
+ val blobPaths = output.getBlobPaths()
+
+ try {
+ assertFalse(res.isError)
+ assertTrue(blobPaths.length == 1)
+ assertTrue(blobPaths(0).toString().contains("blob-"))
+ assertTrue(blobPaths(0).toString().contains("pre-"))
+ assertTrue(blobPaths(0).toString().contains(".suf"))
+ } finally {
+ blobPaths.foreach(Files.delete)
+ Files.delete(blobDir)
+ }
+ }
}
