This is an automated email from the ASF dual-hosted git repository.
mthomsen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/master by this push:
new a5dff7b NIFI-6096: Fixed ValidateRecord handling of nested maps
a5dff7b is described below
commit a5dff7b45cbba788cc499db301315a306f7a8b5c
Author: Matthew Burgess <[email protected]>
AuthorDate: Tue Apr 9 13:49:12 2019 -0400
NIFI-6096: Fixed ValidateRecord handling of nested maps
This closes #3424
Signed-off-by: Mike Thomsen <[email protected]>
---
.../serialization/record/util/DataTypeUtils.java | 2 +-
.../nifi-standard-processors/pom.xml | 2 +
.../processors/standard/TestValidateRecord.java | 48 ++++++++++++++++++++++
.../TestValidateRecord/nested-map-input.json | 18 ++++++++
.../TestValidateRecord/nested-map-schema.avsc | 32 +++++++++++++++
5 files changed, 101 insertions(+), 1 deletion(-)
diff --git
a/nifi-commons/nifi-record/src/main/java/org/apache/nifi/serialization/record/util/DataTypeUtils.java
b/nifi-commons/nifi-record/src/main/java/org/apache/nifi/serialization/record/util/DataTypeUtils.java
index 3411b13..14782a4 100644
---
a/nifi-commons/nifi-record/src/main/java/org/apache/nifi/serialization/record/util/DataTypeUtils.java
+++
b/nifi-commons/nifi-record/src/main/java/org/apache/nifi/serialization/record/util/DataTypeUtils.java
@@ -736,7 +736,7 @@ public class DataTypeUtils {
}
public static boolean isMapTypeCompatible(final Object value) {
- return value != null && value instanceof Map;
+ return value != null && (value instanceof Map || value instanceof
MapRecord);
}
diff --git
a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml
index 03e9bb4..511334d 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml
@@ -552,6 +552,8 @@
<exclude>src/test/resources/TestForkRecord/schema/schema.avsc</exclude>
<exclude>src/test/resources/TestConvertRecord/schema/person.avsc</exclude>
<exclude>src/test/resources/TestConvertRecord/input/person.json</exclude>
+
<exclude>src/test/resources/TestValidateRecord/nested-map-input.json</exclude>
+
<exclude>src/test/resources/TestValidateRecord/nested-map-schema.avsc</exclude>
</excludes>
</configuration>
</plugin>
diff --git
a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestValidateRecord.java
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestValidateRecord.java
index fdb5be4..52b7968 100644
---
a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestValidateRecord.java
+++
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestValidateRecord.java
@@ -20,6 +20,7 @@ package org.apache.nifi.processors.standard;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
+import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.Optional;
@@ -30,6 +31,7 @@ import org.apache.nifi.csv.CSVReader;
import org.apache.nifi.csv.CSVRecordSetWriter;
import org.apache.nifi.csv.CSVUtils;
import org.apache.nifi.json.JsonRecordSetWriter;
+import org.apache.nifi.json.JsonTreeReader;
import org.apache.nifi.reporting.InitializationException;
import org.apache.nifi.schema.access.SchemaAccessUtils;
import org.apache.nifi.schema.access.SchemaNotFoundException;
@@ -360,4 +362,50 @@ public class TestValidateRecord {
invalidFlowFile.assertContentEquals(expectedInvalidContents);
}
+ @Test
+ public void testValidateNestedMap() throws InitializationException,
IOException {
+ final String validateSchema = new
String(Files.readAllBytes(Paths.get("src/test/resources/TestValidateRecord/nested-map-schema.avsc")),
StandardCharsets.UTF_8);
+
+ final JsonTreeReader jsonReader = new JsonTreeReader();
+ runner.addControllerService("reader", jsonReader);
+ runner.setProperty(jsonReader,
SchemaAccessUtils.SCHEMA_ACCESS_STRATEGY, "schema-text-property");
+ runner.setProperty(jsonReader, SchemaAccessUtils.SCHEMA_TEXT,
validateSchema);
+ runner.enableControllerService(jsonReader);
+
+ final JsonRecordSetWriter validWriter = new JsonRecordSetWriter();
+ runner.addControllerService("writer", validWriter);
+ runner.setProperty(validWriter, "Schema Write Strategy",
"full-schema-attribute");
+ runner.enableControllerService(validWriter);
+
+ final MockRecordWriter invalidWriter = new MockRecordWriter("invalid",
true);
+ runner.addControllerService("invalid-writer", invalidWriter);
+ runner.enableControllerService(invalidWriter);
+
+ runner.setProperty(ValidateRecord.RECORD_READER, "reader");
+ runner.setProperty(ValidateRecord.RECORD_WRITER, "writer");
+ runner.setProperty(ValidateRecord.SCHEMA_ACCESS_STRATEGY,
SchemaAccessUtils.SCHEMA_TEXT_PROPERTY);
+ runner.setProperty(ValidateRecord.SCHEMA_TEXT, validateSchema);
+ runner.setProperty(ValidateRecord.INVALID_RECORD_WRITER,
"invalid-writer");
+ runner.setProperty(ValidateRecord.ALLOW_EXTRA_FIELDS, "false");
+
+ // Both records should be valid if strict type checking is off
+ runner.setProperty(ValidateRecord.STRICT_TYPE_CHECKING, "false");
+
runner.enqueue(Paths.get("src/test/resources/TestValidateRecord/nested-map-input.json"));
+ runner.run();
+
+ runner.assertTransferCount(ValidateRecord.REL_VALID, 1);
+ runner.assertTransferCount(ValidateRecord.REL_INVALID, 0);
+ runner.assertTransferCount(ValidateRecord.REL_FAILURE, 0);
+ runner.clearTransferState();
+
+ // The second record should be invalid if strict type checking is on
+ runner.setProperty(ValidateRecord.STRICT_TYPE_CHECKING, "true");
+
runner.enqueue(Paths.get("src/test/resources/TestValidateRecord/nested-map-input.json"));
+ runner.run();
+
+ runner.assertTransferCount(ValidateRecord.REL_VALID, 1);
+ runner.assertTransferCount(ValidateRecord.REL_INVALID, 1);
+ runner.assertTransferCount(ValidateRecord.REL_FAILURE, 0);
+ }
+
}
diff --git
a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestValidateRecord/nested-map-input.json
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestValidateRecord/nested-map-input.json
new file mode 100644
index 0000000..ed8dc3b
--- /dev/null
+++
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestValidateRecord/nested-map-input.json
@@ -0,0 +1,18 @@
+[{
+ "images" : [ {
+ "headers" : {
+ "Accept-Ranges" : "bytes",
+ "Server" : "Apache"
+ },
+ "id" : "205f40e3-2675-4c61-abc1-a0aeb609c023"
+ } ]
+},
+{
+ "images" : [ {
+ "headers" : {
+ "Accept-Ranges" : 4,
+ "Server" : 10
+ },
+ "id" : "205f40e3-2675-4c61-abc1-a0aeb609c023"
+ } ]
+}]
\ No newline at end of file
diff --git
a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestValidateRecord/nested-map-schema.avsc
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestValidateRecord/nested-map-schema.avsc
new file mode 100644
index 0000000..f7dabab
--- /dev/null
+++
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestValidateRecord/nested-map-schema.avsc
@@ -0,0 +1,32 @@
+{
+ "type": "record",
+ "name": "example",
+ "fields": [
+ {
+ "name": "images",
+ "type": {
+ "type": "array",
+ "items": {
+ "type": "record",
+ "name": "images",
+ "fields": [
+ {
+ "name": "id",
+ "type": [
+ "string",
+ "null"
+ ]
+ },
+ {
+ "name": "headers",
+ "type": {
+ "type": "map",
+ "values": "string"
+ }
+ }
+ ]
+ }
+ }
+ }
+ ]
+}
\ No newline at end of file
