This is an automated email from the ASF dual-hosted git repository.
mosermw pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new a9e27e3ead NIFI-14331 Allow for removing all undefined elements (not
only top level keys) from the JSON when using JsonTreeReader and
JsonRecordSetWriter.
a9e27e3ead is described below
commit a9e27e3eada513f41820009e375459fb75bdd8a6
Author: dan-s1 <[email protected]>
AuthorDate: Thu Mar 6 16:01:50 2025 +0000
NIFI-14331 Allow for removing all undefined elements (not only top level
keys) from the JSON
when using JsonTreeReader and JsonRecordSetWriter.
Signed-off-by: Mike Moser <[email protected]>
Closes #9781
---
.../apache/nifi/json/JsonTreeRowRecordReader.java | 9 +---
.../nifi-standard-processors/pom.xml | 3 +-
.../processors/standard/TestConvertRecord.java | 15 ++++---
.../input/personJob_dropfield.json | 30 +++++++++++++
.../TestConvertRecord/input/person_dropfield.json | 9 ----
.../TestConvertRecord/schema/personJob.avsc | 52 ++++++++++++++++++++++
6 files changed, 95 insertions(+), 23 deletions(-)
diff --git
a/nifi-extension-bundles/nifi-extension-utils/nifi-record-utils/nifi-json-record-utils/src/main/java/org/apache/nifi/json/JsonTreeRowRecordReader.java
b/nifi-extension-bundles/nifi-extension-utils/nifi-record-utils/nifi-json-record-utils/src/main/java/org/apache/nifi/json/JsonTreeRowRecordReader.java
index aade3d1040..2cb49a1324 100644
---
a/nifi-extension-bundles/nifi-extension-utils/nifi-record-utils/nifi-json-record-utils/src/main/java/org/apache/nifi/json/JsonTreeRowRecordReader.java
+++
b/nifi-extension-bundles/nifi-extension-utils/nifi-record-utils/nifi-json-record-utils/src/main/java/org/apache/nifi/json/JsonTreeRowRecordReader.java
@@ -136,13 +136,10 @@ public class JsonTreeRowRecordReader extends
AbstractJsonRowRecordReader {
final boolean coerceTypes, final
boolean dropUnknown) throws IOException, MalformedRecordException {
final Map<String, Object> values = new
LinkedHashMap<>(schema.getFieldCount() * 2);
- final JsonNode jsonNodeForSerialization;
if (dropUnknown) {
- jsonNodeForSerialization = jsonNode.deepCopy();
-
// Delete unknown fields for updated serialized representation
- final Iterator<Map.Entry<String, JsonNode>> fields =
jsonNodeForSerialization.fields();
+ final Iterator<Map.Entry<String, JsonNode>> fields =
jsonNode.fields();
while (fields.hasNext()) {
final Map.Entry<String, JsonNode> field = fields.next();
final String fieldName = field.getKey();
@@ -172,8 +169,6 @@ public class JsonTreeRowRecordReader extends
AbstractJsonRowRecordReader {
values.put(fieldName, value);
}
} else {
- jsonNodeForSerialization = jsonNode;
-
final Iterator<String> fieldNames = jsonNode.fieldNames();
while (fieldNames.hasNext()) {
final String fieldName = fieldNames.next();
@@ -194,7 +189,7 @@ public class JsonTreeRowRecordReader extends
AbstractJsonRowRecordReader {
}
}
- final Supplier<String> supplier = jsonNodeForSerialization::toString;
+ final Supplier<String> supplier = jsonNode::toString;
return new MapRecord(schema, values, SerializedForm.of(supplier,
"application/json"), false, dropUnknown);
}
diff --git
a/nifi-extension-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml
b/nifi-extension-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml
index 461c0d7ab9..91b4aaa886 100644
---
a/nifi-extension-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml
+++
b/nifi-extension-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml
@@ -628,9 +628,10 @@
<exclude>src/test/resources/TestConvertRecord/input/person_bad_enum.json</exclude>
<exclude>src/test/resources/TestConvertRecord/input/person_long_id.json</exclude>
<exclude>src/test/resources/TestConvertRecord/input/person.json</exclude>
-
<exclude>src/test/resources/TestConvertRecord/input/person_dropfield.json</exclude>
+
<exclude>src/test/resources/TestConvertRecord/input/personJob_dropfield.json</exclude>
<exclude>src/test/resources/TestConvertRecord/schema/person_with_union_enum_string.avsc</exclude>
<exclude>src/test/resources/TestConvertRecord/schema/person.avsc</exclude>
+
<exclude>src/test/resources/TestConvertRecord/schema/personJob.avsc</exclude>
<exclude>src/test/resources/TestCountText/jabberwocky.txt</exclude>
<exclude>src/test/resources/TestExtractGrok/apache.log</exclude>
<exclude>src/test/resources/TestExtractGrok/patterns</exclude>
diff --git
a/nifi-extension-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestConvertRecord.java
b/nifi-extension-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestConvertRecord.java
index 5b6c2e38c4..95cdb3ab55 100644
---
a/nifi-extension-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestConvertRecord.java
+++
b/nifi-extension-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestConvertRecord.java
@@ -411,21 +411,21 @@ public class TestConvertRecord {
@Test
public void testJSONDroppingUnknownFields() throws
InitializationException, IOException {
+ final String personJobSchema =
+
Files.readString(Paths.get("src/test/resources/TestConvertRecord/schema/personJob.avsc"));
final JsonTreeReader jsonReader = new JsonTreeReader();
runner.addControllerService(READER_ID, jsonReader);
-
runner.setProperty(jsonReader,
SchemaAccessUtils.SCHEMA_ACCESS_STRATEGY,
SchemaAccessUtils.SCHEMA_TEXT_PROPERTY);
- runner.setProperty(jsonReader, SchemaAccessUtils.SCHEMA_TEXT,
PERSON_SCHEMA);
+ runner.setProperty(jsonReader, SchemaAccessUtils.SCHEMA_TEXT,
personJobSchema);
runner.enableControllerService(jsonReader);
-
final JsonRecordSetWriter jsonWriter = new JsonRecordSetWriter();
runner.addControllerService(WRITER_ID, jsonWriter);
runner.setProperty(jsonWriter,
SchemaAccessUtils.SCHEMA_ACCESS_STRATEGY,
SchemaAccessUtils.SCHEMA_TEXT_PROPERTY);
- runner.setProperty(jsonWriter, SchemaAccessUtils.SCHEMA_TEXT,
PERSON_SCHEMA);
+ runner.setProperty(jsonWriter, SchemaAccessUtils.SCHEMA_TEXT,
personJobSchema);
runner.setProperty(jsonWriter, "Schema Write Strategy",
"full-schema-attribute");
runner.enableControllerService(jsonWriter);
-
runner.enqueue(Paths.get("src/test/resources/TestConvertRecord/input/person_dropfield.json"));
+
runner.enqueue(Paths.get("src/test/resources/TestConvertRecord/input/personJob_dropfield.json"));
runner.setProperty(ConvertRecord.RECORD_READER, READER_ID);
runner.setProperty(ConvertRecord.RECORD_WRITER, WRITER_ID);
@@ -434,7 +434,10 @@ public class TestConvertRecord {
runner.assertAllFlowFilesTransferred(ConvertRecord.REL_SUCCESS, 1);
final MockFlowFile flowFile =
runner.getFlowFilesForRelationship(ConvertRecord.REL_SUCCESS).getFirst();
-
assertFalse(flowFile.getContent().contains("fieldThatShouldBeRemoved"));
+
+ // This covers all the cases
+ // "undefinedKeyInObject", "undefinedKey", "undefinedObjectArray",
"undefinedObject", "undefinedScalarArray"
+ assertFalse(flowFile.getContent().contains("undefined"));
}
@Test
diff --git
a/nifi-extension-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestConvertRecord/input/personJob_dropfield.json
b/nifi-extension-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestConvertRecord/input/personJob_dropfield.json
new file mode 100644
index 0000000000..d9a77e9d38
--- /dev/null
+++
b/nifi-extension-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestConvertRecord/input/personJob_dropfield.json
@@ -0,0 +1,30 @@
+[
+ {
+ "id": 485,
+ "name": {
+ "last": "Doe",
+ "first": "John",
+ "undefinedKeyInObject": "whatever"
+ },
+ "status": "ACTIVE",
+ "undefinedKey" : "whatever",
+ "jobs": [
+ {
+ "jobId": "1",
+ "jobName": "someJob",
+ "undefinedKeyInObject": "whatever"
+ }
+ ],
+ "undefinedObjectArray" : [
+ {
+ "elementId" : "1",
+ "elementName" : "someName"
+ }
+ ],
+ "undefinedObject" : {
+ "elementId" : "2",
+ "elementName" : "someName"
+ },
+ "undefinedScalarArray" : [1, 2, 3]
+ }
+]
\ No newline at end of file
diff --git
a/nifi-extension-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestConvertRecord/input/person_dropfield.json
b/nifi-extension-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestConvertRecord/input/person_dropfield.json
deleted file mode 100644
index 50f955f5d2..0000000000
---
a/nifi-extension-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestConvertRecord/input/person_dropfield.json
+++ /dev/null
@@ -1,9 +0,0 @@
-[ {
- "id" : 485,
- "name" : {
- "last" : "Doe",
- "first" : "John"
- },
- "status" : "ACTIVE",
- "fieldThatShouldBeRemoved": "Test"
-} ]
\ No newline at end of file
diff --git
a/nifi-extension-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestConvertRecord/schema/personJob.avsc
b/nifi-extension-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestConvertRecord/schema/personJob.avsc
new file mode 100644
index 0000000000..213d635220
--- /dev/null
+++
b/nifi-extension-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestConvertRecord/schema/personJob.avsc
@@ -0,0 +1,52 @@
+{
+ "name": "personWithJobRecord",
+ "namespace": "nifi",
+ "type": "record",
+ "fields": [
+ { "name": "id", "type": "int" },
+ { "name": "name", "type": {
+ "type": "record",
+ "name": "nameRecord",
+ "fields": [
+ { "name": "last", "type": "string" },
+ { "name": "first", "type": "string" }
+ ]
+ }
+ },
+ {
+ "name": "status",
+ "type": ["null", {
+ "type": "enum",
+ "name": "statusEnum",
+ "symbols": [
+ "ACTIVE",
+ "INACTIVE"
+ ]
+ }],
+ "default": null
+ },
+ {
+ "name" : "jobs",
+ "type" : [
+ "null",
+ {
+ "type" : "array",
+ "items": {
+ "type": "record",
+ "name": "jobType",
+ "fields": [
+ {
+ "name": "jobId",
+ "type":
["null", "string"]
+ },
+ {
+ "name":
"jobName",
+ "type":
["null", "string"]
+ }
+ ]
+ }
+ }
+ ]
+ }
+ ]
+}
\ No newline at end of file
