This is an automated email from the ASF dual-hosted git repository.
exceptionfactory pushed a commit to branch support/nifi-1.x
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/support/nifi-1.x by this push:
new b468d4f5af NIFI-11959 Corrected single-line comment handling for Jolt
JSON
b468d4f5af is described below
commit b468d4f5af5bdd2b0ab9bbbd6cffe97b1ad59f8f
Author: dan-s1 <[email protected]>
AuthorDate: Mon Sep 11 19:18:35 2023 +0000
NIFI-11959 Corrected single-line comment handling for Jolt JSON
Added new lines back to the read in JSON spec to allow for single line
comments to be parsed correctly and ignored.
This closes #7678
Signed-off-by: David Handermann <[email protected]>
(cherry picked from commit 0280497c629538bdec8d9f06a3a68561d7a091ed)
---
.../nifi-jolt-record-processors/pom.xml | 1 +
.../jolt/record/JoltTransformRecord.java | 2 +-
.../jolt/record/TestJoltTransformRecord.java | 18 ++++++++++---
.../chainrSpecWithSingleLineComment.json | 30 ++++++++++++++++++++++
.../nifi-standard-processors/pom.xml | 1 +
.../processors/standard/JoltTransformJSON.java | 2 +-
.../processors/standard/TestJoltTransformJSON.java | 17 +++++++++---
.../chainrSpecWithSingleLineComment.json | 30 ++++++++++++++++++++++
8 files changed, 93 insertions(+), 8 deletions(-)
diff --git
a/nifi-nar-bundles/nifi-jolt-record-bundle/nifi-jolt-record-processors/pom.xml
b/nifi-nar-bundles/nifi-jolt-record-bundle/nifi-jolt-record-processors/pom.xml
index 5db87c94fd..b869e2414f 100644
---
a/nifi-nar-bundles/nifi-jolt-record-bundle/nifi-jolt-record-processors/pom.xml
+++
b/nifi-nar-bundles/nifi-jolt-record-bundle/nifi-jolt-record-processors/pom.xml
@@ -104,6 +104,7 @@
<excludes combine.children="append">
<exclude>src/test/resources/TestJoltTransformRecord/input.json</exclude>
<exclude>src/test/resources/TestJoltTransformRecord/chainrSpec.json</exclude>
+
<exclude>src/test/resources/TestJoltTransformRecord/chainrSpecWithSingleLineComment.json</exclude>
<exclude>src/test/resources/TestJoltTransformRecord/customChainrSpec.json</exclude>
<exclude>src/test/resources/TestJoltTransformRecord/chainrOutput.json</exclude>
<exclude>src/test/resources/TestJoltTransformRecord/cardrSpec.json</exclude>
diff --git
a/nifi-nar-bundles/nifi-jolt-record-bundle/nifi-jolt-record-processors/src/main/java/org/apache/nifi/processors/jolt/record/JoltTransformRecord.java
b/nifi-nar-bundles/nifi-jolt-record-bundle/nifi-jolt-record-processors/src/main/java/org/apache/nifi/processors/jolt/record/JoltTransformRecord.java
index 0467cf1b2c..5f0bf54edb 100644
---
a/nifi-nar-bundles/nifi-jolt-record-bundle/nifi-jolt-record-processors/src/main/java/org/apache/nifi/processors/jolt/record/JoltTransformRecord.java
+++
b/nifi-nar-bundles/nifi-jolt-record-bundle/nifi-jolt-record-processors/src/main/java/org/apache/nifi/processors/jolt/record/JoltTransformRecord.java
@@ -488,7 +488,7 @@ public class JoltTransformRecord extends AbstractProcessor {
private String readTransform(final PropertyValue propertyValue) {
final ResourceReference resourceReference = propertyValue.asResource();
try (final BufferedReader reader = new BufferedReader(new
InputStreamReader(resourceReference.read()))) {
- return reader.lines().collect(Collectors.joining());
+ return
reader.lines().collect(Collectors.joining(System.lineSeparator()));
} catch (final IOException e) {
throw new UncheckedIOException("Read JOLT Transform failed", e);
}
diff --git
a/nifi-nar-bundles/nifi-jolt-record-bundle/nifi-jolt-record-processors/src/test/java/org/apache/nifi/processors/jolt/record/TestJoltTransformRecord.java
b/nifi-nar-bundles/nifi-jolt-record-bundle/nifi-jolt-record-processors/src/test/java/org/apache/nifi/processors/jolt/record/TestJoltTransformRecord.java
index 97c28bd87b..ef784afbc5 100644
---
a/nifi-nar-bundles/nifi-jolt-record-bundle/nifi-jolt-record-processors/src/test/java/org/apache/nifi/processors/jolt/record/TestJoltTransformRecord.java
+++
b/nifi-nar-bundles/nifi-jolt-record-bundle/nifi-jolt-record-processors/src/test/java/org/apache/nifi/processors/jolt/record/TestJoltTransformRecord.java
@@ -36,10 +36,14 @@ import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.condition.DisabledOnOs;
import org.junit.jupiter.api.condition.OS;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
import java.io.IOException;
import java.net.URL;
import java.nio.file.Files;
+import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.Collections;
@@ -48,6 +52,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.function.BiFunction;
+import java.util.stream.Stream;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -286,15 +291,16 @@ public class TestJoltTransformRecord {
runner.assertNotValid();
}
- @Test
- public void testTransformInputWithChainr() throws IOException {
+ @ParameterizedTest(name = "{index} {1}")
+ @MethodSource("getChainrArguments")
+ public void testTransformInputWithChainr(Path specPath, String
description) throws IOException {
generateTestData(1, null);
final String outputSchemaText = new
String(Files.readAllBytes(Paths.get("src/test/resources/TestJoltTransformRecord/chainrOutputSchema.avsc")));
runner.setProperty(writer, SchemaAccessUtils.SCHEMA_ACCESS_STRATEGY,
SchemaAccessUtils.SCHEMA_TEXT_PROPERTY);
runner.setProperty(writer, SchemaAccessUtils.SCHEMA_TEXT,
outputSchemaText);
runner.setProperty(writer, "Pretty Print JSON", "true");
runner.enableControllerService(writer);
- final String spec = new
String(Files.readAllBytes(Paths.get("src/test/resources/TestJoltTransformRecord/chainrSpec.json")));
+ final String spec = new String(Files.readAllBytes(specPath));
runner.setProperty(JoltTransformRecord.JOLT_SPEC, spec);
runner.enqueue(new byte[0]);
runner.run();
@@ -683,6 +689,12 @@ public class TestJoltTransformRecord {
runner.assertNotValid();
}
+ private static Stream<Arguments> getChainrArguments() {
+ return Stream.of(
+
Arguments.of(Paths.get("src/test/resources/TestJoltTransformRecord/chainrSpec.json"),
"has no single line comments"),
+
Arguments.of(Paths.get("src/test/resources/TestJoltTransformRecord/chainrSpecWithSingleLineComment.json"),
"has a single line comment"));
+ }
+
private void generateTestData(int numRecords, final BiFunction<Integer,
MockRecordParser, Void> recordGenerator) {
if (recordGenerator == null) {
diff --git
a/nifi-nar-bundles/nifi-jolt-record-bundle/nifi-jolt-record-processors/src/test/resources/TestJoltTransformRecord/chainrSpecWithSingleLineComment.json
b/nifi-nar-bundles/nifi-jolt-record-bundle/nifi-jolt-record-processors/src/test/resources/TestJoltTransformRecord/chainrSpecWithSingleLineComment.json
new file mode 100644
index 0000000000..3b8d0d0b6b
--- /dev/null
+++
b/nifi-nar-bundles/nifi-jolt-record-bundle/nifi-jolt-record-processors/src/test/resources/TestJoltTransformRecord/chainrSpecWithSingleLineComment.json
@@ -0,0 +1,30 @@
+[
+ // Single line comment
+ {
+ "operation": "shift",
+ "spec": {
+ "rating": {
+ "primary": {
+ "value": "Rating",
+ "max": "RatingRange"
+ },
+ "*": {
+ "max": "SecondaryRatings.&1.Range",
+ "value": "SecondaryRatings.&1.Value",
+ "$": "SecondaryRatings.&1.Id"
+ }
+ }
+ }
+ },
+ {
+ "operation": "default",
+ "spec": {
+ "Range": 5,
+ "SecondaryRatings": {
+ "*": {
+ "Range": 5
+ }
+ }
+ }
+ }
+]
diff --git
a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml
index 14bb834914..afbf5cec34 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml
@@ -604,6 +604,7 @@
<exclude>src/test/resources/TestJoltTransformJson/cardrSpec.json</exclude>
<exclude>src/test/resources/TestJoltTransformJson/chainrOutput.json</exclude>
<exclude>src/test/resources/TestJoltTransformJson/chainrSpec.json</exclude>
+
<exclude>src/test/resources/TestJoltTransformJson/chainrSpecWithSingleLineComment.json</exclude>
<exclude>src/test/resources/TestJoltTransformJson/customChainrSpec.json</exclude>
<exclude>src/test/resources/TestJoltTransformJson/defaultrELOutput.json</exclude>
<exclude>src/test/resources/TestJoltTransformJson/defaultrELSpec.json</exclude>
diff --git
a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/JoltTransformJSON.java
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/JoltTransformJSON.java
index 7542cd4564..2b66f8937f 100644
---
a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/JoltTransformJSON.java
+++
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/JoltTransformJSON.java
@@ -396,7 +396,7 @@ public class JoltTransformJSON extends AbstractProcessor {
private String readTransform(final PropertyValue propertyValue) {
final ResourceReference resourceReference = propertyValue.asResource();
try (final BufferedReader reader = new BufferedReader(new
InputStreamReader(resourceReference.read()))) {
- return reader.lines().collect(Collectors.joining());
+ return
reader.lines().collect(Collectors.joining(System.lineSeparator()));
} catch (final IOException e) {
throw new UncheckedIOException("Read JOLT Transform failed", e);
}
diff --git
a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestJoltTransformJSON.java
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestJoltTransformJSON.java
index 3a37d6c698..045a14959a 100644
---
a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestJoltTransformJSON.java
+++
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestJoltTransformJSON.java
@@ -26,6 +26,9 @@ import org.apache.nifi.util.StringUtils;
import org.apache.nifi.util.TestRunner;
import org.apache.nifi.util.TestRunners;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
import java.io.ByteArrayInputStream;
import java.io.IOException;
@@ -36,6 +39,7 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
+import java.util.stream.Stream;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -205,10 +209,12 @@ public class TestJoltTransformJSON {
runner.assertNotValid();
}
- @Test
- public void testTransformInputWithChainr() throws IOException {
+ @ParameterizedTest(name = "{index} {1}")
+ @MethodSource("getChainrArguments")
+ /*NOTE: Even though description is not used in the actual test, it needs
to be declared in order to use it in the ParameterizedTest name argument*/
+ public void testTransformInputWithChainr(Path specPath, String
description) throws IOException {
final TestRunner runner = TestRunners.newTestRunner(new
JoltTransformJSON());
- final String spec = new
String(Files.readAllBytes(Paths.get("src/test/resources/TestJoltTransformJson/chainrSpec.json")));
+ final String spec = new String(Files.readAllBytes(specPath));
runner.setProperty(JoltTransformJSON.JOLT_SPEC, spec);
runner.enqueue(JSON_INPUT);
runner.run();
@@ -528,4 +534,9 @@ public class TestJoltTransformJSON {
runner.assertNotValid();
}
+ private static Stream<Arguments> getChainrArguments() {
+ return Stream.of(
+
Arguments.of(Paths.get("src/test/resources/TestJoltTransformJson/chainrSpec.json"),
"has no single line comments"),
+
Arguments.of(Paths.get("src/test/resources/TestJoltTransformJson/chainrSpecWithSingleLineComment.json"),
"has a single line comment"));
+ }
}
diff --git
a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestJoltTransformJson/chainrSpecWithSingleLineComment.json
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestJoltTransformJson/chainrSpecWithSingleLineComment.json
new file mode 100644
index 0000000000..3b8d0d0b6b
--- /dev/null
+++
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestJoltTransformJson/chainrSpecWithSingleLineComment.json
@@ -0,0 +1,30 @@
+[
+ // Single line comment
+ {
+ "operation": "shift",
+ "spec": {
+ "rating": {
+ "primary": {
+ "value": "Rating",
+ "max": "RatingRange"
+ },
+ "*": {
+ "max": "SecondaryRatings.&1.Range",
+ "value": "SecondaryRatings.&1.Value",
+ "$": "SecondaryRatings.&1.Id"
+ }
+ }
+ }
+ },
+ {
+ "operation": "default",
+ "spec": {
+ "Range": 5,
+ "SecondaryRatings": {
+ "*": {
+ "Range": 5
+ }
+ }
+ }
+ }
+]
