Repository: nifi Updated Branches: refs/heads/master 52cf9a795 -> 68c592ea4
http://git-wip-us.apache.org/repos/asf/nifi/blob/a88d3bfa/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/java/org/apache/nifi/grok/TestGrokRecordReader.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/java/org/apache/nifi/grok/TestGrokRecordReader.java b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/java/org/apache/nifi/grok/TestGrokRecordReader.java new file mode 100644 index 0000000..3757ab1 --- /dev/null +++ b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/java/org/apache/nifi/grok/TestGrokRecordReader.java @@ -0,0 +1,190 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.grok; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.Collections; + +import org.apache.nifi.serialization.MalformedRecordException; +import org.apache.nifi.serialization.record.Record; +import org.junit.Test; + +import io.thekraken.grok.api.Grok; +import io.thekraken.grok.api.exception.GrokException; + +public class TestGrokRecordReader { + + @Test + public void testParseSingleLineLogMessages() throws GrokException, IOException, MalformedRecordException { + try (final InputStream fis = new FileInputStream(new File("src/test/resources/grok/single-line-log-messages.txt"))) { + final Grok grok = new Grok(); + grok.addPatternFromFile("src/main/resources/default-grok-patterns.txt"); + grok.compile("%{TIMESTAMP_ISO8601:timestamp} %{LOGLEVEL:level} %{GREEDYDATA:message}"); + + final GrokRecordReader deserializer = new GrokRecordReader(fis, grok, Collections.emptyMap()); + + final String[] logLevels = new String[] {"INFO", "WARN", "ERROR", "FATAL", "FINE"}; + final String[] messages = new String[] {"Test Message 1", "Red", "Green", "Blue", "Yellow"}; + + for (int i = 0; i < logLevels.length; i++) { + final Object[] values = deserializer.nextRecord().getValues(); + + assertNotNull(values); + assertEquals(4, values.length); // values[] contains 4 elements: timestamp, level, message, STACK_TRACE + assertEquals("2016-11-08 21:24:23,029", values[0]); + assertEquals(logLevels[i], values[1]); + assertEquals(messages[i], values[2]); + assertNull(values[3]); + } + + assertNull(deserializer.nextRecord()); + } + } + + + @Test + public void testParseEmptyMessageWithStackTrace() throws GrokException, IOException, MalformedRecordException { + final Grok grok = new Grok(); + grok.addPatternFromFile("src/main/resources/default-grok-patterns.txt"); + grok.compile("%{TIMESTAMP_ISO8601:timestamp} %{LOGLEVEL:level} \\[%{DATA:thread}\\] %{DATA:class} %{GREEDYDATA:message}"); + + final String msg = "2016-08-04 13:26:32,473 INFO [Leader Election Notification Thread-1] o.a.n.LoggerClass \n" + + "org.apache.nifi.exception.UnitTestException: Testing to ensure we are able to capture stack traces"; + final InputStream bais = new ByteArrayInputStream(msg.getBytes(StandardCharsets.UTF_8)); + final GrokRecordReader deserializer = new GrokRecordReader(bais, grok, Collections.emptyMap()); + + final Object[] values = deserializer.nextRecord().getValues(); + + assertNotNull(values); + assertEquals(6, values.length); // values[] contains 4 elements: timestamp, level, message, STACK_TRACE + assertEquals("2016-08-04 13:26:32,473", values[0]); + assertEquals("INFO", values[1]); + assertEquals("Leader Election Notification Thread-1", values[2]); + assertEquals("o.a.n.LoggerClass", values[3]); + assertEquals("", values[4]); + assertEquals("org.apache.nifi.exception.UnitTestException: Testing to ensure we are able to capture stack traces", values[5]); + } + + + + @Test + public void testParseNiFiSampleLog() throws IOException, GrokException, MalformedRecordException { + try (final InputStream fis = new FileInputStream(new File("src/test/resources/grok/nifi-log-sample.log"))) { + final Grok grok = new Grok(); + grok.addPatternFromFile("src/main/resources/default-grok-patterns.txt"); + grok.compile("%{TIMESTAMP_ISO8601:timestamp} %{LOGLEVEL:level} \\[%{DATA:thread}\\] %{DATA:class} %{GREEDYDATA:message}"); + + final GrokRecordReader deserializer = new GrokRecordReader(fis, grok, Collections.emptyMap()); + + final String[] logLevels = new String[] {"INFO", "INFO", "INFO", "WARN", "WARN"}; + + for (int i = 0; i < logLevels.length; i++) { + final Object[] values = deserializer.nextRecord().getValues(); + + assertNotNull(values); + assertEquals(6, values.length); // values[] contains 6 elements: timestamp, level, thread, class, message, STACK_TRACE + assertEquals(logLevels[i], values[1]); + assertNull(values[5]); + } + + assertNull(deserializer.nextRecord()); + } + } + + @Test + public void testParseNiFiSampleMultilineWithStackTrace() throws IOException, GrokException, MalformedRecordException { + try (final InputStream fis = new FileInputStream(new File("src/test/resources/grok/nifi-log-sample-multiline-with-stacktrace.log"))) { + final Grok grok = new Grok(); + grok.addPatternFromFile("src/main/resources/default-grok-patterns.txt"); + grok.compile("%{TIMESTAMP_ISO8601:timestamp} %{LOGLEVEL:level} \\[%{DATA:thread}\\] %{DATA:class} %{GREEDYDATA:message}?"); + + final GrokRecordReader deserializer = new GrokRecordReader(fis, grok, Collections.emptyMap()); + + final String[] logLevels = new String[] {"INFO", "INFO", "ERROR", "WARN", "WARN"}; + + for (int i = 0; i < logLevels.length; i++) { + final Record record = deserializer.nextRecord(); + final Object[] values = record.getValues(); + + assertNotNull(values); + assertEquals(6, values.length); // values[] contains 6 elements: timestamp, level, thread, class, message, STACK_TRACE + assertEquals(logLevels[i], values[1]); + if ("ERROR".equals(values[1])) { + final String msg = (String) values[4]; + assertEquals("One\nTwo\nThree", msg); + assertNotNull(values[5]); + } else { + assertNull(values[5]); + } + } + + assertNull(deserializer.nextRecord()); + } + } + + + @Test + public void testParseStackTrace() throws GrokException, IOException, MalformedRecordException { + try (final InputStream fis = new FileInputStream(new File("src/test/resources/grok/error-with-stack-trace.log"))) { + final Grok grok = new Grok(); + grok.addPatternFromFile("src/main/resources/default-grok-patterns.txt"); + grok.compile("%{TIMESTAMP_ISO8601:timestamp} %{LOGLEVEL:level} %{GREEDYDATA:message}"); + + final GrokRecordReader deserializer = new GrokRecordReader(fis, grok, Collections.emptyMap()); + + final String[] logLevels = new String[] {"INFO", "ERROR", "INFO"}; + final String[] messages = new String[] {"message without stack trace", + "Log message with stack trace", + "message without stack trace"}; + + for (int i = 0; i < logLevels.length; i++) { + final Object[] values = deserializer.nextRecord().getValues(); + + assertNotNull(values); + assertEquals(4, values.length); // values[] contains 4 elements: timestamp, level, message, STACK_TRACE + assertEquals(logLevels[i], values[1]); + assertEquals(messages[i], values[2]); + + if (values[1].equals("ERROR")) { + final String stackTrace = (String) values[3]; + assertNotNull(stackTrace); + assertTrue(stackTrace.startsWith("org.apache.nifi.exception.UnitTestException: Testing to ensure we are able to capture stack traces")); + assertTrue(stackTrace.contains(" at org.apache.nifi.cluster.coordination.node.NodeClusterCoordinator.getElectedActiveCoordinatorAddress(" + + "NodeClusterCoordinator.java:185) [nifi-framework-cluster-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT]")); + assertTrue(stackTrace.contains("Caused by: org.apache.nifi.exception.UnitTestException: Testing to ensure we are able to capture stack traces")); + assertTrue(stackTrace.contains("at org.apache.nifi.cluster.coordination.node.NodeClusterCoordinator.getElectedActiveCoordinatorAddress(" + + "NodeClusterCoordinator.java:185) [nifi-framework-cluster-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT]")); + assertTrue(stackTrace.endsWith(" ... 12 common frames omitted")); + } + } + + assertNull(deserializer.nextRecord()); + } + } + +} http://git-wip-us.apache.org/repos/asf/nifi/blob/a88d3bfa/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/java/org/apache/nifi/json/TestJsonPathRowRecordReader.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/java/org/apache/nifi/json/TestJsonPathRowRecordReader.java b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/java/org/apache/nifi/json/TestJsonPathRowRecordReader.java new file mode 100644 index 0000000..fa41396 --- /dev/null +++ b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/java/org/apache/nifi/json/TestJsonPathRowRecordReader.java @@ -0,0 +1,292 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.json; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import org.apache.nifi.logging.ComponentLog; +import org.apache.nifi.serialization.MalformedRecordException; +import org.apache.nifi.serialization.record.DataType; +import org.apache.nifi.serialization.record.Record; +import org.apache.nifi.serialization.record.RecordFieldType; +import org.apache.nifi.serialization.record.RecordSchema; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; + +import com.jayway.jsonpath.JsonPath; + +public class TestJsonPathRowRecordReader { + private final LinkedHashMap<String, JsonPath> allJsonPaths = new LinkedHashMap<>(); + + @Before + public void populateJsonPaths() { + allJsonPaths.clear(); + + allJsonPaths.put("id", JsonPath.compile("$.id")); + allJsonPaths.put("name", JsonPath.compile("$.name")); + allJsonPaths.put("balance", JsonPath.compile("$.balance")); + allJsonPaths.put("address", JsonPath.compile("$.address")); + allJsonPaths.put("city", JsonPath.compile("$.city")); + allJsonPaths.put("state", JsonPath.compile("$.state")); + allJsonPaths.put("zipCode", JsonPath.compile("$.zipCode")); + allJsonPaths.put("country", JsonPath.compile("$.country")); + } + + @Test + public void testReadArray() throws IOException, MalformedRecordException { + try (final InputStream in = new FileInputStream(new File("src/test/resources/json/bank-account-array.json")); + final JsonPathRowRecordReader reader = new JsonPathRowRecordReader(allJsonPaths, Collections.emptyMap(), in, Mockito.mock(ComponentLog.class))) { + + final RecordSchema schema = reader.getSchema(); + + final List<String> fieldNames = schema.getFieldNames(); + final List<String> expectedFieldNames = Arrays.asList(new String[] {"id", "name", "balance", "address", "city", "state", "zipCode", "country"}); + assertEquals(expectedFieldNames, fieldNames); + + final List<RecordFieldType> dataTypes = schema.getDataTypes().stream().map(dt -> dt.getFieldType()).collect(Collectors.toList()); + final List<RecordFieldType> expectedTypes = Arrays.asList(new RecordFieldType[] {RecordFieldType.INT, RecordFieldType.STRING, + RecordFieldType.DOUBLE, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING}); + assertEquals(expectedTypes, dataTypes); + + final Object[] firstRecordValues = reader.nextRecord().getValues(); + Assert.assertArrayEquals(new Object[] {1, "John Doe", 4750.89, "123 My Street", "My City", "MS", "11111", "USA"}, firstRecordValues); + + final Object[] secondRecordValues = reader.nextRecord().getValues(); + Assert.assertArrayEquals(new Object[] {2, "Jane Doe", 4820.09, "321 Your Street", "Your City", "NY", "33333", "USA"}, secondRecordValues); + + assertNull(reader.nextRecord()); + } + } + + @Test + public void testSingleJsonElement() throws IOException, MalformedRecordException { + try (final InputStream in = new FileInputStream(new File("src/test/resources/json/single-bank-account.json")); + final JsonPathRowRecordReader reader = new JsonPathRowRecordReader(allJsonPaths, Collections.emptyMap(), in, Mockito.mock(ComponentLog.class))) { + + final RecordSchema schema = reader.getSchema(); + + final List<String> fieldNames = schema.getFieldNames(); + final List<String> expectedFieldNames = Arrays.asList(new String[] {"id", "name", "balance", "address", "city", "state", "zipCode", "country"}); + assertEquals(expectedFieldNames, fieldNames); + + final List<RecordFieldType> dataTypes = schema.getDataTypes().stream().map(dt -> dt.getFieldType()).collect(Collectors.toList()); + final List<RecordFieldType> expectedTypes = Arrays.asList(new RecordFieldType[] {RecordFieldType.INT, RecordFieldType.STRING, + RecordFieldType.DOUBLE, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING}); + assertEquals(expectedTypes, dataTypes); + + final Object[] firstRecordValues = reader.nextRecord().getValues(); + Assert.assertArrayEquals(new Object[] {1, "John Doe", 4750.89, "123 My Street", "My City", "MS", "11111", "USA"}, firstRecordValues); + + assertNull(reader.nextRecord()); + } + } + + + + @Test + public void testElementWithNestedData() throws IOException, MalformedRecordException { + final LinkedHashMap<String, JsonPath> jsonPaths = new LinkedHashMap<>(allJsonPaths); + jsonPaths.put("account", JsonPath.compile("$.account")); + + try (final InputStream in = new FileInputStream(new File("src/test/resources/json/single-element-nested.json")); + final JsonPathRowRecordReader reader = new JsonPathRowRecordReader(jsonPaths, Collections.emptyMap(), in, Mockito.mock(ComponentLog.class))) { + + final RecordSchema schema = reader.getSchema(); + + final List<String> fieldNames = schema.getFieldNames(); + final List<String> expectedFieldNames = Arrays.asList(new String[] {"id", "name", "balance", "address", "city", "state", "zipCode", "country", "account"}); + assertEquals(expectedFieldNames, fieldNames); + + final List<RecordFieldType> dataTypes = schema.getDataTypes().stream().map(dt -> dt.getFieldType()).collect(Collectors.toList()); + final List<RecordFieldType> expectedTypes = Arrays.asList(new RecordFieldType[] {RecordFieldType.INT, RecordFieldType.STRING, RecordFieldType.STRING, + RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.RECORD}); + assertEquals(expectedTypes, dataTypes); + + final Object[] firstRecordValues = reader.nextRecord().getValues(); + final Object[] simpleElements = Arrays.copyOfRange(firstRecordValues, 0, firstRecordValues.length - 1); + Assert.assertArrayEquals(new Object[] {1, "John Doe", null, "123 My Street", "My City", "MS", "11111", "USA"}, simpleElements); + + final Object lastElement = firstRecordValues[firstRecordValues.length - 1]; + assertTrue(lastElement instanceof Record); + final Record record = (Record) lastElement; + assertEquals(42, record.getValue("id")); + assertEquals(4750.89D, record.getValue("balance")); + + assertNull(reader.nextRecord()); + } + } + + @Test + public void testElementWithNestedArray() throws IOException, MalformedRecordException { + final LinkedHashMap<String, JsonPath> jsonPaths = new LinkedHashMap<>(allJsonPaths); + jsonPaths.put("accounts", JsonPath.compile("$.accounts")); + + try (final InputStream in = new FileInputStream(new File("src/test/resources/json/single-element-nested-array.json")); + final JsonPathRowRecordReader reader = new JsonPathRowRecordReader(jsonPaths, Collections.emptyMap(), in, Mockito.mock(ComponentLog.class))) { + + final RecordSchema schema = reader.getSchema(); + + final List<String> fieldNames = schema.getFieldNames(); + final List<String> expectedFieldNames = Arrays.asList(new String[] { + "id", "name", "balance", "address", "city", "state", "zipCode", "country", "accounts"}); + assertEquals(expectedFieldNames, fieldNames); + + final List<RecordFieldType> dataTypes = schema.getDataTypes().stream().map(dt -> dt.getFieldType()).collect(Collectors.toList()); + final List<RecordFieldType> expectedTypes = Arrays.asList(new RecordFieldType[] {RecordFieldType.INT, RecordFieldType.STRING, RecordFieldType.STRING, + RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.ARRAY}); + assertEquals(expectedTypes, dataTypes); + + final Object[] firstRecordValues = reader.nextRecord().getValues(); + final Object[] nonArrayValues = Arrays.copyOfRange(firstRecordValues, 0, firstRecordValues.length - 1); + Assert.assertArrayEquals(new Object[] {1, "John Doe", null, "123 My Street", "My City", "MS", "11111", "USA"}, nonArrayValues); + + final Object lastRecord = firstRecordValues[firstRecordValues.length - 1]; + assertTrue(Object[].class.isAssignableFrom(lastRecord.getClass())); + + final Object[] array = (Object[]) lastRecord; + assertEquals(2, array.length); + final Object firstElement = array[0]; + assertTrue(firstElement instanceof Map); + + final Map<?, ?> firstMap = (Map<?, ?>) firstElement; + assertEquals(42, firstMap.get("id")); + assertEquals(4750.89D, firstMap.get("balance")); + + final Object secondElement = array[1]; + assertTrue(secondElement instanceof Map); + final Map<?, ?> secondMap = (Map<?, ?>) secondElement; + assertEquals(43, secondMap.get("id")); + assertEquals(48212.38D, secondMap.get("balance")); + + assertNull(reader.nextRecord()); + } + } + + @Test + public void testReadArrayDifferentSchemas() throws IOException, MalformedRecordException { + try (final InputStream in = new FileInputStream(new File("src/test/resources/json/bank-account-array-different-schemas.json")); + final JsonPathRowRecordReader reader = new JsonPathRowRecordReader(allJsonPaths, Collections.emptyMap(), in, Mockito.mock(ComponentLog.class))) { + + final RecordSchema schema = reader.getSchema(); + + final List<String> fieldNames = schema.getFieldNames(); + final List<String> expectedFieldNames = Arrays.asList(new String[] {"id", "name", "balance", "address", "city", "state", "zipCode", "country"}); + assertEquals(expectedFieldNames, fieldNames); + + final List<RecordFieldType> dataTypes = schema.getDataTypes().stream().map(dt -> dt.getFieldType()).collect(Collectors.toList()); + final List<RecordFieldType> expectedTypes = Arrays.asList(new RecordFieldType[] {RecordFieldType.INT, RecordFieldType.STRING, + RecordFieldType.DOUBLE, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING}); + assertEquals(expectedTypes, dataTypes); + + final Object[] firstRecordValues = reader.nextRecord().getValues(); + Assert.assertArrayEquals(new Object[] {1, "John Doe", 4750.89, "123 My Street", "My City", "MS", "11111", "USA"}, firstRecordValues); + + final Object[] secondRecordValues = reader.nextRecord().getValues(); + Assert.assertArrayEquals(new Object[] {2, "Jane Doe", 4820.09, "321 Your Street", "Your City", "NY", "33333", null}, secondRecordValues); + + final Object[] thirdRecordValues = reader.nextRecord().getValues(); + Assert.assertArrayEquals(new Object[] {3, "Jake Doe", 4751.89, "124 My Street", "My City", "MS", "11111", "USA"}, thirdRecordValues); + + assertNull(reader.nextRecord()); + } + } + + @Test + public void testReadArrayDifferentSchemasWithOverride() throws IOException, MalformedRecordException { + final LinkedHashMap<String, JsonPath> jsonPaths = new LinkedHashMap<>(allJsonPaths); + jsonPaths.put("address2", JsonPath.compile("$.address2")); + final Map<String, DataType> typeOverrides = Collections.singletonMap("address2", RecordFieldType.STRING.getDataType()); + + try (final InputStream in = new FileInputStream(new File("src/test/resources/json/bank-account-array-different-schemas.json")); + final JsonPathRowRecordReader reader = new JsonPathRowRecordReader(jsonPaths, typeOverrides, in, Mockito.mock(ComponentLog.class))) { + final RecordSchema schema = reader.getSchema(); + + final List<String> fieldNames = schema.getFieldNames(); + final List<String> expectedFieldNames = Arrays.asList(new String[] {"id", "name", "balance", "address", "city", "state", "zipCode", "country", "address2"}); + assertEquals(expectedFieldNames, fieldNames); + + final List<RecordFieldType> dataTypes = schema.getDataTypes().stream().map(dt -> dt.getFieldType()).collect(Collectors.toList()); + final List<RecordFieldType> expectedTypes = Arrays.asList(new RecordFieldType[] {RecordFieldType.INT, RecordFieldType.STRING, RecordFieldType.DOUBLE, RecordFieldType.STRING, + RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING}); + assertEquals(expectedTypes, dataTypes); + + final Object[] firstRecordValues = reader.nextRecord().getValues(); + Assert.assertArrayEquals(new Object[] {1, "John Doe", 4750.89, "123 My Street", "My City", "MS", "11111", "USA", null}, firstRecordValues); + + final Object[] secondRecordValues = reader.nextRecord().getValues(); + Assert.assertArrayEquals(new Object[] {2, "Jane Doe", 4820.09, "321 Your Street", "Your City", "NY", "33333", null, null}, secondRecordValues); + + final Object[] thirdRecordValues = reader.nextRecord().getValues(); + Assert.assertArrayEquals(new Object[] {3, "Jake Doe", 4751.89, "124 My Street", "My City", "MS", "11111", "USA", "Apt. #12"}, thirdRecordValues); + + assertNull(reader.nextRecord()); + } + } + + @Test + public void testPrimitiveTypeArrays() throws IOException, MalformedRecordException { + final LinkedHashMap<String, JsonPath> jsonPaths = new LinkedHashMap<>(allJsonPaths); + jsonPaths.put("accountIds", JsonPath.compile("$.accountIds")); + + try (final InputStream in = new FileInputStream(new File("src/test/resources/json/primitive-type-array.json")); + final JsonPathRowRecordReader reader = new JsonPathRowRecordReader(jsonPaths, Collections.emptyMap(), in, Mockito.mock(ComponentLog.class))) { + + final RecordSchema schema = reader.getSchema(); + + final List<String> fieldNames = schema.getFieldNames(); + final List<String> expectedFieldNames = Arrays.asList(new String[] {"id", "name", "balance", "address", "city", "state", "zipCode", "country", "accountIds"}); + assertEquals(expectedFieldNames, fieldNames); + + final List<RecordFieldType> dataTypes = schema.getDataTypes().stream().map(dt -> dt.getFieldType()).collect(Collectors.toList()); + final List<RecordFieldType> expectedTypes = Arrays.asList(new RecordFieldType[] {RecordFieldType.INT, RecordFieldType.STRING, RecordFieldType.DOUBLE, RecordFieldType.STRING, + RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.ARRAY}); + assertEquals(expectedTypes, dataTypes); + + final Object[] firstRecordValues = reader.nextRecord().getValues(); + + final Object[] nonArrayValues = Arrays.copyOfRange(firstRecordValues, 0, firstRecordValues.length - 1); + Assert.assertArrayEquals(new Object[] {1, "John Doe", 4750.89D, "123 My Street", "My City", "MS", "11111", "USA"}, nonArrayValues); + + final Object lastRecord = firstRecordValues[firstRecordValues.length - 1]; + assertNotNull(lastRecord); + assertTrue(Object[].class.isAssignableFrom(lastRecord.getClass())); + + final Object[] array = (Object[]) lastRecord; + Assert.assertArrayEquals(new Object[] {1, 2, 3}, array); + + assertNull(reader.nextRecord()); + assertNull(reader.nextRecord()); + } + } +} http://git-wip-us.apache.org/repos/asf/nifi/blob/a88d3bfa/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/java/org/apache/nifi/json/TestJsonTreeRowRecordReader.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/java/org/apache/nifi/json/TestJsonTreeRowRecordReader.java b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/java/org/apache/nifi/json/TestJsonTreeRowRecordReader.java new file mode 100644 index 0000000..c5ee0e3 --- /dev/null +++ b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/java/org/apache/nifi/json/TestJsonTreeRowRecordReader.java @@ -0,0 +1,266 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.json; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import org.apache.nifi.logging.ComponentLog; +import org.apache.nifi.serialization.MalformedRecordException; +import org.apache.nifi.serialization.record.DataType; +import org.apache.nifi.serialization.record.Record; +import org.apache.nifi.serialization.record.RecordFieldType; +import org.apache.nifi.serialization.record.RecordSchema; +import org.junit.Assert; +import org.junit.Test; +import org.mockito.Mockito; + +public class TestJsonTreeRowRecordReader { + + @Test + public void testReadArray() throws IOException, MalformedRecordException { + try (final InputStream in = new FileInputStream(new File("src/test/resources/json/bank-account-array.json")); + final JsonTreeRowRecordReader reader = new JsonTreeRowRecordReader(in, Mockito.mock(ComponentLog.class), Collections.emptyMap())) { + + final RecordSchema schema = reader.getSchema(); + + final List<String> fieldNames = schema.getFieldNames(); + final List<String> expectedFieldNames = Arrays.asList(new String[] {"id", "name", "balance", "address", "city", "state", "zipCode", "country"}); + assertEquals(expectedFieldNames, fieldNames); + + final List<RecordFieldType> dataTypes = schema.getDataTypes().stream().map(dt -> dt.getFieldType()).collect(Collectors.toList()); + final List<RecordFieldType> expectedTypes = Arrays.asList(new RecordFieldType[] {RecordFieldType.INT, RecordFieldType.STRING, + RecordFieldType.DOUBLE, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING}); + assertEquals(expectedTypes, dataTypes); + + final Object[] firstRecordValues = reader.nextRecord().getValues(); + Assert.assertArrayEquals(new Object[] {1, "John Doe", 4750.89, "123 My Street", "My City", "MS", "11111", "USA"}, firstRecordValues); + + final Object[] secondRecordValues = reader.nextRecord().getValues(); + Assert.assertArrayEquals(new Object[] {2, "Jane Doe", 4820.09, "321 Your Street", "Your City", "NY", "33333", "USA"}, secondRecordValues); + + assertNull(reader.nextRecord()); + } + } + + @Test + public void testSingleJsonElement() throws IOException, MalformedRecordException { + try (final InputStream in = new FileInputStream(new File("src/test/resources/json/single-bank-account.json")); + final JsonTreeRowRecordReader reader = new JsonTreeRowRecordReader(in, Mockito.mock(ComponentLog.class), Collections.emptyMap())) { + + final RecordSchema schema = reader.getSchema(); + + final List<String> fieldNames = schema.getFieldNames(); + final List<String> expectedFieldNames = Arrays.asList(new String[] {"id", "name", "balance", "address", "city", "state", "zipCode", "country"}); + assertEquals(expectedFieldNames, fieldNames); + + final List<RecordFieldType> dataTypes = schema.getDataTypes().stream().map(dt -> dt.getFieldType()).collect(Collectors.toList()); + final List<RecordFieldType> expectedTypes = Arrays.asList(new RecordFieldType[] {RecordFieldType.INT, RecordFieldType.STRING, + RecordFieldType.DOUBLE, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING}); + assertEquals(expectedTypes, dataTypes); + + final Object[] firstRecordValues = reader.nextRecord().getValues(); + Assert.assertArrayEquals(new Object[] {1, "John Doe", 4750.89, "123 My Street", "My City", "MS", "11111", "USA"}, firstRecordValues); + + assertNull(reader.nextRecord()); + } + } + + @Test + public void testElementWithNestedData() throws IOException, MalformedRecordException { + try (final InputStream in = new FileInputStream(new File("src/test/resources/json/single-element-nested.json")); + final JsonTreeRowRecordReader reader = new JsonTreeRowRecordReader(in, Mockito.mock(ComponentLog.class), Collections.emptyMap())) { + + final RecordSchema schema = reader.getSchema(); + + final List<String> fieldNames = schema.getFieldNames(); + final List<String> expectedFieldNames = Arrays.asList(new String[] {"id", "name", "address", "city", "state", "zipCode", "country", "account"}); + assertEquals(expectedFieldNames, fieldNames); + + final List<RecordFieldType> dataTypes = schema.getDataTypes().stream().map(dt -> dt.getFieldType()).collect(Collectors.toList()); + final List<RecordFieldType> expectedTypes = Arrays.asList(new RecordFieldType[] {RecordFieldType.INT, RecordFieldType.STRING, + RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.RECORD}); + assertEquals(expectedTypes, dataTypes); + + final Object[] firstRecordValues = reader.nextRecord().getValues(); + final Object[] allButLast = Arrays.copyOfRange(firstRecordValues, 0, firstRecordValues.length - 1); + Assert.assertArrayEquals(new Object[] {1, "John Doe", "123 My Street", "My City", "MS", "11111", "USA"}, allButLast); + + final Object last = firstRecordValues[firstRecordValues.length - 1]; + assertTrue(Record.class.isAssignableFrom(last.getClass())); + final Record record = (Record) last; + assertEquals(42, record.getValue("id")); + assertEquals(4750.89, record.getValue("balance")); + + assertNull(reader.nextRecord()); + } + } + + @Test + public void testElementWithNestedArray() throws IOException, MalformedRecordException { + try (final InputStream in = new FileInputStream(new File("src/test/resources/json/single-element-nested-array.json")); + final JsonTreeRowRecordReader reader = new JsonTreeRowRecordReader(in, Mockito.mock(ComponentLog.class), Collections.emptyMap())) { + + final RecordSchema schema = reader.getSchema(); + + final List<String> fieldNames = schema.getFieldNames(); + final List<String> expectedFieldNames = Arrays.asList(new String[] { + "id", "name", "address", "city", "state", "zipCode", "country", "accounts"}); + assertEquals(expectedFieldNames, fieldNames); + + final List<RecordFieldType> dataTypes = schema.getDataTypes().stream().map(dt -> dt.getFieldType()).collect(Collectors.toList()); + final List<RecordFieldType> expectedTypes = Arrays.asList(new RecordFieldType[] {RecordFieldType.INT, RecordFieldType.STRING, + RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.ARRAY}); + assertEquals(expectedTypes, dataTypes); + + final Object[] firstRecordValues = reader.nextRecord().getValues(); + final Object[] nonArrayValues = Arrays.copyOfRange(firstRecordValues, 0, firstRecordValues.length - 1); + Assert.assertArrayEquals(new Object[] {1, "John Doe", "123 My Street", "My City", "MS", "11111", "USA"}, nonArrayValues); + + final Object lastRecord = firstRecordValues[firstRecordValues.length - 1]; + assertTrue(Object[].class.isAssignableFrom(lastRecord.getClass())); + + assertNull(reader.nextRecord()); + } + } + + @Test + public void testReadArrayDifferentSchemas() throws IOException, MalformedRecordException { + try (final InputStream in = new FileInputStream(new File("src/test/resources/json/bank-account-array-different-schemas.json")); + final JsonTreeRowRecordReader reader = new JsonTreeRowRecordReader(in, Mockito.mock(ComponentLog.class), Collections.emptyMap())) { + + final RecordSchema schema = reader.getSchema(); + + final List<String> fieldNames = schema.getFieldNames(); + final List<String> expectedFieldNames = Arrays.asList(new String[] {"id", "name", "balance", "address", "city", "state", "zipCode", "country"}); + assertEquals(expectedFieldNames, fieldNames); + + final List<RecordFieldType> dataTypes = schema.getDataTypes().stream().map(dt -> dt.getFieldType()).collect(Collectors.toList()); + final List<RecordFieldType> expectedTypes = Arrays.asList(new RecordFieldType[] {RecordFieldType.INT, RecordFieldType.STRING, + RecordFieldType.DOUBLE, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING}); + assertEquals(expectedTypes, dataTypes); + + final Object[] firstRecordValues = reader.nextRecord().getValues(); + Assert.assertArrayEquals(new Object[] {1, "John Doe", 4750.89, "123 My Street", "My City", "MS", "11111", "USA"}, firstRecordValues); + + final Object[] secondRecordValues = reader.nextRecord().getValues(); + Assert.assertArrayEquals(new Object[] {2, "Jane Doe", 4820.09, "321 Your Street", "Your City", "NY", "33333", null}, secondRecordValues); + + final Object[] thirdRecordValues = reader.nextRecord().getValues(); + Assert.assertArrayEquals(new Object[] {3, "Jake Doe", 4751.89, "124 My Street", "My City", "MS", "11111", "USA"}, thirdRecordValues); + + assertNull(reader.nextRecord()); + } + } + + @Test + public void testReadArrayDifferentSchemasWithOverride() throws IOException, MalformedRecordException { + final Map<String, DataType> overrides = new HashMap<>(); + overrides.put("address2", RecordFieldType.STRING.getDataType()); + + try (final InputStream in = new FileInputStream(new File("src/test/resources/json/bank-account-array-different-schemas.json")); + final JsonTreeRowRecordReader reader = new JsonTreeRowRecordReader(in, Mockito.mock(ComponentLog.class), overrides)) { + + final RecordSchema schema = reader.getSchema(); + + final List<String> fieldNames = schema.getFieldNames(); + final List<String> expectedFieldNames = Arrays.asList(new String[] {"id", "name", "balance", "address", "city", "state", "zipCode", "country", "address2"}); + assertEquals(expectedFieldNames, fieldNames); + + final List<RecordFieldType> dataTypes = schema.getDataTypes().stream().map(dt -> dt.getFieldType()).collect(Collectors.toList()); + final List<RecordFieldType> expectedTypes = Arrays.asList(new RecordFieldType[] {RecordFieldType.INT, RecordFieldType.STRING, RecordFieldType.DOUBLE, RecordFieldType.STRING, + RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING}); + assertEquals(expectedTypes, dataTypes); + + final Object[] firstRecordValues = reader.nextRecord().getValues(); + Assert.assertArrayEquals(new Object[] {1, "John Doe", 4750.89, "123 My Street", "My City", "MS", "11111", "USA", null}, firstRecordValues); + + final Object[] secondRecordValues = reader.nextRecord().getValues(); + Assert.assertArrayEquals(new Object[] {2, "Jane Doe", 4820.09, "321 Your Street", "Your City", "NY", "33333", null, null}, secondRecordValues); + + final Object[] thirdRecordValues = reader.nextRecord().getValues(); + Assert.assertArrayEquals(new Object[] {3, "Jake Doe", 4751.89, "124 My Street", "My City", "MS", "11111", "USA", "Apt. #12"}, thirdRecordValues); + + assertNull(reader.nextRecord()); + } + } + + @Test + public void testReadArrayDifferentSchemasWithOptionalElementOverridden() throws IOException, MalformedRecordException { + final Map<String, DataType> overrides = new HashMap<>(); + overrides.put("balance", RecordFieldType.DOUBLE.getDataType()); + + try (final InputStream in = new FileInputStream(new File("src/test/resources/json/bank-account-array-optional-balance.json")); + final JsonTreeRowRecordReader reader = new JsonTreeRowRecordReader(in, Mockito.mock(ComponentLog.class), overrides)) { + + final RecordSchema schema = reader.getSchema(); + + final List<String> fieldNames = schema.getFieldNames(); + final List<String> expectedFieldNames = Arrays.asList(new String[] {"id", "name", "balance", "address", "city", "state", "zipCode", "country"}); + assertEquals(expectedFieldNames, fieldNames); + + final List<RecordFieldType> dataTypes = schema.getDataTypes().stream().map(dt -> dt.getFieldType()).collect(Collectors.toList()); + final List<RecordFieldType> expectedTypes = Arrays.asList(new RecordFieldType[] {RecordFieldType.INT, RecordFieldType.STRING, RecordFieldType.DOUBLE, RecordFieldType.STRING, + RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING}); + assertEquals(expectedTypes, dataTypes); + + final Object[] firstRecordValues = reader.nextRecord().getValues(); + Assert.assertArrayEquals(new Object[] {1, "John Doe", 4750.89, "123 My Street", "My City", "MS", "11111", "USA"}, firstRecordValues); + + final Object[] secondRecordValues = reader.nextRecord().getValues(); + Assert.assertArrayEquals(new Object[] {2, "Jane Doe", null, "321 Your Street", "Your City", "NY", "33333", "USA"}, secondRecordValues); + + final Object[] thirdRecordValues = reader.nextRecord().getValues(); + Assert.assertArrayEquals(new Object[] {3, "Jimmy Doe", null, "321 Your Street", "Your City", "NY", "33333", "USA"}, thirdRecordValues); + + assertNull(reader.nextRecord()); + } + } + + + @Test + public void testReadUnicodeCharacters() throws IOException, MalformedRecordException { + try (final InputStream in = new FileInputStream(new File("src/test/resources/json/json-with-unicode.json")); + final JsonTreeRowRecordReader reader = new JsonTreeRowRecordReader(in, Mockito.mock(ComponentLog.class), Collections.emptyMap())) { + + final Object[] firstRecordValues = reader.nextRecord().getValues(); + + final Object secondValue = firstRecordValues[1]; + assertTrue(secondValue instanceof Long); + assertEquals(832036744985577473L, secondValue); + + final Object unicodeValue = firstRecordValues[2]; + assertEquals("\u3061\u3083\u6ce3\u304d\u305d\u3046", unicodeValue); + + assertNull(reader.nextRecord()); + } + } + +} http://git-wip-us.apache.org/repos/asf/nifi/blob/a88d3bfa/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/java/org/apache/nifi/json/TestWriteJsonResult.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/java/org/apache/nifi/json/TestWriteJsonResult.java b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/java/org/apache/nifi/json/TestWriteJsonResult.java new file mode 100644 index 0000000..f9849ba --- /dev/null +++ b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/java/org/apache/nifi/json/TestWriteJsonResult.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.json; + +import static org.junit.Assert.assertEquals; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.math.BigInteger; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.sql.Date; +import java.sql.Time; +import java.sql.Timestamp; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import org.apache.nifi.logging.ComponentLog; +import org.apache.nifi.serialization.SimpleRecordSchema; +import org.apache.nifi.serialization.record.DataType; +import org.apache.nifi.serialization.record.MapRecord; +import org.apache.nifi.serialization.record.Record; +import org.apache.nifi.serialization.record.RecordField; +import org.apache.nifi.serialization.record.RecordFieldType; +import org.apache.nifi.serialization.record.RecordSchema; +import org.apache.nifi.serialization.record.RecordSet; +import org.junit.Test; +import org.mockito.Mockito; + +public class TestWriteJsonResult { + + @Test + public void testDataTypes() throws IOException, ParseException { + final WriteJsonResult writer = new WriteJsonResult(Mockito.mock(ComponentLog.class), true, RecordFieldType.DATE.getDefaultFormat(), + RecordFieldType.TIME.getDefaultFormat(), RecordFieldType.TIMESTAMP.getDefaultFormat()); + + final List<RecordField> fields = new ArrayList<>(); + for (final RecordFieldType fieldType : RecordFieldType.values()) { + if (fieldType == RecordFieldType.CHOICE) { + final List<DataType> possibleTypes = new ArrayList<>(); + possibleTypes.add(RecordFieldType.INT.getDataType()); + possibleTypes.add(RecordFieldType.LONG.getDataType()); + + fields.add(new RecordField(fieldType.name().toLowerCase(), fieldType.getDataType(possibleTypes))); + } else { + fields.add(new RecordField(fieldType.name().toLowerCase(), fieldType.getDataType())); + } + } + final RecordSchema schema = new SimpleRecordSchema(fields); + + final long time = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss.SSS").parse("2017/01/01 17:00:00.000").getTime(); + final Map<String, Object> valueMap = new LinkedHashMap<>(); + valueMap.put("string", "string"); + valueMap.put("boolean", true); + valueMap.put("byte", (byte) 1); + valueMap.put("char", 'c'); + valueMap.put("short", (short) 8); + valueMap.put("int", 9); + valueMap.put("bigint", BigInteger.valueOf(8L)); + valueMap.put("long", 8L); + valueMap.put("float", 8.0F); + valueMap.put("double", 8.0D); + valueMap.put("date", new Date(time)); + valueMap.put("time", new Time(time)); + valueMap.put("timestamp", new Timestamp(time)); + valueMap.put("record", null); + valueMap.put("array", null); + valueMap.put("choice", 48L); + + final Record record = new MapRecord(schema, valueMap); + final RecordSet rs = RecordSet.of(schema, record); + + final String output; + try (final ByteArrayOutputStream baos = new ByteArrayOutputStream()) { + writer.write(rs, baos); + output = baos.toString(); + } + + final String expected = new String(Files.readAllBytes(Paths.get("src/test/resources/json/output/dataTypes.json"))); + assertEquals(expected, output); + } + +} http://git-wip-us.apache.org/repos/asf/nifi/blob/a88d3bfa/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/csv/extra-white-space.csv ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/csv/extra-white-space.csv b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/csv/extra-white-space.csv new file mode 100644 index 0000000..4531083 --- /dev/null +++ b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/csv/extra-white-space.csv @@ -0,0 +1,9 @@ +id, name, balance, address, city, state, zipCode, country +1, John Doe, "4750.89", "123 My Street", My City, MS, 11111, USA + + + + + 2, Jane Doe, 4820.09, 321 Your Street, Your City, NY, 33333, USA + + http://git-wip-us.apache.org/repos/asf/nifi/blob/a88d3bfa/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/csv/multi-bank-account.csv ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/csv/multi-bank-account.csv b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/csv/multi-bank-account.csv new file mode 100644 index 0000000..9f761c6 --- /dev/null +++ b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/csv/multi-bank-account.csv @@ -0,0 +1,3 @@ +id, name, balance, address, city, state, zipCode, country +1, John Doe, "4750.89", "123 My Street", My City, MS, 11111, USA +2, Jane Doe, 4820.09, 321 Your Street, Your City, NY, 33333, USA \ No newline at end of file http://git-wip-us.apache.org/repos/asf/nifi/blob/a88d3bfa/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/csv/single-bank-account.csv ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/csv/single-bank-account.csv b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/csv/single-bank-account.csv new file mode 100644 index 0000000..11ce6d4 --- /dev/null +++ b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/csv/single-bank-account.csv @@ -0,0 +1,2 @@ +id, name, balance, address, city, state, zipCode, country +1, John Doe, "4750.89", "123 My Street", My City, MS, 11111, USA \ No newline at end of file http://git-wip-us.apache.org/repos/asf/nifi/blob/a88d3bfa/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/grok/error-with-stack-trace.log ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/grok/error-with-stack-trace.log b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/grok/error-with-stack-trace.log new file mode 100644 index 0000000..43f1b56 --- /dev/null +++ b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/grok/error-with-stack-trace.log @@ -0,0 +1,25 @@ +2016-11-23 16:00:00,000 INFO message without stack trace +2016-11-23 16:00:02,689 ERROR Log message with stack trace +org.apache.nifi.exception.UnitTestException: Testing to ensure we are able to capture stack traces + at org.apache.nifi.cluster.coordination.node.NodeClusterCoordinator.getElectedActiveCoordinatorAddress(NodeClusterCoordinator.java:185) [nifi-framework-cluster-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT] + at org.apache.nifi.cluster.coordination.node.NodeClusterCoordinator.getElectedActiveCoordinatorAddress(NodeClusterCoordinator.java:185) [nifi-framework-cluster-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT] + at org.apache.nifi.cluster.coordination.node.NodeClusterCoordinator.getElectedActiveCoordinatorAddress(NodeClusterCoordinator.java:185) [nifi-framework-cluster-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT] + at org.apache.nifi.cluster.coordination.node.NodeClusterCoordinator.getElectedActiveCoordinatorAddress(NodeClusterCoordinator.java:185) [nifi-framework-cluster-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT] + at org.apache.nifi.cluster.coordination.node.NodeClusterCoordinator.getElectedActiveCoordinatorAddress(NodeClusterCoordinator.java:185) [nifi-framework-cluster-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT] + at org.apache.nifi.cluster.coordination.node.NodeClusterCoordinator.getElectedActiveCoordinatorAddress(NodeClusterCoordinator.java:185) [nifi-framework-cluster-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT] + at org.apache.nifi.cluster.coordination.node.NodeClusterCoordinator.getElectedActiveCoordinatorAddress(NodeClusterCoordinator.java:185) [nifi-framework-cluster-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT] + at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) [na:1.8.0_45] + at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) [na:1.8.0_45] + at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) [na:1.8.0_45] + at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) [na:1.8.0_45] + at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) [na:1.8.0_45] + at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [na:1.8.0_45] + at java.lang.Thread.run(Thread.java:745) [na:1.8.0_45] +Caused by: org.apache.nifi.exception.UnitTestException: Testing to ensure we are able to capture stack traces + at org.apache.nifi.cluster.coordination.node.NodeClusterCoordinator.getElectedActiveCoordinatorAddress(NodeClusterCoordinator.java:185) [nifi-framework-cluster-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT] + at org.apache.nifi.cluster.coordination.node.NodeClusterCoordinator.getElectedActiveCoordinatorAddress(NodeClusterCoordinator.java:185) [nifi-framework-cluster-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT] + at org.apache.nifi.cluster.coordination.node.NodeClusterCoordinator.getElectedActiveCoordinatorAddress(NodeClusterCoordinator.java:185) [nifi-framework-cluster-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT] + at org.apache.nifi.cluster.coordination.node.NodeClusterCoordinator.getElectedActiveCoordinatorAddress(NodeClusterCoordinator.java:185) [nifi-framework-cluster-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT] + at org.apache.nifi.cluster.coordination.node.NodeClusterCoordinator.getElectedActiveCoordinatorAddress(NodeClusterCoordinator.java:185) [nifi-framework-cluster-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT] + ... 12 common frames omitted +2016-11-23 16:05:00,000 INFO message without stack trace http://git-wip-us.apache.org/repos/asf/nifi/blob/a88d3bfa/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/grok/nifi-log-sample-multiline-with-stacktrace.log ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/grok/nifi-log-sample-multiline-with-stacktrace.log b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/grok/nifi-log-sample-multiline-with-stacktrace.log new file mode 100644 index 0000000..673908f --- /dev/null +++ b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/grok/nifi-log-sample-multiline-with-stacktrace.log @@ -0,0 +1,29 @@ +2016-08-04 13:26:32,473 INFO [Leader Election Notification Thread-1] o.a.n.c.l.e.CuratorLeaderElectionManager org.apache.nifi.controller.leader.election.CuratorLeaderElectionManager$ElectionListener@1fa27ea5 has been interrupted; no longer leader for role 'Cluster Coordinator' +2016-08-04 13:26:32,473 INFO [Leader Election Notification Thread-1] o.a.n.c.l.e.CuratorLeaderElectionManager org.apache.nifi.controller.leader.election.CuratorLeaderElectionManager$ElectionListener@1fa27ea5 This node is no longer leader for role 'Cluster Coordinator' +2016-08-04 13:26:32,474 ERROR [Leader Election Notification Thread-2] o.apache.nifi.controller.FlowController One +Two +Three +org.apache.nifi.exception.UnitTestException: Testing to ensure we are able to capture stack traces + at org.apache.nifi.cluster.coordination.node.NodeClusterCoordinator.getElectedActiveCoordinatorAddress(NodeClusterCoordinator.java:185) [nifi-framework-cluster-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT] + at org.apache.nifi.cluster.coordination.node.NodeClusterCoordinator.getElectedActiveCoordinatorAddress(NodeClusterCoordinator.java:185) [nifi-framework-cluster-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT] + at org.apache.nifi.cluster.coordination.node.NodeClusterCoordinator.getElectedActiveCoordinatorAddress(NodeClusterCoordinator.java:185) [nifi-framework-cluster-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT] + at org.apache.nifi.cluster.coordination.node.NodeClusterCoordinator.getElectedActiveCoordinatorAddress(NodeClusterCoordinator.java:185) [nifi-framework-cluster-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT] + at org.apache.nifi.cluster.coordination.node.NodeClusterCoordinator.getElectedActiveCoordinatorAddress(NodeClusterCoordinator.java:185) [nifi-framework-cluster-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT] + at org.apache.nifi.cluster.coordination.node.NodeClusterCoordinator.getElectedActiveCoordinatorAddress(NodeClusterCoordinator.java:185) [nifi-framework-cluster-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT] + at org.apache.nifi.cluster.coordination.node.NodeClusterCoordinator.getElectedActiveCoordinatorAddress(NodeClusterCoordinator.java:185) [nifi-framework-cluster-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT] + at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) [na:1.8.0_45] + at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) [na:1.8.0_45] + at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) [na:1.8.0_45] + at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) [na:1.8.0_45] + at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) [na:1.8.0_45] + at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [na:1.8.0_45] + at java.lang.Thread.run(Thread.java:745) [na:1.8.0_45] +Caused by: org.apache.nifi.exception.UnitTestException: Testing to ensure we are able to capture stack traces + at org.apache.nifi.cluster.coordination.node.NodeClusterCoordinator.getElectedActiveCoordinatorAddress(NodeClusterCoordinator.java:185) [nifi-framework-cluster-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT] + at org.apache.nifi.cluster.coordination.node.NodeClusterCoordinator.getElectedActiveCoordinatorAddress(NodeClusterCoordinator.java:185) [nifi-framework-cluster-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT] + at org.apache.nifi.cluster.coordination.node.NodeClusterCoordinator.getElectedActiveCoordinatorAddress(NodeClusterCoordinator.java:185) [nifi-framework-cluster-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT] + at org.apache.nifi.cluster.coordination.node.NodeClusterCoordinator.getElectedActiveCoordinatorAddress(NodeClusterCoordinator.java:185) [nifi-framework-cluster-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT] + at org.apache.nifi.cluster.coordination.node.NodeClusterCoordinator.getElectedActiveCoordinatorAddress(NodeClusterCoordinator.java:185) [nifi-framework-cluster-1.0.0-SNAPSHOT.jar:1.0.0-SNAPSHOT] + ... 12 common frames omitted +2016-08-04 13:26:35,475 WARN [Curator-Framework-0] org.apache.curator.ConnectionState Connection attempt unsuccessful after 3008 (greater than max timeout of 3000). Resetting connection and trying again with a new connection. +2016-08-04 13:26:35,479 WARN [Curator-Framework-0] org.apache.curator.ConnectionState Connection attempt unsuccessful after 3007 (greater than max timeout of 3000). Resetting connection and trying again with a new connection. \ No newline at end of file http://git-wip-us.apache.org/repos/asf/nifi/blob/a88d3bfa/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/grok/nifi-log-sample.log ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/grok/nifi-log-sample.log b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/grok/nifi-log-sample.log new file mode 100644 index 0000000..e4c7385 --- /dev/null +++ b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/grok/nifi-log-sample.log @@ -0,0 +1,5 @@ +2016-08-04 13:26:32,473 INFO [Leader Election Notification Thread-1] o.a.n.c.l.e.CuratorLeaderElectionManager org.apache.nifi.controller.leader.election.CuratorLeaderElectionManager$ElectionListener@1fa27ea5 has been interrupted; no longer leader for role 'Cluster Coordinator' +2016-08-04 13:26:32,473 INFO [Leader Election Notification Thread-1] o.a.n.c.l.e.CuratorLeaderElectionManager org.apache.nifi.controller.leader.election.CuratorLeaderElectionManager$ElectionListener@1fa27ea5 This node is no longer leader for role 'Cluster Coordinator' +2016-08-04 13:26:32,474 INFO [Leader Election Notification Thread-2] o.apache.nifi.controller.FlowController This node is no longer Primary Node +2016-08-04 13:26:35,475 WARN [Curator-Framework-0] org.apache.curator.ConnectionState Connection attempt unsuccessful after 3008 (greater than max timeout of 3000). Resetting connection and trying again with a new connection. +2016-08-04 13:26:35,479 WARN [Curator-Framework-0] org.apache.curator.ConnectionState Connection attempt unsuccessful after 3007 (greater than max timeout of 3000). Resetting connection and trying again with a new connection. \ No newline at end of file http://git-wip-us.apache.org/repos/asf/nifi/blob/a88d3bfa/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/grok/single-line-log-messages.txt ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/grok/single-line-log-messages.txt b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/grok/single-line-log-messages.txt new file mode 100644 index 0000000..737ad95 --- /dev/null +++ b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/grok/single-line-log-messages.txt @@ -0,0 +1,5 @@ +2016-11-08 21:24:23,029 INFO Test Message 1 +2016-11-08 21:24:23,029 WARN Red +2016-11-08 21:24:23,029 ERROR Green +2016-11-08 21:24:23,029 FATAL Blue +2016-11-08 21:24:23,029 FINE Yellow \ No newline at end of file http://git-wip-us.apache.org/repos/asf/nifi/blob/a88d3bfa/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/json/bank-account-array-different-schemas.json ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/json/bank-account-array-different-schemas.json b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/json/bank-account-array-different-schemas.json new file mode 100644 index 0000000..e79a6d7 --- /dev/null +++ b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/json/bank-account-array-different-schemas.json @@ -0,0 +1,30 @@ +[ + { + "id": 1, + "name": "John Doe", + "balance": 4750.89, + "address": "123 My Street", + "city": "My City", + "state": "MS", + "zipCode": "11111", + "country": "USA" + }, { + "id": 2, + "name": "Jane Doe", + "balance": 4820.09, + "address": "321 Your Street", + "city": "Your City", + "state": "NY", + "zipCode": "33333" + }, { + "id": 3, + "name": "Jake Doe", + "balance": 4751.89, + "address": "124 My Street", + "address2": "Apt. #12", + "city": "My City", + "state": "MS", + "zipCode": "11111", + "country": "USA" + } +] \ No newline at end of file http://git-wip-us.apache.org/repos/asf/nifi/blob/a88d3bfa/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/json/bank-account-array-optional-balance.json ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/json/bank-account-array-optional-balance.json b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/json/bank-account-array-optional-balance.json new file mode 100644 index 0000000..cb614f1 --- /dev/null +++ b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/json/bank-account-array-optional-balance.json @@ -0,0 +1,29 @@ +[ + { + "id": 1, + "name": "John Doe", + "balance": 4750.89, + "address": "123 My Street", + "city": "My City", + "state": "MS", + "zipCode": "11111", + "country": "USA" + }, { + "id": 2, + "name": "Jane Doe", + "balance": null, + "address": "321 Your Street", + "city": "Your City", + "state": "NY", + "zipCode": "33333", + "country": "USA" + }, { + "id": 3, + "name": "Jimmy Doe", + "address": "321 Your Street", + "city": "Your City", + "state": "NY", + "zipCode": "33333", + "country": "USA" + } +] \ No newline at end of file http://git-wip-us.apache.org/repos/asf/nifi/blob/a88d3bfa/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/json/bank-account-array.json ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/json/bank-account-array.json b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/json/bank-account-array.json new file mode 100644 index 0000000..d821cc1 --- /dev/null +++ b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/json/bank-account-array.json @@ -0,0 +1,21 @@ +[ + { + "id": 1, + "name": "John Doe", + "balance": 4750.89, + "address": "123 My Street", + "city": "My City", + "state": "MS", + "zipCode": "11111", + "country": "USA" + }, { + "id": 2, + "name": "Jane Doe", + "balance": 4820.09, + "address": "321 Your Street", + "city": "Your City", + "state": "NY", + "zipCode": "33333", + "country": "USA" + } +] \ No newline at end of file http://git-wip-us.apache.org/repos/asf/nifi/blob/a88d3bfa/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/json/json-with-unicode.json ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/json/json-with-unicode.json b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/json/json-with-unicode.json new file mode 100644 index 0000000..880fabf --- /dev/null +++ b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/json/json-with-unicode.json @@ -0,0 +1,9 @@ +{ + "created_at":"Thu Feb 16 01:19:42 +0000 2017", + "id":832036744985577473, + "unicode":"\u3061\u3083\u6ce3\u304d\u305d\u3046", + "from":{ + "id":788946702264507903, + "name":"john" + } +} http://git-wip-us.apache.org/repos/asf/nifi/blob/a88d3bfa/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/json/output/dataTypes.json ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/json/output/dataTypes.json b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/json/output/dataTypes.json new file mode 100644 index 0000000..40c28dd --- /dev/null +++ b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/json/output/dataTypes.json @@ -0,0 +1,18 @@ +[ { + "string" : "string", + "boolean" : true, + "byte" : 1, + "char" : "c", + "short" : 8, + "int" : 9, + "bigint" : 8, + "long" : 8, + "float" : 8.0, + "double" : 8.0, + "date" : "2017-01-01", + "time" : "17:00:00", + "timestamp" : "2017-01-01 17:00:00", + "record" : null, + "choice" : 48, + "array" : null +} ] \ No newline at end of file http://git-wip-us.apache.org/repos/asf/nifi/blob/a88d3bfa/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/json/primitive-type-array.json ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/json/primitive-type-array.json b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/json/primitive-type-array.json new file mode 100644 index 0000000..47e3276 --- /dev/null +++ b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/json/primitive-type-array.json @@ -0,0 +1,13 @@ +[ + { + "id": 1, + "name": "John Doe", + "balance": 4750.89, + "address": "123 My Street", + "city": "My City", + "state": "MS", + "zipCode": "11111", + "country": "USA", + "accountIds": [1, 2, 3] + } +] \ No newline at end of file http://git-wip-us.apache.org/repos/asf/nifi/blob/a88d3bfa/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/json/single-bank-account.json ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/json/single-bank-account.json b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/json/single-bank-account.json new file mode 100644 index 0000000..a8d6890 --- /dev/null +++ b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/json/single-bank-account.json @@ -0,0 +1,10 @@ +{ + "id": 1, + "name": "John Doe", + "balance": 4750.89, + "address": "123 My Street", + "city": "My City", + "state": "MS", + "zipCode": "11111", + "country": "USA" +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/nifi/blob/a88d3bfa/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/json/single-element-nested-array.json ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/json/single-element-nested-array.json b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/json/single-element-nested-array.json new file mode 100644 index 0000000..0bca3a0 --- /dev/null +++ b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/json/single-element-nested-array.json @@ -0,0 +1,16 @@ +{ + "id": 1, + "name": "John Doe", + "address": "123 My Street", + "city": "My City", + "state": "MS", + "zipCode": "11111", + "country": "USA", + "accounts": [{ + "id": 42, + "balance": 4750.89 + }, { + "id": 43, + "balance": 48212.38 + }] +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/nifi/blob/a88d3bfa/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/json/single-element-nested.json ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/json/single-element-nested.json b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/json/single-element-nested.json new file mode 100644 index 0000000..26a59e1 --- /dev/null +++ b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/json/single-element-nested.json @@ -0,0 +1,13 @@ +{ + "id": 1, + "name": "John Doe", + "address": "123 My Street", + "city": "My City", + "state": "MS", + "zipCode": "11111", + "country": "USA", + "account": { + "id": 42, + "balance": 4750.89 + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/nifi/blob/a88d3bfa/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/pom.xml ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/pom.xml b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/pom.xml new file mode 100644 index 0000000..b1ac470 --- /dev/null +++ b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/pom.xml @@ -0,0 +1,30 @@ +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-standard-services</artifactId> + <version>1.2.0-SNAPSHOT</version> + </parent> + + <artifactId>nifi-record-serialization-services-bundle</artifactId> + <packaging>pom</packaging> + + <modules> + <module>nifi-record-serialization-services</module> + <module>nifi-record-serialization-services-nar</module> + </modules> +</project> http://git-wip-us.apache.org/repos/asf/nifi/blob/a88d3bfa/nifi-nar-bundles/nifi-standard-services/nifi-standard-services-api-nar/pom.xml ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-standard-services-api-nar/pom.xml b/nifi-nar-bundles/nifi-standard-services/nifi-standard-services-api-nar/pom.xml index c6e6128..eae3515 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-standard-services-api-nar/pom.xml +++ b/nifi-nar-bundles/nifi-standard-services/nifi-standard-services-api-nar/pom.xml @@ -56,5 +56,10 @@ <artifactId>nifi-hbase-client-service-api</artifactId> <scope>compile</scope> </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-record-serialization-service-api</artifactId> + <scope>compile</scope> + </dependency> </dependencies> </project> http://git-wip-us.apache.org/repos/asf/nifi/blob/a88d3bfa/nifi-nar-bundles/nifi-standard-services/pom.xml ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-standard-services/pom.xml b/nifi-nar-bundles/nifi-standard-services/pom.xml index 7e754be..3948a1b 100644 --- a/nifi-nar-bundles/nifi-standard-services/pom.xml +++ b/nifi-nar-bundles/nifi-standard-services/pom.xml @@ -35,5 +35,7 @@ <module>nifi-dbcp-service-bundle</module> <module>nifi-hbase-client-service-api</module> <module>nifi-hbase_1_1_2-client-service-bundle</module> + <module>nifi-record-serialization-service-api</module> + <module>nifi-record-serialization-services-bundle</module> </modules> </project> http://git-wip-us.apache.org/repos/asf/nifi/blob/a88d3bfa/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 449450d..057832b 100644 --- a/pom.xml +++ b/pom.xml @@ -966,6 +966,11 @@ language governing permissions and limitations under the License. --> </dependency> <dependency> <groupId>org.apache.nifi</groupId> + <artifactId>nifi-record-serialization-service-api</artifactId> + <version>1.2.0-SNAPSHOT</version> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> <artifactId>nifi-distributed-cache-services-nar</artifactId> <version>1.2.0-SNAPSHOT</version> <type>nar</type> @@ -1293,6 +1298,12 @@ language governing permissions and limitations under the License. --> <version>1.2.0-SNAPSHOT</version> <type>nar</type> </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-record-serialization-services-nar</artifactId> + <version>1.2.0-SNAPSHOT</version> + <type>nar</type> + </dependency> <dependency> <groupId>org.apache.nifi</groupId> <artifactId>nifi-evtx-nar</artifactId>
