Repository: nifi Updated Branches: refs/heads/master 824712bff -> 3f4b276b7
NIFI-3910: Fixed issue where CSVRecordReader throws an Exception if a field is missing instead of using a null value This closes #1807. Project: http://git-wip-us.apache.org/repos/asf/nifi/repo Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/3f4b276b Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/3f4b276b Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/3f4b276b Branch: refs/heads/master Commit: 3f4b276b713464ad0ccf13b5164e08d118e2cc25 Parents: 824712b Author: Mark Payne <[email protected]> Authored: Tue May 16 11:09:14 2017 -0400 Committer: Mark Payne <[email protected]> Committed: Tue May 16 13:16:46 2017 -0400 ---------------------------------------------------------------------- .../org/apache/nifi/csv/CSVRecordReader.java | 12 ++++--- .../apache/nifi/csv/TestCSVRecordReader.java | 33 ++++++++++++++++++++ 2 files changed, 40 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/nifi/blob/3f4b276b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/csv/CSVRecordReader.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/csv/CSVRecordReader.java b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/csv/CSVRecordReader.java index 3bcbce2..095f488 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/csv/CSVRecordReader.java +++ b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/csv/CSVRecordReader.java @@ -68,17 +68,19 @@ public class CSVRecordReader implements RecordReader { final Map<String, Object> rowValues = new HashMap<>(schema.getFieldCount()); for (final RecordField recordField : schema.getFields()) { - String rawValue = csvRecord.get(recordField.getFieldName()); - if (rawValue == null) { + String rawValue = null; + final String fieldName = recordField.getFieldName(); + if (csvRecord.isSet(fieldName)) { + rawValue = csvRecord.get(fieldName); + } else { for (final String alias : recordField.getAliases()) { - rawValue = csvRecord.get(alias); - if (rawValue != null) { + if (csvRecord.isSet(alias)) { + rawValue = csvRecord.get(alias); break; } } } - final String fieldName = recordField.getFieldName(); if (rawValue == null) { rowValues.put(fieldName, null); continue; http://git-wip-us.apache.org/repos/asf/nifi/blob/3f4b276b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/java/org/apache/nifi/csv/TestCSVRecordReader.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/java/org/apache/nifi/csv/TestCSVRecordReader.java b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/java/org/apache/nifi/csv/TestCSVRecordReader.java index cb790f1..a02e0b1 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/java/org/apache/nifi/csv/TestCSVRecordReader.java +++ b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/java/org/apache/nifi/csv/TestCSVRecordReader.java @@ -18,6 +18,7 @@ package org.apache.nifi.csv; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import java.io.ByteArrayInputStream; @@ -143,4 +144,36 @@ public class TestCSVRecordReader { assertNull(reader.nextRecord()); } } + + @Test + public void testMissingField() throws IOException, MalformedRecordException { + final List<RecordField> fields = getDefaultFields(); + fields.replaceAll(f -> f.getFieldName().equals("balance") ? new RecordField("balance", doubleDataType) : f); + + final RecordSchema schema = new SimpleRecordSchema(fields); + + final String headerLine = "id, name, balance, address, city, state, zipCode, country"; + final String inputRecord = "1, John, 40.80, 123 My Street, My City, MS, 11111"; + final String csvData = headerLine + "\n" + inputRecord; + final byte[] inputData = csvData.getBytes(); + + try (final InputStream baos = new ByteArrayInputStream(inputData)) { + final CSVRecordReader reader = new CSVRecordReader(baos, Mockito.mock(ComponentLog.class), schema, format, + RecordFieldType.DATE.getDefaultFormat(), RecordFieldType.TIME.getDefaultFormat(), RecordFieldType.TIMESTAMP.getDefaultFormat()); + + final Record record = reader.nextRecord(); + assertNotNull(record); + + assertEquals("1", record.getValue("id")); + assertEquals("John", record.getValue("name")); + assertEquals(40.8D, record.getValue("balance")); + assertEquals("123 My Street", record.getValue("address")); + assertEquals("My City", record.getValue("city")); + assertEquals("MS", record.getValue("state")); + assertEquals("11111", record.getValue("zipCode")); + assertNull(record.getValue("country")); + + assertNull(reader.nextRecord()); + } + } }
