Github user mattyb149 commented on a diff in the pull request:
https://github.com/apache/nifi/pull/709#discussion_r75773261
--- Diff:
nifi-nar-bundles/nifi-kite-bundle/nifi-kite-processors/src/test/java/org/apache/nifi/processors/kite/TestCSVToAvroProcessor.java
---
@@ -191,57 +192,40 @@ public void testEmptyContent() throws IOException {
}
@Test
- public void testCSVProperties() throws IOException {
+ public void testBasicConversionNoErrors() throws IOException {
TestRunner runner =
TestRunners.newTestRunner(ConvertCSVToAvro.class);
- ConvertCSVToAvro processor = new ConvertCSVToAvro();
- ProcessContext context = runner.getProcessContext();
-
- // check defaults
- processor.createCSVProperties(context);
- Assert.assertEquals("Charset should match",
- "utf8", processor.props.charset);
- Assert.assertEquals("Delimiter should match",
- ",", processor.props.delimiter);
- Assert.assertEquals("Quote should match",
- "\"", processor.props.quote);
- Assert.assertEquals("Escape should match",
- "\\", processor.props.escape);
- Assert.assertEquals("Header flag should match",
- false, processor.props.useHeader);
- Assert.assertEquals("Lines to skip should match",
- 0, processor.props.linesToSkip);
+ runner.assertNotValid();
+ runner.setProperty(ConvertCSVToAvro.SCHEMA, SCHEMA.toString());
+ runner.assertValid();
- runner.setProperty(ConvertCSVToAvro.CHARSET, "utf16");
- runner.setProperty(ConvertCSVToAvro.DELIMITER, "|");
- runner.setProperty(ConvertCSVToAvro.QUOTE, "'");
- runner.setProperty(ConvertCSVToAvro.ESCAPE, "\u2603");
- runner.setProperty(ConvertCSVToAvro.HAS_HEADER, "true");
- runner.setProperty(ConvertCSVToAvro.LINES_TO_SKIP, "2");
-
- // check updates
- processor.createCSVProperties(context);
- Assert.assertEquals("Charset should match",
- "utf16", processor.props.charset);
- Assert.assertEquals("Delimiter should match",
- "|", processor.props.delimiter);
- Assert.assertEquals("Quote should match",
- "'", processor.props.quote);
- Assert.assertEquals("Escape should match",
- "\u2603", processor.props.escape);
- Assert.assertEquals("Header flag should match",
- true, processor.props.useHeader);
- Assert.assertEquals("Lines to skip should match",
- 2, processor.props.linesToSkip);
+ runner.enqueue(streamFor("1,green\n2,blue,\n3,grey,12.95"));
+ runner.run();
+
+ long converted = runner.getCounterValue("Converted records");
+ long errors = runner.getCounterValue("Conversion errors");
+ Assert.assertEquals("Should convert 3 rows", 3, converted);
+ Assert.assertEquals("Should reject 0 row", 0, errors);
+
+ runner.assertTransferCount("success", 1);
+ runner.assertTransferCount("failure", 0);
+ runner.assertTransferCount("incompatible", 0);
}
@Test
- public void testBasicConversionNoErrors() throws IOException {
+ public void testExpresionLanguageBasedCSVProperties() throws
IOException {
TestRunner runner =
TestRunners.newTestRunner(ConvertCSVToAvro.class);
runner.assertNotValid();
runner.setProperty(ConvertCSVToAvro.SCHEMA, SCHEMA.toString());
runner.assertValid();
- runner.enqueue(streamFor("1,green\n2,blue,\n3,grey,12.95"));
+ runner.setProperty(ConvertCSVToAvro.DELIMITER, "${csv.delimiter}");
+ runner.setProperty(ConvertCSVToAvro.QUOTE, "${csv.quote}");
+
+ HashMap<String, String> flowFileAttributes = new
HashMap<String,String>();
+ flowFileAttributes.put("csv.delimiter", "|");
+ flowFileAttributes.put("csv.quote", "~");
+
+
runner.enqueue(streamFor("1|green\n2|~blue|field~|\n3|grey|12.95"),
flowFileAttributes);
--- End diff --
Might be helpful to check that the fields are converted as expected (by
reading the resulting Avro file(s)) instead of just the total number of records
produced.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at [email protected] or file a JIRA ticket
with INFRA.
---
