NIFI-1613: ConvertJSONToSQL truncates numeric value wrongly. - Changed boolean value conversion to use Boolean.valueOf. - Updated comments in source code to reflect current situation more clearly. - Updated tests those have been added since the original commits were made.
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/3844a821 Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/3844a821 Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/3844a821 Branch: refs/heads/master Commit: 3844a821f18e6a76e44e289c923be333e49d5251 Parents: 3b2e43b Author: Koji Kawamura <[email protected]> Authored: Wed Jul 5 14:49:32 2017 +0900 Committer: Matt Burgess <[email protected]> Committed: Wed Jul 12 17:02:55 2017 -0400 ---------------------------------------------------------------------- .../processors/standard/ConvertJSONToSQL.java | 21 ++--- .../standard/TestConvertJSONToSQL.java | 90 -------------------- 2 files changed, 6 insertions(+), 105 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/nifi/blob/3844a821/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ConvertJSONToSQL.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ConvertJSONToSQL.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ConvertJSONToSQL.java index ca92ad4..2f2123a 100755 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ConvertJSONToSQL.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ConvertJSONToSQL.java @@ -514,19 +514,10 @@ public class ConvertJSONToSQL extends AbstractProcessor { // only "true" is considered true, everything else is false case Types.BOOLEAN: - switch (fieldValue==null?"":fieldValue) { - case "true": - fieldValue = "true"; - break; - - default: - fieldValue = "false"; - break; - } + fieldValue = Boolean.valueOf(fieldValue).toString(); break; // Don't truncate numeric types. - // Should we check value is indeed number and throw error if not? case Types.TINYINT: case Types.SMALLINT: case Types.INTEGER: @@ -538,11 +529,11 @@ public class ConvertJSONToSQL extends AbstractProcessor { case Types.NUMERIC: break; - // Don't truncate date and time types. - // Should we check value is indeed correct date and/or time and throw error if not? - // We assume date and time is already correct, but because ConvertJSONToSQL is often used together with PutSQL - // maybe we should assure PutSQL correctly understands date and time values. - // Currently PutSQL expect Long numeric values. But JSON usually uses ISO 8601, for example: 2012-04-23T18:25:43.511Z for dates. + // Don't truncate DATE, TIME and TIMESTAMP types. We assume date and time is already correct in long representation. + // Specifically, milliseconds since January 1, 1970, 00:00:00 GMT + // However, for TIMESTAMP, PutSQL accepts optional timestamp format via FlowFile attribute. + // See PutSQL.setParameter method and NIFI-3430 for detail. + // Alternatively, user can use JSONTreeReader and PutDatabaseRecord to handle date format more efficiently. case Types.DATE: case Types.TIME: case Types.TIMESTAMP: http://git-wip-us.apache.org/repos/asf/nifi/blob/3844a821/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestConvertJSONToSQL.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestConvertJSONToSQL.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestConvertJSONToSQL.java index 3f9c52c..2cd66eb 100755 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestConvertJSONToSQL.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestConvertJSONToSQL.java @@ -106,18 +106,9 @@ public class TestConvertJSONToSQL { @Test public void testInsertQuotedIdentifiers() throws InitializationException, ProcessException, SQLException, IOException { final TestRunner runner = TestRunners.newTestRunner(ConvertJSONToSQL.class); - final File tempDir = folder.getRoot(); - final File dbDir = new File(tempDir, "db"); - final DBCPService service = new MockDBCPService(dbDir.getAbsolutePath()); runner.addControllerService("dbcp", service); runner.enableControllerService(service); - try (final Connection conn = service.getConnection()) { - try (final Statement stmt = conn.createStatement()) { - stmt.executeUpdate(createPersons); - } - } - runner.setProperty(ConvertJSONToSQL.CONNECTION_POOL, "dbcp"); runner.setProperty(ConvertJSONToSQL.TABLE_NAME, "PERSONS"); runner.setProperty(ConvertJSONToSQL.STATEMENT_TYPE, "INSERT"); @@ -143,18 +134,9 @@ public class TestConvertJSONToSQL { @Test public void testInsertQuotedTableIdentifier() throws InitializationException, ProcessException, SQLException, IOException { final TestRunner runner = TestRunners.newTestRunner(ConvertJSONToSQL.class); - final File tempDir = folder.getRoot(); - final File dbDir = new File(tempDir, "db"); - final DBCPService service = new MockDBCPService(dbDir.getAbsolutePath()); runner.addControllerService("dbcp", service); runner.enableControllerService(service); - try (final Connection conn = service.getConnection()) { - try (final Statement stmt = conn.createStatement()) { - stmt.executeUpdate(createPersons); - } - } - runner.setProperty(ConvertJSONToSQL.CONNECTION_POOL, "dbcp"); runner.setProperty(ConvertJSONToSQL.TABLE_NAME, "PERSONS"); runner.setProperty(ConvertJSONToSQL.STATEMENT_TYPE, "INSERT"); @@ -234,18 +216,9 @@ public class TestConvertJSONToSQL { @Test public void testUpdateQuotedTableIdentifier() throws InitializationException, ProcessException, SQLException, IOException { final TestRunner runner = TestRunners.newTestRunner(ConvertJSONToSQL.class); - final File tempDir = folder.getRoot(); - final File dbDir = new File(tempDir, "db"); - final DBCPService service = new MockDBCPService(dbDir.getAbsolutePath()); runner.addControllerService("dbcp", service); runner.enableControllerService(service); - try (final Connection conn = service.getConnection()) { - try (final Statement stmt = conn.createStatement()) { - stmt.executeUpdate(createPersons); - } - } - runner.setProperty(ConvertJSONToSQL.CONNECTION_POOL, "dbcp"); runner.setProperty(ConvertJSONToSQL.TABLE_NAME, "PERSONS"); runner.setProperty(ConvertJSONToSQL.STATEMENT_TYPE, "UPDATE"); @@ -297,18 +270,9 @@ public class TestConvertJSONToSQL { @Test public void testMultipleInsertsQuotedIdentifiers() throws InitializationException, ProcessException, SQLException, IOException { final TestRunner runner = TestRunners.newTestRunner(ConvertJSONToSQL.class); - final File tempDir = folder.getRoot(); - final File dbDir = new File(tempDir, "db"); - final DBCPService service = new MockDBCPService(dbDir.getAbsolutePath()); runner.addControllerService("dbcp", service); runner.enableControllerService(service); - try (final Connection conn = service.getConnection()) { - try (final Statement stmt = conn.createStatement()) { - stmt.executeUpdate(createPersons); - } - } - runner.setProperty(ConvertJSONToSQL.CONNECTION_POOL, "dbcp"); runner.setProperty(ConvertJSONToSQL.TABLE_NAME, "PERSONS"); runner.setProperty(ConvertJSONToSQL.STATEMENT_TYPE, "INSERT"); @@ -359,18 +323,9 @@ public class TestConvertJSONToSQL { @Test public void testUpdateBasedOnPrimaryKeyQuotedIdentifier() throws InitializationException, ProcessException, SQLException, IOException { final TestRunner runner = TestRunners.newTestRunner(ConvertJSONToSQL.class); - final File tempDir = folder.getRoot(); - final File dbDir = new File(tempDir, "db"); - final DBCPService service = new MockDBCPService(dbDir.getAbsolutePath()); runner.addControllerService("dbcp", service); runner.enableControllerService(service); - try (final Connection conn = service.getConnection()) { - try (final Statement stmt = conn.createStatement()) { - stmt.executeUpdate(createPersons); - } - } - runner.setProperty(ConvertJSONToSQL.CONNECTION_POOL, "dbcp"); runner.setProperty(ConvertJSONToSQL.TABLE_NAME, "PERSONS"); runner.setProperty(ConvertJSONToSQL.STATEMENT_TYPE, "UPDATE"); @@ -449,18 +404,9 @@ public class TestConvertJSONToSQL { @Test public void testUpdateBasedOnUpdateKeyQuotedIdentifier() throws InitializationException, ProcessException, SQLException, IOException { final TestRunner runner = TestRunners.newTestRunner(ConvertJSONToSQL.class); - final File tempDir = folder.getRoot(); - final File dbDir = new File(tempDir, "db"); - final DBCPService service = new MockDBCPService(dbDir.getAbsolutePath()); runner.addControllerService("dbcp", service); runner.enableControllerService(service); - try (final Connection conn = service.getConnection()) { - try (final Statement stmt = conn.createStatement()) { - stmt.executeUpdate(createPersons); - } - } - runner.setProperty(ConvertJSONToSQL.CONNECTION_POOL, "dbcp"); runner.setProperty(ConvertJSONToSQL.TABLE_NAME, "PERSONS"); runner.setProperty(ConvertJSONToSQL.STATEMENT_TYPE, "UPDATE"); @@ -796,18 +742,9 @@ public class TestConvertJSONToSQL { @Test public void testDelete() throws InitializationException, ProcessException, SQLException, IOException { final TestRunner runner = TestRunners.newTestRunner(ConvertJSONToSQL.class); - final File tempDir = folder.getRoot(); - final File dbDir = new File(tempDir, "db"); - final DBCPService service = new MockDBCPService(dbDir.getAbsolutePath()); runner.addControllerService("dbcp", service); runner.enableControllerService(service); - try (final Connection conn = service.getConnection()) { - try (final Statement stmt = conn.createStatement()) { - stmt.executeUpdate(createPersons); - } - } - runner.setProperty(ConvertJSONToSQL.CONNECTION_POOL, "dbcp"); runner.setProperty(ConvertJSONToSQL.TABLE_NAME, "PERSONS"); runner.setProperty(ConvertJSONToSQL.STATEMENT_TYPE, "DELETE"); @@ -831,18 +768,9 @@ public class TestConvertJSONToSQL { @Test public void testDeleteQuotedIdentifiers() throws InitializationException, ProcessException, SQLException, IOException { final TestRunner runner = TestRunners.newTestRunner(ConvertJSONToSQL.class); - final File tempDir = folder.getRoot(); - final File dbDir = new File(tempDir, "db"); - final DBCPService service = new MockDBCPService(dbDir.getAbsolutePath()); runner.addControllerService("dbcp", service); runner.enableControllerService(service); - try (final Connection conn = service.getConnection()) { - try (final Statement stmt = conn.createStatement()) { - stmt.executeUpdate(createPersons); - } - } - runner.setProperty(ConvertJSONToSQL.CONNECTION_POOL, "dbcp"); runner.setProperty(ConvertJSONToSQL.TABLE_NAME, "PERSONS"); runner.setProperty(ConvertJSONToSQL.STATEMENT_TYPE, "DELETE"); @@ -868,18 +796,9 @@ public class TestConvertJSONToSQL { @Test public void testDeleteQuotedTableIdentifier() throws InitializationException, ProcessException, SQLException, IOException { final TestRunner runner = TestRunners.newTestRunner(ConvertJSONToSQL.class); - final File tempDir = folder.getRoot(); - final File dbDir = new File(tempDir, "db"); - final DBCPService service = new MockDBCPService(dbDir.getAbsolutePath()); runner.addControllerService("dbcp", service); runner.enableControllerService(service); - try (final Connection conn = service.getConnection()) { - try (final Statement stmt = conn.createStatement()) { - stmt.executeUpdate(createPersons); - } - } - runner.setProperty(ConvertJSONToSQL.CONNECTION_POOL, "dbcp"); runner.setProperty(ConvertJSONToSQL.TABLE_NAME, "PERSONS"); runner.setProperty(ConvertJSONToSQL.STATEMENT_TYPE, "DELETE"); @@ -905,18 +824,9 @@ public class TestConvertJSONToSQL { @Test public void testDeleteWithNullValue() throws InitializationException, ProcessException, SQLException, IOException { final TestRunner runner = TestRunners.newTestRunner(ConvertJSONToSQL.class); - final File tempDir = folder.getRoot(); - final File dbDir = new File(tempDir, "db"); - final DBCPService service = new MockDBCPService(dbDir.getAbsolutePath()); runner.addControllerService("dbcp", service); runner.enableControllerService(service); - try (final Connection conn = service.getConnection()) { - try (final Statement stmt = conn.createStatement()) { - stmt.executeUpdate(createPersons); - } - } - runner.setProperty(ConvertJSONToSQL.CONNECTION_POOL, "dbcp"); runner.setProperty(ConvertJSONToSQL.TABLE_NAME, "PERSONS"); runner.setProperty(ConvertJSONToSQL.STATEMENT_TYPE, "DELETE");
