NIFI-1685 Optimize database Junit tests to reduce execution time. Removed commented blocks.
This closes #304. Signed-off-by: Joe Skora <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/nifi/repo Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/65c18b6e Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/65c18b6e Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/65c18b6e Branch: refs/heads/NIFI-1654 Commit: 65c18b6ef55fe3fcfa34803f8a327a8432838e37 Parents: 7b5583f Author: Toivo Adams <[email protected]> Authored: Fri Mar 25 18:51:00 2016 +0200 Committer: Joe Skora <[email protected]> Committed: Sat Apr 16 20:33:01 2016 -0400 ---------------------------------------------------------------------- .../nifi/processors/standard/TestPutSQL.java | 179 ++++++------------- .../standard/util/TestJdbcCommon.java | 63 +++---- 2 files changed, 78 insertions(+), 164 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/nifi/blob/65c18b6e/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestPutSQL.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestPutSQL.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestPutSQL.java index 301ad75..b5ebca5 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestPutSQL.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestPutSQL.java @@ -39,39 +39,47 @@ import org.apache.nifi.util.MockFlowFile; import org.apache.nifi.util.TestRunner; import org.apache.nifi.util.TestRunners; import org.junit.BeforeClass; -import org.junit.Rule; +import org.junit.ClassRule; import org.junit.Test; import org.junit.rules.TemporaryFolder; import org.mockito.Mockito; public class TestPutSQL { private static final String createPersons = "CREATE TABLE PERSONS (id integer primary key, name varchar(100), code integer)"; - private static final String createPersonsAutoId = "CREATE TABLE PERSONS (id INTEGER NOT NULL GENERATED ALWAYS AS IDENTITY (START WITH 1), name VARCHAR(100), code INTEGER check(code <= 100))"; + private static final String createPersonsAutoId = "CREATE TABLE PERSONS_AI (id INTEGER NOT NULL GENERATED ALWAYS AS IDENTITY (START WITH 1), name VARCHAR(100), code INTEGER check(code <= 100))"; - @Rule - public TemporaryFolder folder = new TemporaryFolder(); + @ClassRule + public static TemporaryFolder folder = new TemporaryFolder(); + + /** + * Setting up Connection pooling is expensive operation. + * So let's do this only once and reuse MockDBCPService in each test. + */ + static protected DBCPService service; @BeforeClass - public static void setup() { + public static void setupClass() throws ProcessException, SQLException { System.setProperty("derby.stream.error.file", "target/derby.log"); - } - - @Test - public void testDirectStatements() throws InitializationException, ProcessException, SQLException, IOException { - final TestRunner runner = TestRunners.newTestRunner(PutSQL.class); final File tempDir = folder.getRoot(); final File dbDir = new File(tempDir, "db"); - final DBCPService service = new MockDBCPService(dbDir.getAbsolutePath()); - runner.addControllerService("dbcp", service); - runner.enableControllerService(service); - + service = new MockDBCPService(dbDir.getAbsolutePath()); try (final Connection conn = service.getConnection()) { try (final Statement stmt = conn.createStatement()) { stmt.executeUpdate(createPersons); + stmt.executeUpdate(createPersonsAutoId); } } + } + @Test + public void testDirectStatements() throws InitializationException, ProcessException, SQLException, IOException { + final TestRunner runner = TestRunners.newTestRunner(PutSQL.class); + runner.addControllerService("dbcp", service); + runner.enableControllerService(service); runner.setProperty(PutSQL.CONNECTION_POOL, "dbcp"); + + recreateTable("PERSONS", createPersons); + runner.enqueue("INSERT INTO PERSONS (ID, NAME, CODE) VALUES (1, 'Mark', 84)".getBytes()); runner.run(); @@ -107,21 +115,13 @@ public class TestPutSQL { @Test public void testInsertWithGeneratedKeys() throws InitializationException, ProcessException, SQLException, IOException { final TestRunner runner = TestRunners.newTestRunner(PutSQL.class); - final File tempDir = folder.getRoot(); - final File dbDir = new File(tempDir, "db"); - final DBCPService service = new MockDBCPService(dbDir.getAbsolutePath()); runner.addControllerService("dbcp", service); runner.enableControllerService(service); - - try (final Connection conn = service.getConnection()) { - try (final Statement stmt = conn.createStatement()) { - stmt.executeUpdate(createPersonsAutoId); - } - } - runner.setProperty(PutSQL.OBTAIN_GENERATED_KEYS, "true"); runner.setProperty(PutSQL.CONNECTION_POOL, "dbcp"); - runner.enqueue("INSERT INTO PERSONS (NAME, CODE) VALUES ('Mark', 84)".getBytes()); + + recreateTable("PERSONS_AI",createPersonsAutoId); + runner.enqueue("INSERT INTO PERSONS_AI (NAME, CODE) VALUES ('Mark', 84)".getBytes()); runner.run(); runner.assertAllFlowFilesTransferred(PutSQL.REL_SUCCESS, 1); @@ -130,7 +130,7 @@ public class TestPutSQL { try (final Connection conn = service.getConnection()) { try (final Statement stmt = conn.createStatement()) { - final ResultSet rs = stmt.executeQuery("SELECT * FROM PERSONS"); + final ResultSet rs = stmt.executeQuery("SELECT * FROM PERSONS_AI"); assertTrue(rs.next()); assertEquals(1, rs.getInt(1)); assertEquals("Mark", rs.getString(2)); @@ -144,24 +144,14 @@ public class TestPutSQL { @Test public void testFailInMiddleWithBadStatement() throws InitializationException, ProcessException, SQLException, IOException { final TestRunner runner = TestRunners.newTestRunner(PutSQL.class); - final File tempDir = folder.getRoot(); - final File dbDir = new File(tempDir, "db"); - final DBCPService service = new MockDBCPService(dbDir.getAbsolutePath()); runner.addControllerService("dbcp", service); runner.enableControllerService(service); - - try (final Connection conn = service.getConnection()) { - try (final Statement stmt = conn.createStatement()) { - stmt.executeUpdate(createPersonsAutoId); - } - } - runner.setProperty(PutSQL.OBTAIN_GENERATED_KEYS, "false"); runner.setProperty(PutSQL.CONNECTION_POOL, "dbcp"); - runner.enqueue("INSERT INTO PERSONS (NAME, CODE) VALUES ('Mark', 84)".getBytes()); - runner.enqueue("INSERT INTO PERSONS".getBytes()); // intentionally wrong syntax - runner.enqueue("INSERT INTO PERSONS (NAME, CODE) VALUES ('Tom', 3)".getBytes()); - runner.enqueue("INSERT INTO PERSONS (NAME, CODE) VALUES ('Harry', 44)".getBytes()); + runner.enqueue("INSERT INTO PERSONS_AI (NAME, CODE) VALUES ('Mark', 84)".getBytes()); + runner.enqueue("INSERT INTO PERSONS_AI".getBytes()); // intentionally wrong syntax + runner.enqueue("INSERT INTO PERSONS_AI (NAME, CODE) VALUES ('Tom', 3)".getBytes()); + runner.enqueue("INSERT INTO PERSONS_AI (NAME, CODE) VALUES ('Harry', 44)".getBytes()); runner.run(); runner.assertTransferCount(PutSQL.REL_FAILURE, 1); @@ -172,18 +162,8 @@ public class TestPutSQL { @Test public void testFailInMiddleWithBadParameterType() throws InitializationException, ProcessException, SQLException, IOException { final TestRunner runner = TestRunners.newTestRunner(PutSQL.class); - final File tempDir = folder.getRoot(); - final File dbDir = new File(tempDir, "db"); - final DBCPService service = new MockDBCPService(dbDir.getAbsolutePath()); runner.addControllerService("dbcp", service); runner.enableControllerService(service); - - try (final Connection conn = service.getConnection()) { - try (final Statement stmt = conn.createStatement()) { - stmt.executeUpdate(createPersonsAutoId); - } - } - runner.setProperty(PutSQL.OBTAIN_GENERATED_KEYS, "false"); runner.setProperty(PutSQL.CONNECTION_POOL, "dbcp"); @@ -195,7 +175,7 @@ public class TestPutSQL { badAttributes.put("sql.args.1.type", String.valueOf(Types.VARCHAR)); badAttributes.put("sql.args.1.value", "hello"); - final byte[] data = "INSERT INTO PERSONS (NAME, CODE) VALUES ('Mark', ?)".getBytes(); + final byte[] data = "INSERT INTO PERSONS_AI (NAME, CODE) VALUES ('Mark', ?)".getBytes(); runner.enqueue(data, goodAttributes); runner.enqueue(data, badAttributes); runner.enqueue(data, goodAttributes); @@ -210,21 +190,13 @@ public class TestPutSQL { @Test public void testFailInMiddleWithBadParameterValue() throws InitializationException, ProcessException, SQLException, IOException { final TestRunner runner = TestRunners.newTestRunner(PutSQL.class); - final File tempDir = folder.getRoot(); - final File dbDir = new File(tempDir, "db"); - final DBCPService service = new MockDBCPService(dbDir.getAbsolutePath()); runner.addControllerService("dbcp", service); runner.enableControllerService(service); - - try (final Connection conn = service.getConnection()) { - try (final Statement stmt = conn.createStatement()) { - stmt.executeUpdate(createPersonsAutoId); - } - } - runner.setProperty(PutSQL.OBTAIN_GENERATED_KEYS, "false"); runner.setProperty(PutSQL.CONNECTION_POOL, "dbcp"); + recreateTable("PERSONS_AI",createPersonsAutoId); + final Map<String, String> goodAttributes = new HashMap<>(); goodAttributes.put("sql.args.1.type", String.valueOf(Types.INTEGER)); goodAttributes.put("sql.args.1.value", "84"); @@ -233,7 +205,7 @@ public class TestPutSQL { badAttributes.put("sql.args.1.type", String.valueOf(Types.INTEGER)); badAttributes.put("sql.args.1.value", "9999"); - final byte[] data = "INSERT INTO PERSONS (NAME, CODE) VALUES ('Mark', ?)".getBytes(); + final byte[] data = "INSERT INTO PERSONS_AI (NAME, CODE) VALUES ('Mark', ?)".getBytes(); runner.enqueue(data, goodAttributes); runner.enqueue(data, badAttributes); runner.enqueue(data, goodAttributes); @@ -246,7 +218,7 @@ public class TestPutSQL { try (final Connection conn = service.getConnection()) { try (final Statement stmt = conn.createStatement()) { - final ResultSet rs = stmt.executeQuery("SELECT * FROM PERSONS"); + final ResultSet rs = stmt.executeQuery("SELECT * FROM PERSONS_AI"); assertTrue(rs.next()); assertEquals(1, rs.getInt(1)); assertEquals("Mark", rs.getString(2)); @@ -260,30 +232,26 @@ public class TestPutSQL { @Test public void testUsingSqlDataTypesWithNegativeValues() throws InitializationException, ProcessException, SQLException, IOException { final TestRunner runner = TestRunners.newTestRunner(PutSQL.class); - final File tempDir = folder.getRoot(); - final File dbDir = new File(tempDir, "db"); - final DBCPService service = new MockDBCPService(dbDir.getAbsolutePath()); - runner.addControllerService("dbcp", service); - runner.enableControllerService(service); - try (final Connection conn = service.getConnection()) { try (final Statement stmt = conn.createStatement()) { - stmt.executeUpdate("CREATE TABLE PERSONS (id integer primary key, name varchar(100), code bigint)"); + stmt.executeUpdate("CREATE TABLE PERSONS2 (id integer primary key, name varchar(100), code bigint)"); } } + runner.addControllerService("dbcp", service); + runner.enableControllerService(service); runner.setProperty(PutSQL.CONNECTION_POOL, "dbcp"); final Map<String, String> attributes = new HashMap<>(); attributes.put("sql.args.1.type", "-5"); attributes.put("sql.args.1.value", "84"); - runner.enqueue("INSERT INTO PERSONS (ID, NAME, CODE) VALUES (1, 'Mark', ?)".getBytes(), attributes); + runner.enqueue("INSERT INTO PERSONS2 (ID, NAME, CODE) VALUES (1, 'Mark', ?)".getBytes(), attributes); runner.run(); runner.assertAllFlowFilesTransferred(PutSQL.REL_SUCCESS, 1); try (final Connection conn = service.getConnection()) { try (final Statement stmt = conn.createStatement()) { - final ResultSet rs = stmt.executeQuery("SELECT * FROM PERSONS"); + final ResultSet rs = stmt.executeQuery("SELECT * FROM PERSONS2"); assertTrue(rs.next()); assertEquals(1, rs.getInt(1)); assertEquals("Mark", rs.getString(2)); @@ -296,19 +264,12 @@ public class TestPutSQL { @Test public void testStatementsWithPreparedParameters() throws InitializationException, ProcessException, SQLException, IOException { final TestRunner runner = TestRunners.newTestRunner(PutSQL.class); - final File tempDir = folder.getRoot(); - final File dbDir = new File(tempDir, "db"); - final DBCPService service = new MockDBCPService(dbDir.getAbsolutePath()); runner.addControllerService("dbcp", service); runner.enableControllerService(service); + runner.setProperty(PutSQL.CONNECTION_POOL, "dbcp"); - try (final Connection conn = service.getConnection()) { - try (final Statement stmt = conn.createStatement()) { - stmt.executeUpdate(createPersons); - } - } + recreateTable("PERSONS", createPersons); - runner.setProperty(PutSQL.CONNECTION_POOL, "dbcp"); final Map<String, String> attributes = new HashMap<>(); attributes.put("sql.args.1.type", String.valueOf(Types.INTEGER)); attributes.put("sql.args.1.value", "1"); @@ -364,18 +325,8 @@ public class TestPutSQL { @Test public void testMultipleStatementsWithinFlowFile() throws InitializationException, ProcessException, SQLException, IOException { final TestRunner runner = TestRunners.newTestRunner(PutSQL.class); - final File tempDir = folder.getRoot(); - final File dbDir = new File(tempDir, "db"); - final DBCPService service = new MockDBCPService(dbDir.getAbsolutePath()); runner.addControllerService("dbcp", service); runner.enableControllerService(service); - - try (final Connection conn = service.getConnection()) { - try (final Statement stmt = conn.createStatement()) { - stmt.executeUpdate(createPersons); - } - } - runner.setProperty(PutSQL.CONNECTION_POOL, "dbcp"); final String sql = "INSERT INTO PERSONS (ID, NAME, CODE) VALUES (?, ?, ?); " + @@ -411,18 +362,8 @@ public class TestPutSQL { @Test public void testWithNullParameter() throws InitializationException, ProcessException, SQLException, IOException { final TestRunner runner = TestRunners.newTestRunner(PutSQL.class); - final File tempDir = folder.getRoot(); - final File dbDir = new File(tempDir, "db"); - final DBCPService service = new MockDBCPService(dbDir.getAbsolutePath()); runner.addControllerService("dbcp", service); runner.enableControllerService(service); - - try (final Connection conn = service.getConnection()) { - try (final Statement stmt = conn.createStatement()) { - stmt.executeUpdate(createPersons); - } - } - runner.setProperty(PutSQL.CONNECTION_POOL, "dbcp"); final Map<String, String> attributes = new HashMap<>(); attributes.put("sql.args.1.type", String.valueOf(Types.INTEGER)); @@ -453,20 +394,12 @@ public class TestPutSQL { @Test public void testInvalidStatement() throws InitializationException, ProcessException, SQLException, IOException { final TestRunner runner = TestRunners.newTestRunner(PutSQL.class); - final File tempDir = folder.getRoot(); - final File dbDir = new File(tempDir, "db"); - final DBCPService service = new MockDBCPService(dbDir.getAbsolutePath()); runner.addControllerService("dbcp", service); runner.enableControllerService(service); - - try (final Connection conn = service.getConnection()) { - try (final Statement stmt = conn.createStatement()) { - stmt.executeUpdate(createPersons); - } - } - runner.setProperty(PutSQL.CONNECTION_POOL, "dbcp"); + recreateTable("PERSONS", createPersons); + final String sql = "INSERT INTO PERSONS (ID, NAME, CODE) VALUES (?, ?, ?); " + "UPDATE SOME_RANDOM_TABLE NAME='George' WHERE ID=?; "; final Map<String, String> attributes = new HashMap<>(); @@ -532,21 +465,13 @@ public class TestPutSQL { @Test public void testMultipleFlowFilesSuccessfulInTransaction() throws InitializationException, ProcessException, SQLException, IOException { final TestRunner runner = TestRunners.newTestRunner(PutSQL.class); - final File tempDir = folder.getRoot(); - final File dbDir = new File(tempDir, "db"); - final DBCPService service = new MockDBCPService(dbDir.getAbsolutePath()); runner.addControllerService("dbcp", service); runner.enableControllerService(service); - - try (final Connection conn = service.getConnection()) { - try (final Statement stmt = conn.createStatement()) { - stmt.executeUpdate(createPersons); - } - } - runner.setProperty(PutSQL.CONNECTION_POOL, "dbcp"); runner.setProperty(PutSQL.BATCH_SIZE, "1"); + recreateTable("PERSONS", createPersons); + final Map<String, String> attributes = new HashMap<>(); attributes.put("sql.args.1.type", String.valueOf(Types.INTEGER)); attributes.put("sql.args.1.value", "1"); @@ -599,9 +524,6 @@ public class TestPutSQL { @Test public void testTransactionTimeout() throws InitializationException, ProcessException, SQLException, IOException { final TestRunner runner = TestRunners.newTestRunner(PutSQL.class); - final File tempDir = folder.getRoot(); - final File dbDir = new File(tempDir, "db"); - final DBCPService service = new MockDBCPService(dbDir.getAbsolutePath()); runner.addControllerService("dbcp", service); runner.enableControllerService(service); @@ -697,4 +619,13 @@ public class TestPutSQL { } } } + + private void recreateTable(String tableName, String createSQL) throws ProcessException, SQLException { + try (final Connection conn = service.getConnection()) { + try (final Statement stmt = conn.createStatement()) { + stmt.executeUpdate("drop table " + tableName); + stmt.executeUpdate(createSQL); + } + } + } } http://git-wip-us.apache.org/repos/asf/nifi/blob/65c18b6e/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/util/TestJdbcCommon.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/util/TestJdbcCommon.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/util/TestJdbcCommon.java index da9e163..37933a1 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/util/TestJdbcCommon.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/util/TestJdbcCommon.java @@ -46,7 +46,7 @@ import org.apache.avro.generic.GenericRecord; import org.apache.avro.io.DatumReader; import org.junit.Assert; import org.junit.BeforeClass; -import org.junit.Rule; +import org.junit.ClassRule; import org.junit.Test; import org.junit.rules.TemporaryFolder; import org.mockito.Mockito; @@ -55,33 +55,35 @@ import org.mockito.stubbing.Answer; public class TestJdbcCommon { - @Rule - public TemporaryFolder folder = new TemporaryFolder(); + static final String createTable = "create table restaurants(id integer, name varchar(20), city varchar(50))"; + static final String dropTable = "drop table restaurants"; - @BeforeClass - public static void setup() { - System.setProperty("derby.stream.error.file", "target/derby.log"); - } + @ClassRule + public static TemporaryFolder folder = new TemporaryFolder(); - String createTable = "create table restaurants(id integer, name varchar(20), city varchar(50))"; - String dropTable = "drop table restaurants"; - - @Test - public void testCreateSchema() throws ClassNotFoundException, SQLException { + /** + * Setting up Connection is expensive operation. + * So let's do this only once and reuse Connection in each test. + */ + static protected Connection con; + @BeforeClass + public static void setup() throws ClassNotFoundException, SQLException { + System.setProperty("derby.stream.error.file", "target/derby.log"); + Class.forName("org.apache.derby.jdbc.EmbeddedDriver"); // remove previous test database, if any folder.delete(); - final Connection con = createConnection(folder.getRoot().getAbsolutePath()); - final Statement st = con.createStatement(); - - try { - st.executeUpdate(dropTable); - } catch (final Exception e) { - // table may not exist, this is not serious problem. + String location = folder.getRoot().getAbsolutePath(); + con = DriverManager.getConnection("jdbc:derby:" + location + ";create=true"); + try (final Statement stmt = con.createStatement()) { + stmt.executeUpdate(createTable); } + } - st.executeUpdate(createTable); + @Test + public void testCreateSchema() throws ClassNotFoundException, SQLException { + final Statement st = con.createStatement(); st.executeUpdate("insert into restaurants values (1, 'Irifunes', 'San Mateo')"); st.executeUpdate("insert into restaurants values (2, 'Estradas', 'Daly City')"); st.executeUpdate("insert into restaurants values (3, 'Prime Rib House', 'San Francisco')"); @@ -100,7 +102,7 @@ public class TestJdbcCommon { assertNotNull(schema.getField("CITY")); st.close(); - con.close(); +// con.close(); } @Test @@ -120,7 +122,6 @@ public class TestJdbcCommon { // may have different table names assertEquals("NiFi_ExecuteSQL_Record", schema.getName()); assertNull(schema.getField("ID")); - } @Test @@ -144,20 +145,7 @@ public class TestJdbcCommon { @Test public void testConvertToBytes() throws ClassNotFoundException, SQLException, IOException { - // remove previous test database, if any - folder.delete(); - - final Connection con = createConnection(folder.getRoot().getAbsolutePath()); final Statement st = con.createStatement(); - - try { - st.executeUpdate(dropTable); - } catch (final Exception e) { - // table may not exist, this is not serious problem. - } - - st.executeUpdate(createTable); - st.executeUpdate("insert into restaurants values (1, 'Irifunes', 'San Mateo')"); st.executeUpdate("insert into restaurants values (2, 'Estradas', 'Daly City')"); st.executeUpdate("insert into restaurants values (3, 'Prime Rib House', 'San Francisco')"); @@ -342,9 +330,4 @@ public class TestJdbcCommon { assertNotNull(clazz); } - private Connection createConnection(String location) throws ClassNotFoundException, SQLException { - Class.forName("org.apache.derby.jdbc.EmbeddedDriver"); - return DriverManager.getConnection("jdbc:derby:" + location + ";create=true"); - } - }
