Updated Branches: refs/heads/sqoop2 27271af13 -> 3fd8bdffc
SQOOP-946: Sqoop2: Integration: Enhance the POC test case (Jarek Jarcec Cecho via Kate Ting) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/3fd8bdff Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/3fd8bdff Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/3fd8bdff Branch: refs/heads/sqoop2 Commit: 3fd8bdffc9510cc9c5bbd159e9036891171496a8 Parents: 27271af Author: Kate Ting <[email protected]> Authored: Tue Mar 26 04:15:05 2013 -0400 Committer: Kate Ting <[email protected]> Committed: Tue Mar 26 04:15:05 2013 -0400 ---------------------------------------------------------------------- .../apache/sqoop/integration/TomcatTestCase.java | 31 ++++- .../integration/connector/ConnectorTestCase.java | 92 +++++++++++++- .../connector/jdbc/generic/TableImportTest.java | 99 ++++++++------- 3 files changed, 164 insertions(+), 58 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/3fd8bdff/test/src/test/java/org/apache/sqoop/integration/TomcatTestCase.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/sqoop/integration/TomcatTestCase.java b/test/src/test/java/org/apache/sqoop/integration/TomcatTestCase.java index 5631144..6eb3184 100644 --- a/test/src/test/java/org/apache/sqoop/integration/TomcatTestCase.java +++ b/test/src/test/java/org/apache/sqoop/integration/TomcatTestCase.java @@ -23,6 +23,8 @@ import org.apache.sqoop.client.SqoopClient; import org.apache.sqoop.test.minicluster.TomcatSqoopMiniCluster; import org.junit.After; import org.junit.Before; +import org.junit.Rule; +import org.junit.rules.TestName; import java.io.BufferedReader; import java.io.File; @@ -45,17 +47,29 @@ abstract public class TomcatTestCase { private static final Logger LOG = Logger.getLogger(TomcatTestCase.class); + @Rule public TestName name = new TestName(); + /** - * Temporary path that will be used for this test. + * Temporary base path that will be used for tests. * - * By default we will take look for sqoop.integration.tmpdir property that is + * By default we will take a look for sqoop.integration.tmpdir property that is * filled up by maven. If the test is not started from maven (IDE) we will * pick up configured java.io.tmpdir value. The last results is /tmp/ directory * in case that no property is set. */ - private final String TMP_PATH = - System.getProperty("sqoop.integration.tmpdir", System.getProperty("java.io.tmpdir", "/tmp")) - + "/sqoop-cargo-tests/" + getClass().getName() + "/"; + private static final String TMP_PATH_BASE = + System.getProperty("sqoop.integration.tmpdir", System.getProperty("java.io.tmpdir", "/tmp")) + "/sqoop-cargo-tests/"; + + /** + * Temporary directory that will be used by the test. + * + * We will take TMP_PATH_BASE and append two subdirectories. First will be named + * after fully qualified class name of current test class, second directory will + * be named after current test method name. For example: + * + * TMP_PATH_BASE/org.apache.sqoop.TestClass/testMethod/ + */ + private String tmpPath; /** * Tomcat based Sqoop mini cluster @@ -69,9 +83,14 @@ abstract public class TomcatTestCase { @Before public void startServer() throws Exception { + // Set up the temporary path + tmpPath = TMP_PATH_BASE + getClass().getName() + "/" + name.getMethodName() + "/"; + + // Set up and start server cluster = new TomcatSqoopMiniCluster(getTemporaryPath()); cluster.start(); + // Initialize Sqoop Client API client = new SqoopClient(getServerUrl()); } @@ -90,7 +109,7 @@ abstract public class TomcatTestCase { } public String getTemporaryPath() { - return TMP_PATH; + return tmpPath; } /** http://git-wip-us.apache.org/repos/asf/sqoop/blob/3fd8bdff/test/src/test/java/org/apache/sqoop/integration/connector/ConnectorTestCase.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/sqoop/integration/connector/ConnectorTestCase.java b/test/src/test/java/org/apache/sqoop/integration/connector/ConnectorTestCase.java index 14f7dc4..837a1aa 100644 --- a/test/src/test/java/org/apache/sqoop/integration/connector/ConnectorTestCase.java +++ b/test/src/test/java/org/apache/sqoop/integration/connector/ConnectorTestCase.java @@ -18,11 +18,21 @@ package org.apache.sqoop.integration.connector; import org.apache.log4j.Logger; +import org.apache.sqoop.framework.configuration.OutputFormat; +import org.apache.sqoop.framework.configuration.StorageType; import org.apache.sqoop.integration.TomcatTestCase; +import org.apache.sqoop.model.MConnection; +import org.apache.sqoop.model.MFormList; +import org.apache.sqoop.model.MJob; +import org.apache.sqoop.model.MPersistableEntity; import org.apache.sqoop.test.db.DatabaseProvider; import org.apache.sqoop.test.db.DatabaseProviderFactory; -import org.junit.After; -import org.junit.Before; +import org.apache.sqoop.validation.Status; +import org.junit.AfterClass; +import org.junit.BeforeClass; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotSame; /** * Base test case for connector testing. @@ -33,17 +43,18 @@ abstract public class ConnectorTestCase extends TomcatTestCase { private static final Logger LOG = Logger.getLogger(ConnectorTestCase.class); - protected DatabaseProvider provider; + protected static DatabaseProvider provider; - @Before - public void startProvider() throws Exception { + @BeforeClass + public static void startProvider() throws Exception { provider = DatabaseProviderFactory.getProvider(System.getProperties()); LOG.info("Starting database provider: " + provider.getClass().getName()); provider.start(); } - @After - public void stopProvider() { + @AfterClass + public static void stopProvider() { + LOG.info("Stopping database provider: " + provider.getClass().getName()); provider.stop(); } @@ -62,4 +73,71 @@ abstract public class ConnectorTestCase extends TomcatTestCase { protected void insertRow(Object ...values) { provider.insertRow(getTableName(), values); } + + /** + * Fill connection form based on currently active provider. + * + * @param connection MConnection object to fill + */ + protected void fillConnectionForm(MConnection connection) { + MFormList forms = connection.getConnectorPart(); + forms.getStringInput("connection.jdbcDriver").setValue(provider.getJdbcDriver()); + forms.getStringInput("connection.connectionString").setValue(provider.getConnectionUrl()); + forms.getStringInput("connection.username").setValue(provider.getConnectionUsername()); + forms.getStringInput("connection.password").setValue(provider.getConnectionPassword()); + } + + /** + * Fill output form with specific storage and output type. Mapreduce output directory + * will be set to default test value. + * + * @param job MJOb object to fill + * @param storage Storage type that should be set + * @param output Output type that should be set + */ + protected void fillOutputForm(MJob job, StorageType storage, OutputFormat output) { + MFormList forms = job.getFrameworkPart(); + forms.getEnumInput("output.storageType").setValue(storage); + forms.getEnumInput("output.outputFormat").setValue(output); + forms.getStringInput("output.outputDirectory").setValue(getMapreduceDirectory()); + } + + /** + * Create table cities. + */ + protected void createAndLoadTableCities() { + createTable("id", + "id", "int", + "country", "varchar(50)", + "city", "varchar(50)" + ); + insertRow(1, "USA", "San Francisco"); + insertRow(2, "USA", "Sunnyvale"); + insertRow(3, "Czech Republic", "Brno"); + insertRow(4, "USA", "Palo Alto"); + } + + /** + * Create connection. + * + * With asserts to make sure that it was created correctly. + * + * @param connection + */ + protected void createConnection(MConnection connection) { + assertEquals(Status.FINE, getClient().createConnection(connection)); + assertNotSame(MPersistableEntity.PERSISTANCE_ID_DEFAULT, connection.getPersistenceId()); + } + + /** + * Create job. + * + * With asserts to make sure that it was created correctly. + * + * @param job + */ + protected void createJob(MJob job) { + assertEquals(Status.FINE, getClient().createJob(job)); + assertNotSame(MPersistableEntity.PERSISTANCE_ID_DEFAULT, job.getPersistenceId()); + } } http://git-wip-us.apache.org/repos/asf/sqoop/blob/3fd8bdff/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/TableImportTest.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/TableImportTest.java b/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/TableImportTest.java index 05a799c..adcfbaf 100644 --- a/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/TableImportTest.java +++ b/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/TableImportTest.java @@ -24,78 +24,87 @@ import org.apache.sqoop.integration.connector.ConnectorTestCase; import org.apache.sqoop.model.MConnection; import org.apache.sqoop.model.MFormList; import org.apache.sqoop.model.MJob; -import org.apache.sqoop.model.MPersistableEntity; import org.apache.sqoop.model.MSubmission; -import org.apache.sqoop.validation.Status; import org.junit.Test; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotSame; import static org.junit.Assert.assertTrue; /** - * Proof of concept implementation of first "real" integration test. - * - * Will be improved when client API will be created. + * Import simple table with various configurations. */ public class TableImportTest extends ConnectorTestCase { private static final Logger LOG = Logger.getLogger(TableImportTest.class); - /** - * This test is proof of concept. - * - * It will be refactored once we will create reasonable client interface. - */ @Test - public void testBasicTableImport() throws Exception { - createTable("id", - "id", "int", - "txt", "varchar(50)" - ); - insertRow(1, "San Francisco"); - insertRow(2, "Sunnyvale"); - insertRow(3, "Brno"); - - // Connection creation and job submission will be refactored once - // the client API for embedding Sqoop client will be ready. + public void testBasicImport() throws Exception { + createAndLoadTableCities(); // Connection creation MConnection connection = getClient().newConnection(1L); + fillConnectionForm(connection); + createConnection(connection); + + // Job creation + MJob job = getClient().newJob(connection.getPersistenceId(), MJob.Type.IMPORT); - MFormList forms; // Connector values - forms = connection.getConnectorPart(); - forms.getStringInput("connection.jdbcDriver").setValue(provider.getJdbcDriver()); - forms.getStringInput("connection.connectionString").setValue(provider.getConnectionUrl()); - forms.getStringInput("connection.username").setValue(provider.getConnectionUsername()); - forms.getStringInput("connection.password").setValue(provider.getConnectionPassword()); + MFormList forms = job.getConnectorPart(); + forms.getStringInput("table.tableName").setValue(provider.escapeTableName(getTableName())); + forms.getStringInput("table.partitionColumn").setValue(provider.escapeColumnName("id")); // Framework values - // No need to set anything + fillOutputForm(job, StorageType.HDFS, OutputFormat.TEXT_FILE); + createJob(job); - assertEquals(Status.FINE, getClient().createConnection(connection)); - assertNotSame(MPersistableEntity.PERSISTANCE_ID_DEFAULT, connection.getPersistenceId()); + MSubmission submission = getClient().startSubmission(job.getPersistenceId()); + assertTrue(submission.getStatus().isRunning()); + + // Wait until the job finish - this active waiting will be removed once + // Sqoop client API will get blocking support. + do { + Thread.sleep(5000); + submission = getClient().getSubmissionStatus(job.getPersistenceId()); + } while(submission.getStatus().isRunning()); + + // Assert correct output + assertMapreduceOutput( + "1,'USA','San Francisco'", + "2,'USA','Sunnyvale'", + "3,'Czech Republic','Brno'", + "4,'USA','Palo Alto'" + ); + + // Clean up testing table + dropTable(); + } + + @Test + public void testColumns() throws Exception { + createAndLoadTableCities(); + + // Connection creation + MConnection connection = getClient().newConnection(1L); + fillConnectionForm(connection); + + createConnection(connection); // Job creation MJob job = getClient().newJob(connection.getPersistenceId(), MJob.Type.IMPORT); // Connector values - forms = job.getConnectorPart(); + MFormList forms = job.getConnectorPart(); forms.getStringInput("table.tableName").setValue(provider.escapeTableName(getTableName())); forms.getStringInput("table.partitionColumn").setValue(provider.escapeColumnName("id")); + forms.getStringInput("table.columns").setValue(provider.escapeColumnName("id") + "," + provider.escapeColumnName("country")); // Framework values - forms = job.getFrameworkPart(); - forms.getEnumInput("output.storageType").setValue(StorageType.HDFS); - forms.getEnumInput("output.outputFormat").setValue(OutputFormat.TEXT_FILE); - forms.getStringInput("output.outputDirectory").setValue(getMapreduceDirectory()); - - assertEquals(Status.FINE, getClient().createJob(job)); - assertNotSame(MPersistableEntity.PERSISTANCE_ID_DEFAULT, job.getPersistenceId()); + fillOutputForm(job, StorageType.HDFS, OutputFormat.TEXT_FILE); + createJob(job); MSubmission submission = getClient().startSubmission(job.getPersistenceId()); assertTrue(submission.getStatus().isRunning()); - // Wait until the job finish + // Wait until the job finish - this active waiting will be removed once + // Sqoop client API will get blocking support. do { Thread.sleep(5000); submission = getClient().getSubmissionStatus(job.getPersistenceId()); @@ -103,13 +112,13 @@ public class TableImportTest extends ConnectorTestCase { // Assert correct output assertMapreduceOutput( - "1,'San Francisco'", - "2,'Sunnyvale'", - "3,'Brno'" + "1,'USA'", + "2,'USA'", + "3,'Czech Republic'", + "4,'USA'" ); // Clean up testing table dropTable(); } - }
