Repository: sqoop Updated Branches: refs/heads/sqoop2 d95e7537a -> 5b6ac608c
SQOOP-1572: Sqoop2: Duplicate Column Name in Multiple Tables Import Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/5b6ac608 Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/5b6ac608 Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/5b6ac608 Branch: refs/heads/sqoop2 Commit: 5b6ac608c33a3701fd9452cd45dc727d1d64e60b Parents: d95e753 Author: Gwen Shapira <[email protected]> Authored: Fri Jan 23 10:03:11 2015 -0800 Committer: Gwen Shapira <[email protected]> Committed: Fri Jan 23 10:03:11 2015 -0800 ---------------------------------------------------------------------- .../jdbc/GenericJdbcFromInitializer.java | 4 +- docs/src/site/sphinx/Connectors.rst | 3 +- .../jdbc/generic/FromRDBMSToHDFSTest.java | 98 ++++++++++++++++++++ 3 files changed, 102 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/5b6ac608/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcFromInitializer.java ---------------------------------------------------------------------- diff --git a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcFromInitializer.java b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcFromInitializer.java index ff42949..425b2cc 100644 --- a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcFromInitializer.java +++ b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcFromInitializer.java @@ -85,9 +85,9 @@ public class GenericJdbcFromInitializer extends Initializer<LinkConfiguration, F rsmt = rs.getMetaData(); for (int i = 1 ; i <= rsmt.getColumnCount(); i++) { - String columnName = rsmt.getColumnName(i); + String columnName = rsmt.getColumnLabel(i); if (StringUtils.isEmpty(columnName)) { - columnName = rsmt.getColumnLabel(i); + columnName = rsmt.getColumnName(i); if (StringUtils.isEmpty(columnName)) { columnName = "Column " + i; } http://git-wip-us.apache.org/repos/asf/sqoop/blob/5b6ac608/docs/src/site/sphinx/Connectors.rst ---------------------------------------------------------------------- diff --git a/docs/src/site/sphinx/Connectors.rst b/docs/src/site/sphinx/Connectors.rst index bcc5b43..f67e187 100644 --- a/docs/src/site/sphinx/Connectors.rst +++ b/docs/src/site/sphinx/Connectors.rst @@ -74,7 +74,7 @@ Inputs associated with the Job configuration for the FROM direction include: | | | *Optional*. See note below. | | +-----------------------------+---------+-------------------------------------------------------------------------+---------------------------------------------+ | Table SQL statement | String | The SQL statement used to perform a **free form query**. | ``SELECT COUNT(*) FROM test ${CONDITIONS}`` | -| | | *Optional*. See note below. | | +| | | *Optional*. See notes below. | | +-----------------------------+---------+-------------------------------------------------------------------------+---------------------------------------------+ | Table column names | String | Columns to extract from the JDBC data source. | col1,col2 | | | | *Optional* Comma separated list of columns. | | @@ -94,6 +94,7 @@ Inputs associated with the Job configuration for the FROM direction include: 1. *Table name* and *Table SQL statement* are mutually exclusive. If *Table name* is provided, the *Table SQL statement* should not be provided. If *Table SQL statement* is provided then *Table name* should not be provided. 2. *Table column names* should be provided only if *Table name* is provided. +3. If there are columns with similar names, column aliases are required. For example: ``SELECT table1.id as "i", table2.id as "j" FROM table1 INNER JOIN table2 ON table1.id = table2.id``. **TO Job Configuration** ++++++++++++++++++++++++ http://git-wip-us.apache.org/repos/asf/sqoop/blob/5b6ac608/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/FromRDBMSToHDFSTest.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/FromRDBMSToHDFSTest.java b/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/FromRDBMSToHDFSTest.java index 85b9d2d..aa9f212 100644 --- a/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/FromRDBMSToHDFSTest.java +++ b/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/FromRDBMSToHDFSTest.java @@ -122,4 +122,102 @@ public class FromRDBMSToHDFSTest extends ConnectorTestCase { // Clean up testing table dropTable(); } + + @Test + public void testSql() throws Exception { + createAndLoadTableCities(); + + // RDBMS link + MLink rdbmsLink = getClient().createLink("generic-jdbc-connector"); + fillRdbmsLinkConfig(rdbmsLink); + saveLink(rdbmsLink); + + // HDFS link + MLink hdfsLink = getClient().createLink("hdfs-connector"); + saveLink(hdfsLink); + + // Job creation + MJob job = getClient().createJob(rdbmsLink.getPersistenceId(), hdfsLink.getPersistenceId()); + + // Connector values + MConfigList configs = job.getJobConfig(Direction.FROM); + configs.getStringInput("fromJobConfig.sql").setValue("SELECT " + provider.escapeColumnName("id") + + " FROM " + provider.escapeTableName(getTableName()) + " WHERE ${CONDITIONS}"); + configs.getStringInput("fromJobConfig.partitionColumn").setValue(provider.escapeColumnName("id")); + fillHdfsToConfig(job, ToFormat.TEXT_FILE); + saveJob(job); + + MSubmission submission = getClient().startJob(job.getPersistenceId()); + assertTrue(submission.getStatus().isRunning()); + + // Wait until the job finish - this active waiting will be removed once + // Sqoop client API will get blocking support. + do { + Thread.sleep(5000); + submission = getClient().getJobStatus(job.getPersistenceId()); + } while(submission.getStatus().isRunning()); + + // Assert correct output + assertTo( + "1", + "2", + "3", + "4" + ); + + // Clean up testing table + dropTable(); + } + + @Test + public void testDuplicateColumns() throws Exception { + createAndLoadTableCities(); + + // RDBMS link + MLink rdbmsLink = getClient().createLink("generic-jdbc-connector"); + fillRdbmsLinkConfig(rdbmsLink); + saveLink(rdbmsLink); + + // HDFS link + MLink hdfsLink = getClient().createLink("hdfs-connector"); + saveLink(hdfsLink); + + // Job creation + MJob job = getClient().createJob(rdbmsLink.getPersistenceId(), hdfsLink.getPersistenceId()); + + // Connector values + String partitionColumn = provider.escapeTableName(getTableName()) + "." + provider.escapeColumnName("id"); + MConfigList configs = job.getJobConfig(Direction.FROM); + configs.getStringInput("fromJobConfig.sql").setValue( + "SELECT " + provider.escapeColumnName("id") + " as " + provider.escapeColumnName("i") + ", " + + provider.escapeColumnName("id") + " as " + provider.escapeColumnName("j") + + " FROM " + provider.escapeTableName(getTableName()) + " WHERE ${CONDITIONS}"); + configs.getStringInput("fromJobConfig.partitionColumn").setValue(partitionColumn); + configs.getStringInput("fromJobConfig.boundaryQuery").setValue( + "SELECT MIN(" + partitionColumn + "), MAX(" + partitionColumn + ") FROM " + + provider.escapeTableName(getTableName())); + fillHdfsToConfig(job, ToFormat.TEXT_FILE); + saveJob(job); + + MSubmission submission = getClient().startJob(job.getPersistenceId()); + assertTrue(submission.getStatus().isRunning()); + + // Wait until the job finish - this active waiting will be removed once + // Sqoop client API will get blocking support. + do { + Thread.sleep(5000); + submission = getClient().getJobStatus(job.getPersistenceId()); + } while(submission.getStatus().isRunning()); + + // Assert correct output + assertTo( + "1,1", + "2,2", + "3,3", + "4,4" + ); + + // Clean up testing table + dropTable(); + } }
