SQOOP-814: Start using configuration objects in Generic JDBC Connector (Jarek Jarcec Cecho via Cheolsoo Park)
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/f89c8e59 Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/f89c8e59 Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/f89c8e59 Branch: refs/heads/sqoop2 Commit: f89c8e59f2826f899b7a85815be7940d705d5499 Parents: acfc0ae Author: Cheolsoo Park <[email protected]> Authored: Tue Jan 22 15:41:26 2013 -0800 Committer: Cheolsoo Park <[email protected]> Committed: Tue Jan 22 15:41:26 2013 -0800 ---------------------------------------------------------------------- .../jdbc/GenericJdbcConnectorConstants.java | 9 --- .../jdbc/GenericJdbcExportInitializer.java | 19 ------ .../connector/jdbc/GenericJdbcExportLoader.java | 18 ++---- .../connector/jdbc/GenericJdbcImportExtractor.java | 27 +++----- .../jdbc/GenericJdbcImportInitializer.java | 18 ----- .../jdbc/GenericJdbcImportPartitioner.java | 12 +--- .../sqoop/connector/jdbc/TestExportLoader.java | 20 +++--- .../sqoop/connector/jdbc/TestImportExtractor.java | 49 +++++++-------- 8 files changed, 54 insertions(+), 118 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/f89c8e59/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcConnectorConstants.java ---------------------------------------------------------------------- diff --git a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcConnectorConstants.java b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcConnectorConstants.java index 09056d3..23f19b5 100644 --- a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcConnectorConstants.java +++ b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcConnectorConstants.java @@ -33,15 +33,6 @@ public final class GenericJdbcConnectorConstants { public static final String PREFIX_CONNECTOR_JDBC_CONFIG = "org.apache.sqoop.connector.jdbc."; - public static final String CONNECTOR_JDBC_DRIVER = - PREFIX_CONNECTOR_JDBC_CONFIG + "driver"; - public static final String CONNECTOR_JDBC_URL = - PREFIX_CONNECTOR_JDBC_CONFIG + "url"; - public static final String CONNECTOR_JDBC_USERNAME = - PREFIX_CONNECTOR_JDBC_CONFIG + "username"; - public static final String CONNECTOR_JDBC_PASSWORD = - PREFIX_CONNECTOR_JDBC_CONFIG + "password"; - public static final String CONNECTOR_JDBC_PARTITION_COLUMNNAME = PREFIX_CONNECTOR_JDBC_CONFIG + "partition.columnname"; public static final String CONNECTOR_JDBC_PARTITION_COLUMNTYPE = http://git-wip-us.apache.org/repos/asf/sqoop/blob/f89c8e59/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcExportInitializer.java ---------------------------------------------------------------------- diff --git a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcExportInitializer.java b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcExportInitializer.java index 2b0ed00..531dd2e 100644 --- a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcExportInitializer.java +++ b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcExportInitializer.java @@ -26,7 +26,6 @@ import org.apache.sqoop.common.MutableContext; import org.apache.sqoop.common.SqoopException; import org.apache.sqoop.connector.jdbc.configuration.ConnectionConfiguration; import org.apache.sqoop.connector.jdbc.configuration.ExportJobConfiguration; -import org.apache.sqoop.job.Constants; import org.apache.sqoop.job.etl.Initializer; import org.apache.sqoop.utils.ClassUtils; @@ -64,30 +63,12 @@ public class GenericJdbcExportInitializer extends Initializer<ConnectionConfigur GenericJdbcConnectorError.GENERIC_JDBC_CONNECTOR_0012, "JDBC Driver"); } - context.setString( - GenericJdbcConnectorConstants.CONNECTOR_JDBC_DRIVER, - driver); if (url == null) { throw new SqoopException( GenericJdbcConnectorError.GENERIC_JDBC_CONNECTOR_0012, "Connection string"); } - context.setString( - GenericJdbcConnectorConstants.CONNECTOR_JDBC_URL, - url); - - if (username != null) { - context.setString( - GenericJdbcConnectorConstants.CONNECTOR_JDBC_USERNAME, - username); - } - - if (password != null) { - context.setString( - GenericJdbcConnectorConstants.CONNECTOR_JDBC_PASSWORD, - password); - } executor = new GenericJdbcExecutor(driver, url, username, password); } http://git-wip-us.apache.org/repos/asf/sqoop/blob/f89c8e59/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcExportLoader.java ---------------------------------------------------------------------- diff --git a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcExportLoader.java b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcExportLoader.java index ec047b4..31dbcea 100644 --- a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcExportLoader.java +++ b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcExportLoader.java @@ -32,19 +32,13 @@ public class GenericJdbcExportLoader extends Loader<ConnectionConfiguration, Exp @Override public void load(ImmutableContext context, ConnectionConfiguration connection, ExportJobConfiguration job, DataReader reader) throws Exception{ - String driver = context.getString( - GenericJdbcConnectorConstants.CONNECTOR_JDBC_DRIVER); - String url = context.getString( - GenericJdbcConnectorConstants.CONNECTOR_JDBC_URL); - String username = context.getString( - GenericJdbcConnectorConstants.CONNECTOR_JDBC_USERNAME); - String password = context.getString( - GenericJdbcConnectorConstants.CONNECTOR_JDBC_PASSWORD); - GenericJdbcExecutor executor = new GenericJdbcExecutor( - driver, url, username, password); + String driver = connection.connection.jdbcDriver; + String url = connection.connection.connectionString; + String username = connection.connection.username; + String password = connection.connection.password; + GenericJdbcExecutor executor = new GenericJdbcExecutor(driver, url, username, password); - String sql = context.getString( - GenericJdbcConnectorConstants.CONNECTOR_JDBC_DATA_SQL); + String sql = context.getString(GenericJdbcConnectorConstants.CONNECTOR_JDBC_DATA_SQL); executor.beginBatch(sql); try { int numberOfRows = 0; http://git-wip-us.apache.org/repos/asf/sqoop/blob/f89c8e59/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcImportExtractor.java ---------------------------------------------------------------------- diff --git a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcImportExtractor.java b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcImportExtractor.java index c3ecdda..f4389a3 100644 --- a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcImportExtractor.java +++ b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcImportExtractor.java @@ -37,24 +37,17 @@ public class GenericJdbcImportExtractor extends Extractor<ConnectionConfiguratio private long rowsRead = 0; @Override public void run(ImmutableContext context, ConnectionConfiguration connection, ImportJobConfiguration job, Partition partition, DataWriter writer) { - String driver = context.getString( - GenericJdbcConnectorConstants.CONNECTOR_JDBC_DRIVER); - String url = context.getString( - GenericJdbcConnectorConstants.CONNECTOR_JDBC_URL); - String username = context.getString( - GenericJdbcConnectorConstants.CONNECTOR_JDBC_USERNAME); - String password = context.getString( - GenericJdbcConnectorConstants.CONNECTOR_JDBC_PASSWORD); - GenericJdbcExecutor executor = new GenericJdbcExecutor( - driver, url, username, password); + String driver = connection.connection.jdbcDriver; + String url = connection.connection.connectionString; + String username = connection.connection.username; + String password = connection.connection.password; + GenericJdbcExecutor executor = new GenericJdbcExecutor(driver, url, username, password); + + String query = context.getString(GenericJdbcConnectorConstants.CONNECTOR_JDBC_DATA_SQL); + String conditions = ((GenericJdbcImportPartition)partition).getConditions(); + query = query.replace(GenericJdbcConnectorConstants.SQL_CONDITIONS_TOKEN, conditions); + LOG.info("Using query: " + query); - String query = context.getString( - GenericJdbcConnectorConstants.CONNECTOR_JDBC_DATA_SQL); - String conditions = - ((GenericJdbcImportPartition)partition).getConditions(); - query = query.replace( - GenericJdbcConnectorConstants.SQL_CONDITIONS_TOKEN, conditions); - LOG.debug("Using query: " + query); rowsRead = 0; ResultSet resultSet = executor.executeQuery(query); http://git-wip-us.apache.org/repos/asf/sqoop/blob/f89c8e59/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcImportInitializer.java ---------------------------------------------------------------------- diff --git a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcImportInitializer.java b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcImportInitializer.java index 781c7f2..c6336fa 100644 --- a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcImportInitializer.java +++ b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcImportInitializer.java @@ -73,30 +73,12 @@ public class GenericJdbcImportInitializer extends Initializer<ConnectionConfigur GenericJdbcConnectorError.GENERIC_JDBC_CONNECTOR_0012, "JDBC Driver"); } - context.setString( - GenericJdbcConnectorConstants.CONNECTOR_JDBC_DRIVER, - driver); if (url == null) { throw new SqoopException( GenericJdbcConnectorError.GENERIC_JDBC_CONNECTOR_0012, "Connection string"); } - context.setString( - GenericJdbcConnectorConstants.CONNECTOR_JDBC_URL, - url); - - if (username != null) { - context.setString( - GenericJdbcConnectorConstants.CONNECTOR_JDBC_USERNAME, - username); - } - - if (password != null) { - context.setString( - GenericJdbcConnectorConstants.CONNECTOR_JDBC_PASSWORD, - password); - } executor = new GenericJdbcExecutor(driver, url, username, password); } http://git-wip-us.apache.org/repos/asf/sqoop/blob/f89c8e59/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcImportPartitioner.java ---------------------------------------------------------------------- diff --git a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcImportPartitioner.java b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcImportPartitioner.java index 2313e65..b5dbc61 100644 --- a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcImportPartitioner.java +++ b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcImportPartitioner.java @@ -39,14 +39,10 @@ public class GenericJdbcImportPartitioner extends Partitioner<ConnectionConfigur @Override public List<Partition> getPartitions(ImmutableContext context, long maxPartitions, ConnectionConfiguration connection, ImportJobConfiguration job) { numberPartitions = maxPartitions; - partitionColumnName = context.getString( - GenericJdbcConnectorConstants.CONNECTOR_JDBC_PARTITION_COLUMNNAME); - partitionColumnType = Integer.parseInt(context.getString( - GenericJdbcConnectorConstants.CONNECTOR_JDBC_PARTITION_COLUMNTYPE)); - partitionMinValue = context.getString( - GenericJdbcConnectorConstants.CONNECTOR_JDBC_PARTITION_MINVALUE); - partitionMaxValue = context.getString( - GenericJdbcConnectorConstants.CONNECTOR_JDBC_PARTITION_MAXVALUE); + partitionColumnName = context.getString(GenericJdbcConnectorConstants.CONNECTOR_JDBC_PARTITION_COLUMNNAME); + partitionColumnType = context.getInt(GenericJdbcConnectorConstants.CONNECTOR_JDBC_PARTITION_COLUMNTYPE, -1); + partitionMinValue = context.getString(GenericJdbcConnectorConstants.CONNECTOR_JDBC_PARTITION_MINVALUE); + partitionMaxValue = context.getString(GenericJdbcConnectorConstants.CONNECTOR_JDBC_PARTITION_MAXVALUE); switch (partitionColumnType) { case Types.TINYINT: http://git-wip-us.apache.org/repos/asf/sqoop/blob/f89c8e59/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestExportLoader.java ---------------------------------------------------------------------- diff --git a/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestExportLoader.java b/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestExportLoader.java index bf3c5e2..5892cbb 100644 --- a/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestExportLoader.java +++ b/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestExportLoader.java @@ -23,6 +23,8 @@ import junit.framework.TestCase; import org.apache.sqoop.common.MutableContext; import org.apache.sqoop.common.MutableMapContext; +import org.apache.sqoop.connector.jdbc.configuration.ConnectionConfiguration; +import org.apache.sqoop.connector.jdbc.configuration.ExportJobConfiguration; import org.apache.sqoop.job.etl.Loader; import org.apache.sqoop.job.io.DataReader; @@ -59,20 +61,20 @@ public class TestExportLoader extends TestCase { public void testInsert() throws Exception { MutableContext context = new MutableMapContext(); - context.setString( - GenericJdbcConnectorConstants.CONNECTOR_JDBC_DRIVER, - GenericJdbcTestConstants.DRIVER); - context.setString( - GenericJdbcConnectorConstants.CONNECTOR_JDBC_URL, - GenericJdbcTestConstants.URL); + ConnectionConfiguration connectionConfig = new ConnectionConfiguration(); + + connectionConfig.connection.jdbcDriver = GenericJdbcTestConstants.DRIVER; + connectionConfig.connection.connectionString = GenericJdbcTestConstants.URL; + + ExportJobConfiguration jobConfig = new ExportJobConfiguration(); + context.setString(GenericJdbcConnectorConstants.CONNECTOR_JDBC_DATA_SQL, - "INSERT INTO " + executor.delimitIdentifier(tableName) - + " VALUES (?,?,?)"); + "INSERT INTO " + executor.delimitIdentifier(tableName) + " VALUES (?,?,?)"); Loader loader = new GenericJdbcExportLoader(); DummyReader reader = new DummyReader(); - loader.load(context, null, null, reader); + loader.load(context, connectionConfig, jobConfig, reader); int index = START; ResultSet rs = executor.executeQuery("SELECT * FROM " http://git-wip-us.apache.org/repos/asf/sqoop/blob/f89c8e59/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestImportExtractor.java ---------------------------------------------------------------------- diff --git a/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestImportExtractor.java b/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestImportExtractor.java index 168e572..0be713e 100644 --- a/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestImportExtractor.java +++ b/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestImportExtractor.java @@ -65,18 +65,16 @@ public class TestImportExtractor extends TestCase { public void testQuery() throws Exception { MutableContext context = new MutableMapContext(); - context.setString( - GenericJdbcConnectorConstants.CONNECTOR_JDBC_DRIVER, - GenericJdbcTestConstants.DRIVER); - context.setString( - GenericJdbcConnectorConstants.CONNECTOR_JDBC_URL, - GenericJdbcTestConstants.URL); - context.setString(GenericJdbcConnectorConstants.CONNECTOR_JDBC_DATA_SQL, - "SELECT * FROM " + executor.delimitIdentifier(tableName) - + " WHERE ${CONDITIONS}"); - ConnectionConfiguration connConf = new ConnectionConfiguration(); - ImportJobConfiguration jobConf = new ImportJobConfiguration(); + ConnectionConfiguration connectionConfig = new ConnectionConfiguration(); + + connectionConfig.connection.jdbcDriver = GenericJdbcTestConstants.DRIVER; + connectionConfig.connection.connectionString = GenericJdbcTestConstants.URL; + + ImportJobConfiguration jobConfig = new ImportJobConfiguration(); + + context.setString(GenericJdbcConnectorConstants.CONNECTOR_JDBC_DATA_SQL, + "SELECT * FROM " + executor.delimitIdentifier(tableName) + " WHERE ${CONDITIONS}"); GenericJdbcImportPartition partition; @@ -85,33 +83,32 @@ public class TestImportExtractor extends TestCase { partition = new GenericJdbcImportPartition(); partition.setConditions("-50.0 <= DCOL AND DCOL < -16.6666666666666665"); - extractor.run(context, connConf, jobConf, partition, writer); + extractor.run(context, connectionConfig, jobConfig, partition, writer); partition = new GenericJdbcImportPartition(); partition.setConditions("-16.6666666666666665 <= DCOL AND DCOL < 16.666666666666667"); - extractor.run(context, connConf, jobConf, partition, writer); + extractor.run(context, connectionConfig, jobConfig, partition, writer); partition = new GenericJdbcImportPartition(); partition.setConditions("16.666666666666667 <= DCOL AND DCOL <= 50.0"); - extractor.run(context, connConf, jobConf, partition, writer); + extractor.run(context, connectionConfig, jobConfig, partition, writer); } public void testSubquery() throws Exception { MutableContext context = new MutableMapContext(); - context.setString( - GenericJdbcConnectorConstants.CONNECTOR_JDBC_DRIVER, - GenericJdbcTestConstants.DRIVER); - context.setString( - GenericJdbcConnectorConstants.CONNECTOR_JDBC_URL, - GenericJdbcTestConstants.URL); + + ConnectionConfiguration connectionConfig = new ConnectionConfiguration(); + + connectionConfig.connection.jdbcDriver = GenericJdbcTestConstants.DRIVER; + connectionConfig.connection.connectionString = GenericJdbcTestConstants.URL; + + ImportJobConfiguration jobConfig = new ImportJobConfiguration(); + context.setString(GenericJdbcConnectorConstants.CONNECTOR_JDBC_DATA_SQL, "SELECT SQOOP_SUBQUERY_ALIAS.ICOL,SQOOP_SUBQUERY_ALIAS.VCOL FROM " + "(SELECT * FROM " + executor.delimitIdentifier(tableName) + " WHERE ${CONDITIONS}) SQOOP_SUBQUERY_ALIAS"); - ConnectionConfiguration connConf = new ConnectionConfiguration(); - ImportJobConfiguration jobConf = new ImportJobConfiguration(); - GenericJdbcImportPartition partition; Extractor extractor = new GenericJdbcImportExtractor(); @@ -119,15 +116,15 @@ public class TestImportExtractor extends TestCase { partition = new GenericJdbcImportPartition(); partition.setConditions("-50 <= ICOL AND ICOL < -16"); - extractor.run(context, connConf, jobConf, partition, writer); + extractor.run(context, connectionConfig, jobConfig, partition, writer); partition = new GenericJdbcImportPartition(); partition.setConditions("-16 <= ICOL AND ICOL < 17"); - extractor.run(context, connConf, jobConf, partition, writer); + extractor.run(context, connectionConfig, jobConfig, partition, writer); partition = new GenericJdbcImportPartition(); partition.setConditions("17 <= ICOL AND ICOL < 50"); - extractor.run(context, connConf, jobConf, partition, writer); + extractor.run(context, connectionConfig, jobConfig, partition, writer); } public class DummyWriter extends DataWriter {
