Updated Branches: refs/heads/trunk 73cc549d9 -> ab4bfb9a8
SQOOP-1124: Direct export to PostgreSQL with PGBulkloadManager fails without explicit port number settings (Masatake Iwasaki via Jarek Jarcec Cecho) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/ab4bfb9a Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/ab4bfb9a Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/ab4bfb9a Branch: refs/heads/trunk Commit: ab4bfb9a8281d714b6549ba9d394d39e5c9c4d4a Parents: 73cc549 Author: Jarek Jarcec Cecho <[email protected]> Authored: Thu Jul 11 08:34:34 2013 -0700 Committer: Jarek Jarcec Cecho <[email protected]> Committed: Thu Jul 11 08:34:34 2013 -0700 ---------------------------------------------------------------------- build.xml | 24 +++++++++++++++++++ .../postgresql/PGBulkloadExportJob.java | 1 - .../postgresql/PGBulkloadExportMapper.java | 6 +++-- .../manager/PGBulkloadManagerManualTest.java | 25 +++++++++++++++----- 4 files changed, 47 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/ab4bfb9a/build.xml ---------------------------------------------------------------------- diff --git a/build.xml b/build.xml index ef657e3..e3683f8 100644 --- a/build.xml +++ b/build.xml @@ -254,6 +254,18 @@ <property name="sqoop.test.postgresql.connectstring.host_url" value="jdbc:postgresql://localhost/"/> + <property name="sqoop.test.postgresql.database" + value="sqooptest" /> + + <property name="sqoop.test.postgresql.tablespace" + value="sqooptest" /> + + <property name="sqoop.test.postgresql.username" + value="sqooptest" /> + + <property name="sqoop.test.postgresql.pg_bulkload" + value="pg_bulkload" /> + <property name="sqoop.test.sqlserver.connectstring.host_url" value="jdbc:sqlserver://sqlserverhost:1433"/> @@ -818,6 +830,18 @@ <sysproperty key="sqoop.test.postgresql.connectstring.host_url" value="${sqoop.test.postgresql.connectstring.host_url}"/> + <sysproperty key="sqoop.test.postgresql.database" + value="${sqoop.test.postgresql.database}" /> + + <sysproperty key="sqoop.test.postgresql.tablespace" + value="${sqoop.test.postgresql.tablespace}" /> + + <sysproperty key="sqoop.test.postgresql.username" + value="${sqoop.test.postgresql.username}" /> + + <sysproperty key="sqoop.test.postgresql.pg_bulkload" + value="${sqoop.test.postgresql.pg_bulkload}" /> + <sysproperty key="sqoop.test.sqlserver.connectstring.host_url" value="${sqoop.test.sqlserver.connectstring.host_url}"/> http://git-wip-us.apache.org/repos/asf/sqoop/blob/ab4bfb9a/src/java/org/apache/sqoop/mapreduce/postgresql/PGBulkloadExportJob.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/mapreduce/postgresql/PGBulkloadExportJob.java b/src/java/org/apache/sqoop/mapreduce/postgresql/PGBulkloadExportJob.java index 79fb7da..1e2ad9f 100644 --- a/src/java/org/apache/sqoop/mapreduce/postgresql/PGBulkloadExportJob.java +++ b/src/java/org/apache/sqoop/mapreduce/postgresql/PGBulkloadExportJob.java @@ -148,7 +148,6 @@ public class PGBulkloadExportJob extends ExportJobBase { @Override public void runExport() throws ExportException, IOException { - ConnManager cmgr = context.getConnManager(); SqoopOptions options = context.getOptions(); Configuration conf = options.getConf(); DBConfiguration dbConf = null; http://git-wip-us.apache.org/repos/asf/sqoop/blob/ab4bfb9a/src/java/org/apache/sqoop/mapreduce/postgresql/PGBulkloadExportMapper.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/mapreduce/postgresql/PGBulkloadExportMapper.java b/src/java/org/apache/sqoop/mapreduce/postgresql/PGBulkloadExportMapper.java index 333546f..16af84c 100644 --- a/src/java/org/apache/sqoop/mapreduce/postgresql/PGBulkloadExportMapper.java +++ b/src/java/org/apache/sqoop/mapreduce/postgresql/PGBulkloadExportMapper.java @@ -125,8 +125,10 @@ public class PGBulkloadExportMapper + JdbcUrl.getDatabaseName(conf.get(DBConfiguration.URL_PROPERTY))); args.add("--host=" + JdbcUrl.getHostName(conf.get(DBConfiguration.URL_PROPERTY))); - args.add("--port=" - + JdbcUrl.getPort(conf.get(DBConfiguration.URL_PROPERTY))); + int port = JdbcUrl.getPort(conf.get(DBConfiguration.URL_PROPERTY)); + if (port != -1) { + args.add("--port=" + port); + } args.add("--input=stdin"); args.add("--output=" + tmpTableName); args.add("-o"); http://git-wip-us.apache.org/repos/asf/sqoop/blob/ab4bfb9a/src/test/com/cloudera/sqoop/manager/PGBulkloadManagerManualTest.java ---------------------------------------------------------------------- diff --git a/src/test/com/cloudera/sqoop/manager/PGBulkloadManagerManualTest.java b/src/test/com/cloudera/sqoop/manager/PGBulkloadManagerManualTest.java index 0403614..4d03a8b 100644 --- a/src/test/com/cloudera/sqoop/manager/PGBulkloadManagerManualTest.java +++ b/src/test/com/cloudera/sqoop/manager/PGBulkloadManagerManualTest.java @@ -26,7 +26,7 @@ import java.util.Arrays; import java.util.ArrayList; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.mapred.JobConf; import com.cloudera.sqoop.TestExport; import com.cloudera.sqoop.mapreduce.db.DBConfiguration; @@ -64,10 +64,21 @@ public class PGBulkloadManagerManualTest extends TestExport { public static final Log LOG = LogFactory.getLog(PGBulkloadManagerManualTest.class.getName()); private DBConfiguration dbConf; - + static final String HOST_URL = + System.getProperty("sqoop.test.postgresql.connectstring.host_url", + "jdbc:postgresql://localhost/"); + static final String DATABASE = + System.getProperty("sqoop.test.postgresql.database", "sqooptest"); + static final String TABLESPACE = + System.getProperty("sqoop.test.postgresql.tablespace", "sqooptest"); + static final String USERNAME = + System.getProperty("sqoop.test.postgresql.username", "sqooptest"); + static final String PG_BULKLOAD = + System.getProperty("sqoop.test.postgresql.pg_bulkload", "pg_bulkload"); + static final String CONNECT_STRING = HOST_URL + DATABASE; public PGBulkloadManagerManualTest() { - Configuration conf = getConf(); + JobConf conf = new JobConf(getConf()); DBConfiguration.configureDB(conf, "org.postgresql.Driver", getConnectString(), @@ -85,12 +96,12 @@ public class PGBulkloadManagerManualTest extends TestExport { @Override protected String getConnectString() { - return "jdbc:postgresql://localhost:5432/sqooptest"; + return CONNECT_STRING; } protected String getUserName() { - return "sqooptest"; + return USERNAME; } @@ -144,6 +155,8 @@ public class PGBulkloadManagerManualTest extends TestExport { String... additionalArgv) { ArrayList<String> args = new ArrayList<String>(Arrays.asList(additionalArgv)); + args.add("-D"); + args.add("pgbulkload.bin=" + PG_BULKLOAD); args.add("--username"); args.add(getUserName()); args.add("--connection-manager"); @@ -181,7 +194,7 @@ public class PGBulkloadManagerManualTest extends TestExport { public void testExportWithTablespace() throws IOException, SQLException { String[] genericargs = - newStrArray(null, "-Dpgbulkload.staging.tablespace=sqooptest"); + newStrArray(null, "-Dpgbulkload.staging.tablespace=" + TABLESPACE); multiFileTestWithGenericArgs(1, 10, 1, genericargs); }
