Updated Branches: refs/heads/trunk 0affc4ba9 -> 8c3c78b11
SQOOP-901: Allow user to override hardcoded boolean strings in direct PostgreSQL connector (Jarcec Cecho via Cheolsoo Park) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/8c3c78b1 Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/8c3c78b1 Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/8c3c78b1 Branch: refs/heads/trunk Commit: 8c3c78b11cb5c2330e31cbf867d32974a011f268 Parents: 0affc4b Author: Cheolsoo Park <[email protected]> Authored: Sat Feb 23 19:31:59 2013 -0800 Committer: Cheolsoo Park <[email protected]> Committed: Sat Feb 23 19:31:59 2013 -0800 ---------------------------------------------------------------------- src/docs/user/connectors.txt | 16 ++++ .../sqoop/manager/DirectPostgresqlManager.java | 69 ++++++++++++++- .../apache/sqoop/manager/PostgresqlManager.java | 9 ++- .../sqoop/manager/PostgresqlImportTest.java | 57 ++++++++---- 4 files changed, 125 insertions(+), 26 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/8c3c78b1/src/docs/user/connectors.txt ---------------------------------------------------------------------- diff --git a/src/docs/user/connectors.txt b/src/docs/user/connectors.txt index ff424f7..7dd2a2e 100644 --- a/src/docs/user/connectors.txt +++ b/src/docs/user/connectors.txt @@ -98,6 +98,22 @@ Argument Description Default is "public". --------------------------------------------------------------------------------- +The direct connector (used when specified +\--direct+ parameter), offers also +additional extra arguments: + +.Additional supported PostgreSQL extra arguments in direct mode: +[grid="all"] +`----------------------------------------`--------------------------------------- +Argument Description +--------------------------------------------------------------------------------- ++\--boolean-true-string <str>+ String that will be used to encode \ + +true+ value of +boolean+ columns. + Default is "TRUE". ++\--boolean-false-string <str>+ String that will be used to encode \ + +false+ value of +boolean+ columns. + Default is "FALSE". +--------------------------------------------------------------------------------- + Schema support ^^^^^^^^^^^^^^ http://git-wip-us.apache.org/repos/asf/sqoop/blob/8c3c78b1/src/java/org/apache/sqoop/manager/DirectPostgresqlManager.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/manager/DirectPostgresqlManager.java b/src/java/org/apache/sqoop/manager/DirectPostgresqlManager.java index a05bf60..c085218 100644 --- a/src/java/org/apache/sqoop/manager/DirectPostgresqlManager.java +++ b/src/java/org/apache/sqoop/manager/DirectPostgresqlManager.java @@ -30,9 +30,12 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.OptionBuilder; import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.sqoop.cli.RelatedOptions; import org.apache.sqoop.util.PostgreSQLUtils; import com.cloudera.sqoop.SqoopOptions; @@ -54,15 +57,33 @@ import org.apache.sqoop.util.SubstitutionUtils; */ public class DirectPostgresqlManager extends com.cloudera.sqoop.manager.PostgresqlManager { + public static final Log LOG = LogFactory.getLog( DirectPostgresqlManager.class.getName()); + public static final String BOOLEAN_TRUE_STRING = "boolean-true-string"; + public static final String DEFAULT_BOOLEAN_TRUE_STRING = "TRUE"; + + public static final String BOOLEAN_FALSE_STRING = "boolean-false-string"; + public static final String DEFAULT_BOOLEAN_FALSE_STRING = "FALSE"; + public DirectPostgresqlManager(final SqoopOptions opts) { super(opts); + + if (this.booleanFalseString == null) { + this.booleanFalseString = DEFAULT_BOOLEAN_FALSE_STRING; + } + if (booleanTrueString == null) { + this.booleanTrueString = DEFAULT_BOOLEAN_TRUE_STRING; + } } private static final String PSQL_CMD = "psql"; + private String booleanTrueString; + + private String booleanFalseString; + /** Copies data directly into HDFS, adding the user's chosen line terminator char to each record. */ @@ -196,12 +217,18 @@ public class DirectPostgresqlManager sb.append(col); } else { if ("bool".equalsIgnoreCase(columnTypes.get(col))) { - sb.append(String.format("case when %s=true then 'TRUE' " - + "when %s=false then 'FALSE' end as %s", + sb.append(String.format("case when %s=true then " + + "'" + booleanTrueString + "' " + + "when %s=false then " + + "'" + booleanFalseString + "'" + + " end as %s", colEscaped, colEscaped, colEscaped)); } else if ("bit".equalsIgnoreCase(columnTypes.get(col))) { - sb.append(String.format("case when %s=B'1' then 'TRUE' " - + "when %s=B'0' then 'FALSE' end as %s", + sb.append(String.format("case when %s=B'1' then " + + "'" + booleanTrueString + "' " + + "when %s=B'0' then " + + "'" + booleanFalseString + "'" + + " end as %s", colEscaped, colEscaped, colEscaped)); } else { sb.append(colEscaped); @@ -508,4 +535,38 @@ public class DirectPostgresqlManager return false; } // CHECKSTYLE:ON + + /** {@inheritDoc}. */ + @Override + protected void applyExtraArguments(CommandLine cmdLine) { + super.applyExtraArguments(cmdLine); + + if (cmdLine.hasOption(BOOLEAN_TRUE_STRING)) { + String arg = cmdLine.getOptionValue(BOOLEAN_TRUE_STRING); + LOG.info("Loaded TRUE encoding string " + arg); + this.booleanTrueString = arg; + } + if (cmdLine.hasOption(BOOLEAN_FALSE_STRING)) { + String arg = cmdLine.getOptionValue(BOOLEAN_FALSE_STRING); + LOG.info("Loaded FALSE encoding string " + arg); + this.booleanFalseString = arg; + } + } + + /** {@inheritDoc}. */ + @Override + @SuppressWarnings("static-access") + protected RelatedOptions getExtraOptions() { + RelatedOptions extraOptions = super.getExtraOptions(); + + extraOptions.addOption(OptionBuilder.withArgName("string").hasArg() + .withDescription("String to encode TRUE value") + .withLongOpt(BOOLEAN_TRUE_STRING).create()); + + extraOptions.addOption(OptionBuilder.withArgName("string").hasArg() + .withDescription("String to encode FALSE value") + .withLongOpt(BOOLEAN_FALSE_STRING).create()); + + return extraOptions; + } } http://git-wip-us.apache.org/repos/asf/sqoop/blob/8c3c78b1/src/java/org/apache/sqoop/manager/PostgresqlManager.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/manager/PostgresqlManager.java b/src/java/org/apache/sqoop/manager/PostgresqlManager.java index facdc74..609a457 100644 --- a/src/java/org/apache/sqoop/manager/PostgresqlManager.java +++ b/src/java/org/apache/sqoop/manager/PostgresqlManager.java @@ -196,7 +196,7 @@ public class PostgresqlManager * @param args Extra arguments array * @throws ParseException */ - void parseExtraArgs(String[] args) throws ParseException { + private void parseExtraArgs(String[] args) throws ParseException { // No-op when no extra arguments are present if (args == null || args.length == 0) { return; @@ -207,6 +207,11 @@ public class PostgresqlManager CommandLineParser parser = new GnuParser(); CommandLine cmdLine = parser.parse(getExtraOptions(), args, true); + // Apply parsed arguments + applyExtraArguments(cmdLine); + } + + protected void applyExtraArguments(CommandLine cmdLine) { // Apply extra options if (cmdLine.hasOption(SCHEMA)) { String schemaName = cmdLine.getOptionValue(SCHEMA); @@ -222,7 +227,7 @@ public class PostgresqlManager * @return */ @SuppressWarnings("static-access") - private RelatedOptions getExtraOptions() { + protected RelatedOptions getExtraOptions() { // Connection args (common) RelatedOptions extraOptions = new RelatedOptions("PostgreSQL extra options:"); http://git-wip-us.apache.org/repos/asf/sqoop/blob/8c3c78b1/src/test/com/cloudera/sqoop/manager/PostgresqlImportTest.java ---------------------------------------------------------------------- diff --git a/src/test/com/cloudera/sqoop/manager/PostgresqlImportTest.java b/src/test/com/cloudera/sqoop/manager/PostgresqlImportTest.java index ee00c41..3fadff7 100644 --- a/src/test/com/cloudera/sqoop/manager/PostgresqlImportTest.java +++ b/src/test/com/cloudera/sqoop/manager/PostgresqlImportTest.java @@ -139,12 +139,13 @@ public class PostgresqlImportTest extends ImportJobTestCase { connection.commit(); } catch (SQLException e) { LOG.info("Couldn't create schema " + schema + " (is o.k. as long as" - + "the schema already exists.", e); + + "the schema already exists."); connection.rollback(); } String fullTableName = manager.escapeTableName(schema) + "." + manager.escapeTableName(tableName); + LOG.info("Creating table: " + fullTableName); try { // Try to remove the table first. DROP TABLE IF EXISTS didn't @@ -152,8 +153,7 @@ public class PostgresqlImportTest extends ImportJobTestCase { // any exception here if one occurs. st.executeUpdate("DROP TABLE " + fullTableName); } catch (SQLException e) { - LOG.info("Couldn't drop table " + schema + "." + tableName + " (ok)", - e); + LOG.info("Couldn't drop table " + schema + "." + tableName + " (ok)"); // Now we need to reset the transaction. connection.rollback(); } @@ -163,17 +163,18 @@ public class PostgresqlImportTest extends ImportJobTestCase { + manager.escapeColName("name") + " VARCHAR(24) NOT NULL, " + manager.escapeColName("start_date") + " DATE, " + manager.escapeColName("Salary") + " FLOAT, " + + manager.escapeColName("Fired") + " BOOL, " + manager.escapeColName("dept") + " VARCHAR(32))"); st.executeUpdate("INSERT INTO " + fullTableName - + " VALUES(1,'Aaron','2009-05-14',1000000.00,'engineering')"); + + " VALUES(1,'Aaron','2009-05-14',1000000.00,TRUE,'engineering')"); st.executeUpdate("INSERT INTO " + fullTableName - + " VALUES(2,'Bob','2009-04-20',400.00,'sales')"); + + " VALUES(2,'Bob','2009-04-20',400.00,TRUE,'sales')"); st.executeUpdate("INSERT INTO " + fullTableName - + " VALUES(3,'Fred','2009-01-23',15.00,'marketing')"); + + " VALUES(3,'Fred','2009-01-23',15.00,FALSE,'marketing')"); if (nullEntry) { st.executeUpdate("INSERT INTO " + fullTableName - + " VALUES(4,'Mike',NULL,NULL,NULL)"); + + " VALUES(4,'Mike',NULL,NULL,NULL,NULL)"); } connection.commit(); @@ -276,8 +277,8 @@ public class PostgresqlImportTest extends ImportJobTestCase { @Test public void testJdbcBasedImport() throws IOException { String [] expectedResults = { - "2,Bob,2009-04-20,400.0,sales", - "3,Fred,2009-01-23,15.0,marketing", + "2,Bob,2009-04-20,400.0,true,sales", + "3,Fred,2009-01-23,15.0,false,marketing", }; doImportAndVerify(false, expectedResults, TABLE_NAME); @@ -286,8 +287,8 @@ public class PostgresqlImportTest extends ImportJobTestCase { @Test public void testDirectImport() throws IOException { String [] expectedResults = { - "2,Bob,2009-04-20,400,sales", - "3,Fred,2009-01-23,15,marketing", + "2,Bob,2009-04-20,400,TRUE,sales", + "3,Fred,2009-01-23,15,FALSE,marketing", }; doImportAndVerify(true, expectedResults, TABLE_NAME); @@ -309,8 +310,8 @@ public class PostgresqlImportTest extends ImportJobTestCase { @Test public void testTableNameWithSpecialCharacter() throws IOException { String [] expectedResults = { - "2,Bob,2009-04-20,400.0,sales", - "3,Fred,2009-01-23,15.0,marketing", + "2,Bob,2009-04-20,400.0,true,sales", + "3,Fred,2009-01-23,15.0,false,marketing", }; doImportAndVerify(false, expectedResults, SPECIAL_TABLE_NAME); @@ -330,8 +331,8 @@ public class PostgresqlImportTest extends ImportJobTestCase { @Test public void testDifferentSchemaImport() throws IOException { String [] expectedResults = { - "2,Bob,2009-04-20,400.0,sales", - "3,Fred,2009-01-23,15.0,marketing", + "2,Bob,2009-04-20,400.0,true,sales", + "3,Fred,2009-01-23,15.0,false,marketing", }; String [] extraArgs = { "--", @@ -344,8 +345,8 @@ public class PostgresqlImportTest extends ImportJobTestCase { @Test public void testDifferentSchemaImportDirect() throws IOException { String [] expectedResults = { - "2,Bob,2009-04-20,400,sales", - "3,Fred,2009-01-23,15,marketing", + "2,Bob,2009-04-20,400,TRUE,sales", + "3,Fred,2009-01-23,15,FALSE,marketing", }; String [] extraArgs = { "--", @@ -358,9 +359,9 @@ public class PostgresqlImportTest extends ImportJobTestCase { @Test public void testNullEscapeCharacters() throws Exception { String [] expectedResults = { - "2,Bob,2009-04-20,400,sales", - "3,Fred,2009-01-23,15,marketing", - "4,Mike,\\N,\\N,\\N", + "2,Bob,2009-04-20,400,TRUE,sales", + "3,Fred,2009-01-23,15,FALSE,marketing", + "4,Mike,\\N,\\N,\\N,\\N", }; String [] extraArgs = { @@ -370,4 +371,20 @@ public class PostgresqlImportTest extends ImportJobTestCase { doImportAndVerify(true, expectedResults, NULL_TABLE_NAME, extraArgs); } + + @Test + public void testDifferentBooleanValues() throws Exception { + String [] expectedResults = { + "2,Bob,2009-04-20,400,REAL_TRUE,sales", + "3,Fred,2009-01-23,15,REAL_FALSE,marketing", + }; + + String [] extraArgs = { + "--", + "--boolean-true-string", "REAL_TRUE", + "--boolean-false-string", "REAL_FALSE", + }; + + doImportAndVerify(true, expectedResults, TABLE_NAME, extraArgs); + } }
