Repository: sqoop Updated Branches: refs/heads/trunk 02e36db2b -> 5d8152eac
SQOOP-2607: Direct import from Netezza and encoding (Benjamin BONNET via Jarek Jarcec Cecho) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/5d8152ea Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/5d8152ea Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/5d8152ea Branch: refs/heads/trunk Commit: 5d8152eac846c30a87e327ec401c9403020620a5 Parents: 02e36db Author: Jarek Jarcec Cecho <[email protected]> Authored: Fri Oct 9 11:56:07 2015 -0700 Committer: Jarek Jarcec Cecho <[email protected]> Committed: Fri Oct 9 11:56:07 2015 -0700 ---------------------------------------------------------------------- .../apache/sqoop/manager/DirectNetezzaManager.java | 14 +++++++++++++- .../db/netezza/NetezzaExternalTableImportMapper.java | 4 +++- 2 files changed, 16 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/5d8152ea/src/java/org/apache/sqoop/manager/DirectNetezzaManager.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/manager/DirectNetezzaManager.java b/src/java/org/apache/sqoop/manager/DirectNetezzaManager.java index 2ec0770..af15824 100644 --- a/src/java/org/apache/sqoop/manager/DirectNetezzaManager.java +++ b/src/java/org/apache/sqoop/manager/DirectNetezzaManager.java @@ -81,6 +81,10 @@ public class DirectNetezzaManager extends NetezzaManager { public static final String NETEZZA_TRUNC_STRING_LONG_ARG = "trunc-string"; + public static final String NETEZZA_TABLE_ENCODING_OPT = + "netezza.table.encoding"; + public static final String NETEZZA_TABLE_ENCODING_LONG_ARG = + "encoding"; private static final String QUERY_CHECK_DICTIONARY_FOR_TABLE = @@ -286,6 +290,9 @@ public class DirectNetezzaManager extends NetezzaManager { netezzaOpts.addOption(OptionBuilder.withArgName(NETEZZA_IGNORE_ZERO_OPT) .withDescription("Truncate string to declared storage size") .withLongOpt(NETEZZA_IGNORE_ZERO_LONG_ARG).create()); + netezzaOpts.addOption(OptionBuilder.withArgName(NETEZZA_TABLE_ENCODING_OPT) + .hasArg().withDescription("Table encoding") + .withLongOpt(NETEZZA_TABLE_ENCODING_LONG_ARG).create()); return netezzaOpts; } @@ -307,7 +314,12 @@ public class DirectNetezzaManager extends NetezzaManager { String dir = cmdLine.getOptionValue(NETEZZA_LOG_DIR_LONG_ARG); conf.set(NETEZZA_LOG_DIR_OPT, dir); } - + if (cmdLine.hasOption(NETEZZA_TABLE_ENCODING_LONG_ARG)) { + String encoding = cmdLine + .getOptionValue(NETEZZA_TABLE_ENCODING_LONG_ARG); + conf.set(NETEZZA_TABLE_ENCODING_OPT, encoding); + } + conf.setBoolean(NETEZZA_CTRL_CHARS_OPT, cmdLine.hasOption(NETEZZA_CTRL_CHARS_LONG_ARG)); http://git-wip-us.apache.org/repos/asf/sqoop/blob/5d8152ea/src/java/org/apache/sqoop/mapreduce/db/netezza/NetezzaExternalTableImportMapper.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/mapreduce/db/netezza/NetezzaExternalTableImportMapper.java b/src/java/org/apache/sqoop/mapreduce/db/netezza/NetezzaExternalTableImportMapper.java index 939e358..2efea53 100644 --- a/src/java/org/apache/sqoop/mapreduce/db/netezza/NetezzaExternalTableImportMapper.java +++ b/src/java/org/apache/sqoop/mapreduce/db/netezza/NetezzaExternalTableImportMapper.java @@ -184,8 +184,10 @@ public abstract class NetezzaExternalTableImportMapper<K, V> extends extTableThread.start(); // We need to start the reader end first + final String encoding = conf + .get(DirectNetezzaManager.NETEZZA_TABLE_ENCODING_OPT); recordReader = new BufferedReader(new InputStreamReader( - new FileInputStream(nf.getFile()))); + new FileInputStream(nf.getFile()), (null == encoding ? "UTF-8" : encoding))); } abstract protected void writeRecord(Text text, Context context)
