Repository: sqoop Updated Branches: refs/heads/trunk c92f85574 -> d54dddf0b
SQOOP-2326: Fix Netezza trunc-string option handling and unnecessary log directory during imports (Venkat Ranganathan via Jarek Jarcec Cecho) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/d54dddf0 Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/d54dddf0 Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/d54dddf0 Branch: refs/heads/trunk Commit: d54dddf0b81d1d196788b4e6422b14b88c2a05ac Parents: c92f855 Author: Jarek Jarcec Cecho <[email protected]> Authored: Sun Apr 26 09:05:13 2015 -0700 Committer: Jarek Jarcec Cecho <[email protected]> Committed: Sun Apr 26 09:05:13 2015 -0700 ---------------------------------------------------------------------- .../sqoop/manager/DirectNetezzaManager.java | 4 +-- .../NetezzaExternalTableImportMapper.java | 26 +------------------- 2 files changed, 3 insertions(+), 27 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/d54dddf0/src/java/org/apache/sqoop/manager/DirectNetezzaManager.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/manager/DirectNetezzaManager.java b/src/java/org/apache/sqoop/manager/DirectNetezzaManager.java index daddb8c..06fa976 100644 --- a/src/java/org/apache/sqoop/manager/DirectNetezzaManager.java +++ b/src/java/org/apache/sqoop/manager/DirectNetezzaManager.java @@ -293,8 +293,8 @@ public class DirectNetezzaManager extends NetezzaManager { conf.setBoolean(NETEZZA_CTRL_CHARS_OPT, cmdLine.hasOption(NETEZZA_CTRL_CHARS_LONG_ARG)); - conf.setBoolean(NETEZZA_CTRL_CHARS_OPT, - cmdLine.hasOption(NETEZZA_CTRL_CHARS_LONG_ARG)); + conf.setBoolean(NETEZZA_TRUNC_STRING_OPT, + cmdLine.hasOption(NETEZZA_TRUNC_STRING_LONG_ARG)); // Always true for Netezza direct mode access conf.setBoolean(NETEZZA_DATASLICE_ALIGNED_ACCESS_OPT, true); http://git-wip-us.apache.org/repos/asf/sqoop/blob/d54dddf0/src/java/org/apache/sqoop/mapreduce/db/netezza/NetezzaExternalTableImportMapper.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/mapreduce/db/netezza/NetezzaExternalTableImportMapper.java b/src/java/org/apache/sqoop/mapreduce/db/netezza/NetezzaExternalTableImportMapper.java index 486ba7c..939e358 100644 --- a/src/java/org/apache/sqoop/mapreduce/db/netezza/NetezzaExternalTableImportMapper.java +++ b/src/java/org/apache/sqoop/mapreduce/db/netezza/NetezzaExternalTableImportMapper.java @@ -67,8 +67,6 @@ public abstract class NetezzaExternalTableImportMapper<K, V> extends .getLog(NetezzaExternalTableImportMapper.class.getName()); private NetezzaJDBCStatementRunner extTableThread; private PerfCounters counter; - private String localLogDir = null; - private String logDir = null; private File taskAttemptDir = null; private String getSqlStatement(int myId) throws IOException { @@ -78,10 +76,6 @@ public abstract class NetezzaExternalTableImportMapper<K, V> extends String nullValue = conf.get(DirectNetezzaManager.NETEZZA_NULL_VALUE); - boolean ctrlChars = - conf.getBoolean(DirectNetezzaManager.NETEZZA_CTRL_CHARS_OPT, false); - boolean truncString = - conf.getBoolean(DirectNetezzaManager.NETEZZA_TRUNC_STRING_OPT, false); int errorThreshold = conf.getInt( DirectNetezzaManager.NETEZZA_ERROR_THRESHOLD_OPT, 1); @@ -95,12 +89,6 @@ public abstract class NetezzaExternalTableImportMapper<K, V> extends sqlStmt.append("' USING (REMOTESOURCE 'JDBC' "); sqlStmt.append(" BOOLSTYLE 'T_F' "); sqlStmt.append(" CRINSTRING FALSE "); - if (ctrlChars) { - sqlStmt.append(" CTRLCHARS TRUE "); - } - if (truncString) { - sqlStmt.append(" TRUNCSTRING TRUE "); - } sqlStmt.append(" DELIMITER "); sqlStmt.append(Integer.toString(fd)); sqlStmt.append(" ENCODING 'internal' "); @@ -131,13 +119,6 @@ public abstract class NetezzaExternalTableImportMapper<K, V> extends sqlStmt.append(" MAXERRORS ").append(errorThreshold); - File logDirPath = new File(taskAttemptDir, localLogDir); - logDirPath.mkdirs(); - if (logDirPath.canWrite() && logDirPath.isDirectory()) { - sqlStmt.append(" LOGDIR ").append(logDirPath.getAbsolutePath()).append(' '); - } else { - throw new IOException("Unable to create log directory specified"); - } sqlStmt.append(") AS SELECT "); if (cols == null || cols.length == 0) { @@ -213,9 +194,7 @@ public abstract class NetezzaExternalTableImportMapper<K, V> extends public void map(Integer dataSliceId, NullWritable val, Context context) throws IOException, InterruptedException { conf = context.getConfiguration(); - localLogDir = - DirectNetezzaManager.getLocalLogDir(context.getTaskAttemptID()); - logDir = conf.get(DirectNetezzaManager.NETEZZA_LOG_DIR_OPT); + dbc = new DBConfiguration(conf); numMappers = ConfigurationHelper.getConfNumMaps(conf); @@ -250,9 +229,6 @@ public abstract class NetezzaExternalTableImportMapper<K, V> extends extTableThread.printException(); throw new IOException(extTableThread.getException()); } - FileUploader.uploadFilesToDFS(taskAttemptDir.getAbsolutePath(), - localLogDir, logDir, context.getJobID().toString(), - conf); } } }
