Updated Branches: refs/heads/trunk da5ab302e -> ad12695b5
SQOOP-1273: Multiple append jobs can easily end up sharing directories (Jarek Jarcec Cecho via Venkat Ranganathan) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/ad12695b Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/ad12695b Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/ad12695b Branch: refs/heads/trunk Commit: ad12695b59e7f0af09e27da8dca08e9a2be9b6a2 Parents: da5ab30 Author: Venkat Ranganathan <[email protected]> Authored: Wed Jan 29 11:29:47 2014 -0800 Committer: Venkat Ranganathan <[email protected]> Committed: Wed Jan 29 11:29:47 2014 -0800 ---------------------------------------------------------------------- src/java/org/apache/sqoop/tool/ImportTool.java | 6 +++- src/java/org/apache/sqoop/util/AppendUtils.java | 11 ++++--- .../com/cloudera/sqoop/TestAppendUtils.java | 32 +++++++++++++++----- 3 files changed, 37 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/ad12695b/src/java/org/apache/sqoop/tool/ImportTool.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/tool/ImportTool.java b/src/java/org/apache/sqoop/tool/ImportTool.java index 59279b6..6cbb873 100644 --- a/src/java/org/apache/sqoop/tool/ImportTool.java +++ b/src/java/org/apache/sqoop/tool/ImportTool.java @@ -458,7 +458,11 @@ public class ImportTool extends com.cloudera.sqoop.tool.BaseSqoopTool { Path outputPath = null; if (options.isAppendMode()) { // Use temporary path, later removed when appending - outputPath = AppendUtils.getTempAppendDir(tableName); + String salt = tableName; + if(salt == null && options.getSqlQuery() != null) { + salt = Integer.toHexString(options.getSqlQuery().hashCode()); + } + outputPath = AppendUtils.getTempAppendDir(salt); LOG.debug("Using temporary folder: " + outputPath.getName()); } else { // Try in this order: target-dir or warehouse-dir http://git-wip-us.apache.org/repos/asf/sqoop/blob/ad12695b/src/java/org/apache/sqoop/util/AppendUtils.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/util/AppendUtils.java b/src/java/org/apache/sqoop/util/AppendUtils.java index d7cd6c5..f5a9a60 100644 --- a/src/java/org/apache/sqoop/util/AppendUtils.java +++ b/src/java/org/apache/sqoop/util/AppendUtils.java @@ -19,6 +19,7 @@ package org.apache.sqoop.util; import java.io.IOException; +import java.lang.management.ManagementFactory; import java.text.NumberFormat; import java.text.SimpleDateFormat; import java.util.Date; @@ -41,7 +42,7 @@ public class AppendUtils { public static final Log LOG = LogFactory.getLog(AppendUtils.class.getName()); private static final SimpleDateFormat DATE_FORM = new SimpleDateFormat( - "ddHHmmssSSS"); + "ddHHmmssSSSSSSSSS"); private static final String TEMP_IMPORT_ROOT = System.getProperty("sqoop.test.import.rootDir", "_sqoop"); @@ -263,12 +264,14 @@ public class AppendUtils { /** * Creates a unique path object inside the sqoop temporary directory. * - * @param tableName + * @param salt Salt that will be appended at the end of the generated directory. + * Can be arbitrary string, for example table name or query checksum. * @return a path pointing to the temporary directory */ - public static Path getTempAppendDir(String tableName) { + public static Path getTempAppendDir(String salt) { String timeId = DATE_FORM.format(new Date(System.currentTimeMillis())); - String tempDir = TEMP_IMPORT_ROOT + Path.SEPARATOR + timeId + tableName; + String jvmName = ManagementFactory.getRuntimeMXBean().getName().replaceAll("@", "_"); + String tempDir = TEMP_IMPORT_ROOT + Path.SEPARATOR + timeId + "_" + jvmName + "_" + salt; return new Path(tempDir); } http://git-wip-us.apache.org/repos/asf/sqoop/blob/ad12695b/src/test/com/cloudera/sqoop/TestAppendUtils.java ---------------------------------------------------------------------- diff --git a/src/test/com/cloudera/sqoop/TestAppendUtils.java b/src/test/com/cloudera/sqoop/TestAppendUtils.java index ed66b4a..b09dfc6 100644 --- a/src/test/com/cloudera/sqoop/TestAppendUtils.java +++ b/src/test/com/cloudera/sqoop/TestAppendUtils.java @@ -58,7 +58,7 @@ public class TestAppendUtils extends ImportJobTestCase { * * @return the argv as an array of strings. */ - protected ArrayList getOutputlessArgv(boolean includeHadoopFlags, + protected ArrayList getOutputlessArgv(boolean includeHadoopFlags, boolean queryBased, String[] colNames, Configuration conf) { if (null == colNames) { colNames = getColNames(); @@ -76,8 +76,13 @@ public class TestAppendUtils extends ImportJobTestCase { CommonArgs.addHadoopFlags(args); } - args.add("--table"); - args.add(getTableName()); + if(queryBased) { + args.add("--query"); + args.add("SELECT * FROM " + getTableName() + " WHERE $CONDITIONS"); + } else { + args.add("--table"); + args.add(getTableName()); + } args.add("--columns"); args.add(columnsString); args.add("--split-by"); @@ -241,8 +246,7 @@ public class TestAppendUtils extends ImportJobTestCase { /** independent to target-dir. */ public void testAppend() throws IOException { - ArrayList args = getOutputlessArgv(false, HsqldbTestServer.getFieldNames(), - getConf()); + ArrayList args = getOutputlessArgv(false, false, HsqldbTestServer.getFieldNames(), getConf()); args.add("--warehouse-dir"); args.add(getWarehouseDir()); @@ -252,8 +256,7 @@ public class TestAppendUtils extends ImportJobTestCase { /** working with target-dir. */ public void testAppendToTargetDir() throws IOException { - ArrayList args = getOutputlessArgv(false, HsqldbTestServer.getFieldNames(), - getConf()); + ArrayList args = getOutputlessArgv(false, false, HsqldbTestServer.getFieldNames(), getConf()); String targetDir = getWarehouseDir() + "/tempTargetDir"; args.add("--target-dir"); args.add(targetDir); @@ -265,6 +268,21 @@ public class TestAppendUtils extends ImportJobTestCase { } /** + * Query based import should also work in append mode. + * + * @throws IOException + */ + public void testAppendWithQuery() throws IOException { + ArrayList args = getOutputlessArgv(false, true, HsqldbTestServer.getFieldNames(), getConf()); + String targetDir = getWarehouseDir() + "/tempTargetDir"; + args.add("--target-dir"); + args.add(targetDir); + + Path output = new Path(targetDir); + runAppendTest(args, output); + } + + /** * If the append source does not exist, don't crash. */ public void testAppendSrcDoesNotExist() throws IOException {
