Repository: sqoop Updated Branches: refs/heads/trunk 152fe9300 -> a0b730c77
SQOOP-2880: Provide argument for overriding temporary directory (Attila Szabo via Jarek Jarcec Cecho) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/a0b730c7 Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/a0b730c7 Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/a0b730c7 Branch: refs/heads/trunk Commit: a0b730c77e297a62909063289ef37a2b993ff5e1 Parents: 152fe93 Author: Jarek Jarcec Cecho <[email protected]> Authored: Mon Mar 14 09:21:16 2016 -0700 Committer: Jarek Jarcec Cecho <[email protected]> Committed: Mon Mar 14 09:21:16 2016 -0700 ---------------------------------------------------------------------- src/java/org/apache/sqoop/SqoopOptions.java | 16 +++++++++ .../org/apache/sqoop/tool/BaseSqoopTool.java | 11 ++++++ src/java/org/apache/sqoop/tool/ImportTool.java | 2 +- src/java/org/apache/sqoop/util/AppendUtils.java | 7 ++-- .../com/cloudera/sqoop/TestSqoopOptions.java | 36 ++++++++++++++++++++ .../cloudera/sqoop/orm/TestParseMethods.java | 10 ++++++ 6 files changed, 76 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/a0b730c7/src/java/org/apache/sqoop/SqoopOptions.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/SqoopOptions.java b/src/java/org/apache/sqoop/SqoopOptions.java index 17751c7..ff96280 100644 --- a/src/java/org/apache/sqoop/SqoopOptions.java +++ b/src/java/org/apache/sqoop/SqoopOptions.java @@ -55,6 +55,8 @@ import com.cloudera.sqoop.util.StoredAsProperty; */ public class SqoopOptions implements Cloneable { + private static final String OLD_SQOOP_TEST_IMPORT_ROOT_DIR = "sqoop.test.import.rootDir"; + public static final Log LOG = LogFactory.getLog(SqoopOptions.class.getName()); /** @@ -106,6 +108,8 @@ public class SqoopOptions implements Cloneable { @StoredAsProperty("verbose") private boolean verbose; + @StoredAsProperty("temporary.dirRoot") private String tempRootDir; + @StoredAsProperty("mapreduce.job.name") private String mapreduceJobName; @StoredAsProperty("db.connect.string") private String connectString; @@ -1007,6 +1011,10 @@ public class SqoopOptions implements Cloneable { // We do not want to be verbose too much if not explicitly needed this.verbose = false; + //This name of the system property is intentionally OLD_SQOOP_TEST_IMPORT_ROOT_DIR + //to support backward compatibility. Do not exchange it with + //org.apache.sqoop.tool.BaseSqoopTool#TEMP_ROOTDIR_ARG + this.tempRootDir = System.getProperty(OLD_SQOOP_TEST_IMPORT_ROOT_DIR, "_sqoop"); this.isValidationEnabled = false; // validation is disabled by default this.validatorClass = RowCountValidator.class; this.validationThresholdClass = AbsoluteValidationThreshold.class; @@ -1110,6 +1118,14 @@ public class SqoopOptions implements Cloneable { this.verbose = beVerbose; } + public String getTempRootDir() { + return tempRootDir; + } + + public void setTempRootDir(String tempRootDir) { + this.tempRootDir = tempRootDir; + } + /** * Get the temporary directory; guaranteed to end in File.separator * (e.g., '/'). http://git-wip-us.apache.org/repos/asf/sqoop/blob/a0b730c7/src/java/org/apache/sqoop/tool/BaseSqoopTool.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/tool/BaseSqoopTool.java b/src/java/org/apache/sqoop/tool/BaseSqoopTool.java index 50dd67d..fecdf43 100644 --- a/src/java/org/apache/sqoop/tool/BaseSqoopTool.java +++ b/src/java/org/apache/sqoop/tool/BaseSqoopTool.java @@ -162,6 +162,7 @@ public abstract class BaseSqoopTool extends com.cloudera.sqoop.tool.SqoopTool { public static final String SQL_QUERY_SHORT_ARG = "e"; public static final String VERBOSE_ARG = "verbose"; public static final String HELP_ARG = "help"; + public static final String TEMP_ROOTDIR_ARG = "temporary-rootdir"; public static final String UPDATE_KEY_ARG = "update-key"; public static final String UPDATE_MODE_ARG = "update-mode"; public static final String CALL_ARG = "call"; @@ -467,6 +468,12 @@ public abstract class BaseSqoopTool extends com.cloudera.sqoop.tool.SqoopTool { .withDescription("Print usage instructions") .withLongOpt(HELP_ARG) .create()); + commonOpts.addOption(OptionBuilder + .withDescription("Defines the temporary root directory for the import") + .withLongOpt(TEMP_ROOTDIR_ARG) + .hasArg() + .withArgName("rootdir") + .create()); // relax isolation requirements commonOpts.addOption(OptionBuilder .withDescription("Use read-uncommitted isolation for imports") @@ -939,6 +946,10 @@ public abstract class BaseSqoopTool extends com.cloudera.sqoop.tool.SqoopTool { throw new InvalidOptionsException(""); } + if (in.hasOption(TEMP_ROOTDIR_ARG)) { + out.setTempRootDir(in.getOptionValue(TEMP_ROOTDIR_ARG)); + } + if (in.hasOption(CONNECT_STRING_ARG)) { out.setConnectString(in.getOptionValue(CONNECT_STRING_ARG)); } http://git-wip-us.apache.org/repos/asf/sqoop/blob/a0b730c7/src/java/org/apache/sqoop/tool/ImportTool.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/tool/ImportTool.java b/src/java/org/apache/sqoop/tool/ImportTool.java index ad1d48b..ff7b822 100644 --- a/src/java/org/apache/sqoop/tool/ImportTool.java +++ b/src/java/org/apache/sqoop/tool/ImportTool.java @@ -571,7 +571,7 @@ public class ImportTool extends com.cloudera.sqoop.tool.BaseSqoopTool { if(salt == null && options.getSqlQuery() != null) { salt = Integer.toHexString(options.getSqlQuery().hashCode()); } - outputPath = AppendUtils.getTempAppendDir(salt); + outputPath = AppendUtils.getTempAppendDir(salt, options); LOG.debug("Using temporary folder: " + outputPath.getName()); } else { // Try in this order: target-dir or warehouse-dir http://git-wip-us.apache.org/repos/asf/sqoop/blob/a0b730c7/src/java/org/apache/sqoop/util/AppendUtils.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/util/AppendUtils.java b/src/java/org/apache/sqoop/util/AppendUtils.java index 0102ee7..a3082c4 100644 --- a/src/java/org/apache/sqoop/util/AppendUtils.java +++ b/src/java/org/apache/sqoop/util/AppendUtils.java @@ -39,9 +39,6 @@ public class AppendUtils { public static final Log LOG = LogFactory.getLog(AppendUtils.class.getName()); - private static final String TEMP_IMPORT_ROOT = - System.getProperty("sqoop.test.import.rootDir", "_sqoop"); - private static final int PARTITION_DIGITS = 5; private static final String FILEPART_SEPARATOR = "-"; private static final String FILEEXT_SEPARATOR = "."; @@ -273,9 +270,9 @@ public class AppendUtils { * Can be arbitrary string, for example table name or query checksum. * @return a path pointing to the temporary directory */ - public static Path getTempAppendDir(String salt) { + public static Path getTempAppendDir(String salt, SqoopOptions options) { String uuid = UUID.randomUUID().toString().replace("-", ""); - String tempDir = TEMP_IMPORT_ROOT + Path.SEPARATOR + uuid + "_" + salt; + String tempDir = options.getTempRootDir() + Path.SEPARATOR + uuid + "_" + salt; return new Path(tempDir); } http://git-wip-us.apache.org/repos/asf/sqoop/blob/a0b730c7/src/test/com/cloudera/sqoop/TestSqoopOptions.java ---------------------------------------------------------------------- diff --git a/src/test/com/cloudera/sqoop/TestSqoopOptions.java b/src/test/com/cloudera/sqoop/TestSqoopOptions.java index f4660e5..f9d1d54 100644 --- a/src/test/com/cloudera/sqoop/TestSqoopOptions.java +++ b/src/test/com/cloudera/sqoop/TestSqoopOptions.java @@ -390,6 +390,42 @@ public class TestSqoopOptions extends TestCase { connParams, in.getConnectionParams()); } + public void testDefaultTempRootDir() { + SqoopOptions opts = new SqoopOptions(); + + assertEquals("_sqoop", opts.getTempRootDir()); + } + + public void testDefaultLoadedTempRootDir() { + SqoopOptions out = new SqoopOptions(); + Properties props = out.writeProperties(); + SqoopOptions opts = new SqoopOptions(); + opts.loadProperties(props); + + assertEquals("_sqoop", opts.getTempRootDir()); + } + + public void testLoadedTempRootDir() { + SqoopOptions out = new SqoopOptions(); + final String tempRootDir = "customRoot"; + out.setTempRootDir(tempRootDir); + Properties props = out.writeProperties(); + SqoopOptions opts = new SqoopOptions(); + opts.loadProperties(props); + + assertEquals(tempRootDir, opts.getTempRootDir()); + } + + public void testNulledTempRootDir() { + SqoopOptions out = new SqoopOptions(); + out.setTempRootDir(null); + Properties props = out.writeProperties(); + SqoopOptions opts = new SqoopOptions(); + opts.loadProperties(props); + + assertEquals("_sqoop", opts.getTempRootDir()); + } + // test that hadoop-home is accepted as an option public void testHadoopHome() throws Exception { String [] args = { http://git-wip-us.apache.org/repos/asf/sqoop/blob/a0b730c7/src/test/com/cloudera/sqoop/orm/TestParseMethods.java ---------------------------------------------------------------------- diff --git a/src/test/com/cloudera/sqoop/orm/TestParseMethods.java b/src/test/com/cloudera/sqoop/orm/TestParseMethods.java index f222dd7..cec7614 100644 --- a/src/test/com/cloudera/sqoop/orm/TestParseMethods.java +++ b/src/test/com/cloudera/sqoop/orm/TestParseMethods.java @@ -30,6 +30,7 @@ import org.apache.hadoop.mapred.FileInputFormat; import org.apache.hadoop.mapred.FileOutputFormat; import org.apache.hadoop.mapred.JobClient; import org.apache.hadoop.mapred.JobConf; +import org.apache.sqoop.tool.BaseSqoopTool; import com.cloudera.sqoop.SqoopOptions; import com.cloudera.sqoop.SqoopOptions.InvalidOptionsException; @@ -86,6 +87,15 @@ public class TestParseMethods extends ImportJobTestCase { return args.toArray(new String[0]); } + public void testTemporaryRootDirParse() throws Exception { + String customRoot = "customroot"; + String[] args = new String[] {"--"+BaseSqoopTool.TEMP_ROOTDIR_ARG, customRoot}; + + SqoopOptions opts = new ImportTool().parseArguments(args, null, null, true); + + assertEquals(customRoot, opts.getTempRootDir()); + } + public void runParseTest(String fieldTerminator, String lineTerminator, String encloser, String escape, boolean encloseRequired) throws IOException {
