Repository: sqoop Updated Branches: refs/heads/trunk 5c4d06d72 -> c2351511e
SQOOP-2847. Sqoop --incremental + missing parent --target-dir reports success with no data (Jarcec via Hari) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/c2351511 Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/c2351511 Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/c2351511 Branch: refs/heads/trunk Commit: c2351511e618d1abb3cce78eab85545247a233c0 Parents: 5c4d06d Author: Hari Shreedharan <[email protected]> Authored: Tue Mar 1 10:42:21 2016 -0800 Committer: Hari Shreedharan <[email protected]> Committed: Tue Mar 1 10:42:21 2016 -0800 ---------------------------------------------------------------------- src/java/org/apache/sqoop/tool/ImportTool.java | 12 +++++++++- .../cloudera/sqoop/TestIncrementalImport.java | 23 +++++++++++++++++++- 2 files changed, 33 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/c2351511/src/java/org/apache/sqoop/tool/ImportTool.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/tool/ImportTool.java b/src/java/org/apache/sqoop/tool/ImportTool.java index 39af42c..ad1d48b 100644 --- a/src/java/org/apache/sqoop/tool/ImportTool.java +++ b/src/java/org/apache/sqoop/tool/ImportTool.java @@ -430,6 +430,7 @@ public class ImportTool extends com.cloudera.sqoop.tool.BaseSqoopTool { protected void lastModifiedMerge(SqoopOptions options, ImportJobContext context) throws IOException { FileSystem fs = FileSystem.get(options.getConf()); if (context.getDestination() != null && fs.exists(context.getDestination())) { + LOG.info("Final destination exists, will run merge job."); Path userDestDir = getOutputPath(options, context.getTableName(), false); if (fs.exists(userDestDir)) { String tableClassName = null; @@ -461,7 +462,16 @@ public class ImportTool extends com.cloudera.sqoop.tool.BaseSqoopTool { unloadJars(); } else { - fs.rename(context.getDestination(), userDestDir); + // Create parent directory(ies), otherwise fs.rename would fail + if(!fs.exists(userDestDir.getParent())) { + fs.mkdirs(userDestDir.getParent()); + } + + // And finally move the data + LOG.info("Moving data from temporary directory " + context.getDestination() + " to final destination " + userDestDir); + if(!fs.rename(context.getDestination(), userDestDir)) { + throw new RuntimeException("Couldn't move data from temporary directory " + context.getDestination() + " to final destination " + userDestDir); + } } } } http://git-wip-us.apache.org/repos/asf/sqoop/blob/c2351511/src/test/com/cloudera/sqoop/TestIncrementalImport.java ---------------------------------------------------------------------- diff --git a/src/test/com/cloudera/sqoop/TestIncrementalImport.java b/src/test/com/cloudera/sqoop/TestIncrementalImport.java index d28a187..d62532c 100644 --- a/src/test/com/cloudera/sqoop/TestIncrementalImport.java +++ b/src/test/com/cloudera/sqoop/TestIncrementalImport.java @@ -49,6 +49,8 @@ import com.cloudera.sqoop.testutil.CommonArgs; import com.cloudera.sqoop.tool.ImportTool; import com.cloudera.sqoop.tool.JobTool; +import javax.management.Query; + /** * Test the incremental import functionality. * @@ -543,12 +545,16 @@ public class TestIncrementalImport extends TestCase { if (commonArgs) { CommonArgs.addHadoopFlags(args); } + + String [] directoryNames = directoryName.split("/"); + String className = directoryNames[directoryNames.length -1]; + args.add("--connect"); args.add(SOURCE_DB_URL); args.add("--query"); args.add(query); args.add("--class-name"); - args.add(directoryName); + args.add(className); args.add("--target-dir"); args.add(BaseSqoopTestCase.LOCAL_WAREHOUSE_DIR + System.getProperty("file.separator") + directoryName); @@ -775,6 +781,21 @@ public class TestIncrementalImport extends TestCase { assertDirOfNumbers(TABLE_NAME, 0); } + public void testEmptyLastModifiedWithNonExistingParentDirectory() throws Exception { + final String TABLE_NAME = "emptyLastModifiedNoParent"; + final String QUERY = "SELECT id, last_modified FROM \"" + TABLE_NAME + "\" WHERE $CONDITIONS"; + final String DIRECTORY = "non-existing/parents/" + TABLE_NAME; + createTimestampTable(TABLE_NAME, 0, null); + List<String> args = getArgListForQuery(QUERY, DIRECTORY, true, false, false); + + Configuration conf = newConf(); + SqoopOptions options = new SqoopOptions(); + options.setConf(conf); + runImport(options, args); + + assertDirOfNumbers(DIRECTORY, 0); + } + public void testFullLastModifiedImport() throws Exception { // Given a table of rows imported in the past, // see that they are imported.
