Repository: sqoop Updated Branches: refs/heads/trunk 8b6eb33f1 -> c32010f44
SQOOP-1826: NPE in ImportTool.lastModifiedMerge during postgres import (Ricky Nguyen via Jarek Jarcec Cecho) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/c32010f4 Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/c32010f4 Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/c32010f4 Branch: refs/heads/trunk Commit: c32010f440f139b37840df6a3433f875898136d5 Parents: 8b6eb33 Author: Jarek Jarcec Cecho <[email protected]> Authored: Tue Jan 6 10:14:38 2015 +0100 Committer: Jarek Jarcec Cecho <[email protected]> Committed: Tue Jan 6 10:14:38 2015 +0100 ---------------------------------------------------------------------- .../sqoop/manager/DirectPostgresqlManager.java | 2 ++ .../sqoop/manager/PostgresqlImportTest.java | 37 ++++++++++++++++++-- 2 files changed, 36 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/c32010f4/src/java/org/apache/sqoop/manager/DirectPostgresqlManager.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/manager/DirectPostgresqlManager.java b/src/java/org/apache/sqoop/manager/DirectPostgresqlManager.java index 8d4a097..63b0704 100644 --- a/src/java/org/apache/sqoop/manager/DirectPostgresqlManager.java +++ b/src/java/org/apache/sqoop/manager/DirectPostgresqlManager.java @@ -348,6 +348,8 @@ public class DirectPostgresqlManager public void importTable(com.cloudera.sqoop.manager.ImportJobContext context) throws IOException, ImportException { + context.setConnManager(this); + String tableName = context.getTableName(); SqoopOptions options = context.getOptions(); http://git-wip-us.apache.org/repos/asf/sqoop/blob/c32010f4/src/test/com/cloudera/sqoop/manager/PostgresqlImportTest.java ---------------------------------------------------------------------- diff --git a/src/test/com/cloudera/sqoop/manager/PostgresqlImportTest.java b/src/test/com/cloudera/sqoop/manager/PostgresqlImportTest.java index 3b5731b..57dd338 100644 --- a/src/test/com/cloudera/sqoop/manager/PostgresqlImportTest.java +++ b/src/test/com/cloudera/sqoop/manager/PostgresqlImportTest.java @@ -235,15 +235,18 @@ public class PostgresqlImportTest extends ImportJobTestCase { return args.toArray(new String[0]); } - private void doImportAndVerify(boolean isDirect, String [] expectedResults, + private void doImportAndVerify(boolean isDirect, String[] expectedResults, String tableName, String... extraArgs) throws IOException { Path warehousePath = new Path(this.getWarehouseDir()); Path tablePath = new Path(warehousePath, tableName); - Path filePath = new Path(tablePath, "part-m-00000"); + + // if importing with merge step, directory should exist and output should be from a reducer + boolean isMerge = Arrays.asList(extraArgs).contains("--merge-key"); + Path filePath = new Path(tablePath, isMerge ? "part-r-00000" : "part-m-00000"); File tableFile = new File(tablePath.toString()); - if (tableFile.exists() && tableFile.isDirectory()) { + if (tableFile.exists() && tableFile.isDirectory() && !isMerge) { // remove the directory before running the import. FileListing.recursiveDeleteDir(tableFile); } @@ -329,6 +332,34 @@ public class PostgresqlImportTest extends ImportJobTestCase { doImportAndVerify(false, expectedResults, TABLE_NAME, extraArgs); } + public void testDirectIncrementalImport() throws IOException { + String [] expectedResults = { }; + + String [] extraArgs = { "--incremental", "lastmodified", + "--check-column", "start_date", + }; + + doImportAndVerify(true, expectedResults, TABLE_NAME, extraArgs); + } + + public void testDirectIncrementalImportMerge() throws IOException { + String [] expectedResults = { }; + + String [] extraArgs = { "--incremental", "lastmodified", + "--check-column", "start_date", + }; + + doImportAndVerify(true, expectedResults, TABLE_NAME, extraArgs); + + extraArgs = new String[] { "--incremental", "lastmodified", + "--check-column", "start_date", + "--merge-key", "id", + "--last-value", "2009-04-20" + }; + + doImportAndVerify(true, expectedResults, TABLE_NAME, extraArgs); + } + @Test public void testDifferentSchemaImport() throws IOException { String [] expectedResults = {
