Repository: sqoop Updated Branches: refs/heads/trunk b4afcf417 -> 14754342d
SQOOP-2986: Add validation check for --hive-import and --incremental lastmodified (Szabolcs Vasas via Attila Szabo) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/14754342 Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/14754342 Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/14754342 Branch: refs/heads/trunk Commit: 14754342d3a9bd6e146b9628b2e103ff30f310d8 Parents: b4afcf4 Author: Attila Szabo <mau...@apache.org> Authored: Thu Oct 13 15:21:56 2016 +0200 Committer: Attila Szabo <mau...@apache.org> Committed: Thu Oct 13 15:21:56 2016 +0200 ---------------------------------------------------------------------- .../org/apache/sqoop/tool/BaseSqoopTool.java | 8 ++ .../tool/ImportToolValidateOptionsTest.java | 92 ++++++++++++++++++++ 2 files changed, 100 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/14754342/src/java/org/apache/sqoop/tool/BaseSqoopTool.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/tool/BaseSqoopTool.java b/src/java/org/apache/sqoop/tool/BaseSqoopTool.java index b71bc5e..13a9697 100644 --- a/src/java/org/apache/sqoop/tool/BaseSqoopTool.java +++ b/src/java/org/apache/sqoop/tool/BaseSqoopTool.java @@ -230,6 +230,9 @@ public abstract class BaseSqoopTool extends com.cloudera.sqoop.tool.SqoopTool { public static final String AUTORESET_TO_ONE_MAPPER = "autoreset-to-one-mapper"; + static final String HIVE_IMPORT_WITH_LASTMODIFIED_NOT_SUPPORTED = "--incremental lastmodified option for hive imports is not " + + "supported. Please remove the parameter --incremental lastmodified."; + public BaseSqoopTool() { } @@ -1430,6 +1433,11 @@ public abstract class BaseSqoopTool extends com.cloudera.sqoop.tool.SqoopTool { } if (options.doHiveImport() + && options.getIncrementalMode().equals(IncrementalMode.DateLastModified)) { + throw new InvalidOptionsException(HIVE_IMPORT_WITH_LASTMODIFIED_NOT_SUPPORTED); + } + + if (options.doHiveImport() && options.isAppendMode() && !options.getIncrementalMode().equals(IncrementalMode.AppendRows)) { throw new InvalidOptionsException("Append mode for hive imports is not " http://git-wip-us.apache.org/repos/asf/sqoop/blob/14754342/src/test/org/apache/sqoop/tool/ImportToolValidateOptionsTest.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/tool/ImportToolValidateOptionsTest.java b/src/test/org/apache/sqoop/tool/ImportToolValidateOptionsTest.java new file mode 100644 index 0000000..503a863 --- /dev/null +++ b/src/test/org/apache/sqoop/tool/ImportToolValidateOptionsTest.java @@ -0,0 +1,92 @@ +package org.apache.sqoop.tool; + +import com.cloudera.sqoop.SqoopOptions; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +public class ImportToolValidateOptionsTest { + + private static final String TABLE_NAME = "testTableName"; + private static final String CONNECT_STRING = "testConnectString"; + private static final String CHECK_COLUMN_NAME = "checkColumnName"; + + @Rule + public ExpectedException thrown = ExpectedException.none(); + + private ImportTool importTool; + + @Before + public void setup() { + importTool = new ImportTool(); + importTool.extraArguments = new String[0]; + } + + @Test + public void testValidationFailsWithHiveImportAndIncrementalLastmodified() throws Exception { + SqoopOptions options = buildBaseSqoopOptions(); + options.setHiveImport(true); + options.setIncrementalTestColumn(CHECK_COLUMN_NAME); + options.setIncrementalMode(SqoopOptions.IncrementalMode.DateLastModified); + + thrown.expect(SqoopOptions.InvalidOptionsException.class); + thrown.expectMessage(BaseSqoopTool.HIVE_IMPORT_WITH_LASTMODIFIED_NOT_SUPPORTED); + + importTool.validateOptions(options); + } + + /** + * Note that append mode (--append) is designed to be used with HDFS import and not Hive import. + * However this test case is added to make sure that the error message generated is correct even if --append is used. + * + */ + @Test + public void testValidationFailsWithHiveImportAndAppendModeIncrementalLastmodified() throws Exception { + SqoopOptions options = buildBaseSqoopOptions(); + options.setHiveImport(true); + options.setIncrementalTestColumn(CHECK_COLUMN_NAME); + options.setIncrementalMode(SqoopOptions.IncrementalMode.DateLastModified); + options.setAppendMode(true); + + thrown.expect(SqoopOptions.InvalidOptionsException.class); + thrown.expectMessage(BaseSqoopTool.HIVE_IMPORT_WITH_LASTMODIFIED_NOT_SUPPORTED); + + importTool.validateOptions(options); + } + + @Test + public void testValidationSucceedsWithHiveImportAndIncrementalAppendRows() throws Exception { + SqoopOptions options = buildBaseSqoopOptions(); + options.setHiveImport(true); + options.setIncrementalTestColumn(CHECK_COLUMN_NAME); + options.setIncrementalMode(SqoopOptions.IncrementalMode.AppendRows); + + importTool.validateOptions(options); + } + + /** + * Note that append mode (--append) is designed to be used with HDFS import and not Hive import. + * However this test case is added to make sure that SQOOP-2986 does not break the already existing validation. + * + */ + @Test + public void testValidationSucceedsWithHiveImportAndAppendModeAndIncrementalAppendRows() throws Exception { + SqoopOptions options = buildBaseSqoopOptions(); + options.setHiveImport(true); + options.setIncrementalTestColumn(CHECK_COLUMN_NAME); + options.setIncrementalMode(SqoopOptions.IncrementalMode.AppendRows); + options.setAppendMode(true); + + importTool.validateOptions(options); + } + + private SqoopOptions buildBaseSqoopOptions() { + SqoopOptions result = new SqoopOptions(); + result.setTableName(TABLE_NAME); + result.setConnectString(CONNECT_STRING); + return result; + } + +} +