Repository: sqoop Updated Branches: refs/heads/trunk b007e4d59 -> 8f3989d9a
SQOOP-3010: Sqoop should not allow --as-parquetfile with hcatalog jobs or when hive import with create-hive-table is used (Sowmya Ramesh via Venkat Ranganathan) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/8f3989d9 Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/8f3989d9 Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/8f3989d9 Branch: refs/heads/trunk Commit: 8f3989d9ad32167a2d259377f5634b2ee32ee6ed Parents: b007e4d Author: Venkatesan Ranganathan <[email protected]> Authored: Mon Sep 26 05:34:13 2016 -0700 Committer: Venkatesan Ranganathan <[email protected]> Committed: Mon Sep 26 05:34:13 2016 -0700 ---------------------------------------------------------------------- .../org/apache/sqoop/tool/BaseSqoopTool.java | 14 +++++++ .../com/cloudera/sqoop/hive/TestHiveImport.java | 21 ++++++++++ .../apache/sqoop/hcat/TestHCatalogBasic.java | 40 ++++++++++++++++++++ 3 files changed, 75 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/8f3989d9/src/java/org/apache/sqoop/tool/BaseSqoopTool.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/tool/BaseSqoopTool.java b/src/java/org/apache/sqoop/tool/BaseSqoopTool.java index fecdf43..b71bc5e 100644 --- a/src/java/org/apache/sqoop/tool/BaseSqoopTool.java +++ b/src/java/org/apache/sqoop/tool/BaseSqoopTool.java @@ -1421,6 +1421,14 @@ public abstract class BaseSqoopTool extends com.cloudera.sqoop.tool.SqoopTool { + "importing into SequenceFile format."); } + // Hive import and create hive table not compatible for ParquetFile format + if (options.doHiveImport() + && options.doFailIfHiveTableExists() + && options.getFileLayout() == SqoopOptions.FileLayout.ParquetFile) { + throw new InvalidOptionsException("Hive import and create hive table is not compatible with " + + "importing into ParquetFile format."); + } + if (options.doHiveImport() && options.isAppendMode() && !options.getIncrementalMode().equals(IncrementalMode.AppendRows)) { @@ -1598,6 +1606,12 @@ public abstract class BaseSqoopTool extends com.cloudera.sqoop.tool.SqoopTool { + " option." + HELP_STR); } + if (options.getFileLayout() == SqoopOptions.FileLayout.ParquetFile) { + throw new InvalidOptionsException("HCatalog job is not compatible with " + + "SequenceFile format option " + FMT_PARQUETFILE_ARG + + " option." + HELP_STR); + } + if (options.getHCatalogPartitionKeys() != null && options.getHCatalogPartitionValues() == null) { throw new InvalidOptionsException("Either both --hcatalog-partition-keys" http://git-wip-us.apache.org/repos/asf/sqoop/blob/8f3989d9/src/test/com/cloudera/sqoop/hive/TestHiveImport.java ---------------------------------------------------------------------- diff --git a/src/test/com/cloudera/sqoop/hive/TestHiveImport.java b/src/test/com/cloudera/sqoop/hive/TestHiveImport.java index b626964..26d087b 100644 --- a/src/test/com/cloudera/sqoop/hive/TestHiveImport.java +++ b/src/test/com/cloudera/sqoop/hive/TestHiveImport.java @@ -402,6 +402,27 @@ public class TestHiveImport extends ImportJobTestCase { {"test2", 4242, "somestring2"}, {"test", 42, "somestring"}}); } + /** + * Test hive create and --as-parquetfile options validation. + */ + @Test + public void testCreateHiveImportAsParquet() throws ParseException { + final String TABLE_NAME = "CREATE_HIVE_IMPORT_AS_PARQUET"; + setCurTableName(TABLE_NAME); + setNumCols(3); + String [] extraArgs = {"--as-parquetfile", "--create-hive-table"}; + ImportTool tool = new ImportTool(); + + try { + tool.validateOptions(tool.parseArguments(getArgv(false, extraArgs), null, + null, true)); + fail("Expected InvalidOptionsException"); + } catch (InvalidOptionsException ex) { + /* success */ + } + } + + /** Test that dates are coerced properly to strings. */ @Test public void testDate() throws IOException { http://git-wip-us.apache.org/repos/asf/sqoop/blob/8f3989d9/src/test/org/apache/sqoop/hcat/TestHCatalogBasic.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/hcat/TestHCatalogBasic.java b/src/test/org/apache/sqoop/hcat/TestHCatalogBasic.java index 5cd4c26..54b4552 100644 --- a/src/test/org/apache/sqoop/hcat/TestHCatalogBasic.java +++ b/src/test/org/apache/sqoop/hcat/TestHCatalogBasic.java @@ -164,6 +164,24 @@ public class TestHCatalogBasic extends TestCase { } } + public void testHCatExportWithParquetFile() throws Exception { + String[] args = { + "--connect", + "jdbc:db:url", + "--table", + "dbtable", + "--hcatalog-table", + "table", + "--as-parquetfile", + }; + try { + SqoopOptions opts = parseExportArgs(args); + exportTool.validateOptions(opts); + fail("Expected InvalidOptionsException"); + } catch (SqoopOptions.InvalidOptionsException ioe) { + // expected. + } + } public void testHCatImportWithSequenceFile() throws Exception { String[] args = { @@ -184,6 +202,28 @@ public class TestHCatalogBasic extends TestCase { } } + public void testHCatImportWithParquetFile() throws Exception { + String[] args = { + "--hcatalog-table", + "table", + "--create-hcatalog-table", + "--connect", + "jdbc:db:url", + "--table", + "dbtable", + "--hcatalog-table", + "table", + "--as-parquetfile", + }; + try { + SqoopOptions opts = parseImportArgs(args); + importTool.validateOptions(opts); + fail("Expected InvalidOptionsException"); + } catch (SqoopOptions.InvalidOptionsException ioe) { + // expected. + } + } + public void testHCatImportWithAvroFile() throws Exception { String[] args = { "--connect",
