Repository: sqoop Updated Branches: refs/heads/trunk c2351511e -> 2eecf6801
SQOOP-2647: Add option for drop-if-exists when using sqoop hcat import. (Muhammad Ehsan ul Haque via Jarek Jarcec Cecho) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/2eecf680 Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/2eecf680 Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/2eecf680 Branch: refs/heads/trunk Commit: 2eecf68010df83b49eb3965a9a99de0cfbfb5828 Parents: c235151 Author: Jarek Jarcec Cecho <[email protected]> Authored: Fri Mar 4 08:50:15 2016 -0800 Committer: Jarek Jarcec Cecho <[email protected]> Committed: Fri Mar 4 08:50:15 2016 -0800 ---------------------------------------------------------------------- src/docs/user/hcatalog.txt | 5 +++ src/java/org/apache/sqoop/SqoopOptions.java | 10 ++++++ .../mapreduce/hcat/SqoopHCatUtilities.java | 14 ++++++-- .../org/apache/sqoop/tool/BaseSqoopTool.java | 16 +++++++++ .../apache/sqoop/hcat/HCatalogImportTest.java | 38 ++++++++++++++++++++ .../apache/sqoop/hcat/TestHCatalogBasic.java | 35 ++++++++++++++++++ 6 files changed, 116 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/2eecf680/src/docs/user/hcatalog.txt ---------------------------------------------------------------------- diff --git a/src/docs/user/hcatalog.txt b/src/docs/user/hcatalog.txt index 99ae4f5..2ae1d54 100644 --- a/src/docs/user/hcatalog.txt +++ b/src/docs/user/hcatalog.txt @@ -79,6 +79,11 @@ to exist. The table name will be the same as the database table name translated to lower case. Further described in +Automatic Table Creation+ below. ++--drop-and-create-hcatalog-table+:: + +Same as +--create-hcatalog-table+, but does a +drop if exists+ before creating +the table. + +--hcatalog-storage-stanza+:: This option specifies the storage stanza to be appended to the table. http://git-wip-us.apache.org/repos/asf/sqoop/blob/2eecf680/src/java/org/apache/sqoop/SqoopOptions.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/SqoopOptions.java b/src/java/org/apache/sqoop/SqoopOptions.java index 4d30007..17751c7 100644 --- a/src/java/org/apache/sqoop/SqoopOptions.java +++ b/src/java/org/apache/sqoop/SqoopOptions.java @@ -173,6 +173,8 @@ public class SqoopOptions implements Cloneable { private String hCatDatabaseName; @StoredAsProperty("hcatalog.create.table") private boolean hCatCreateTable; + @StoredAsProperty("hcatalog.drop.and.create.table") + private boolean hCatDropAndCreateTable; @StoredAsProperty("hcatalog.storage.stanza") private String hCatStorageStanza; private String hCatHome; // not serialized to metastore. @@ -1477,6 +1479,14 @@ public class SqoopOptions implements Cloneable { this.hCatCreateTable = create; } + public boolean doDropAndCreateHCatalogTable() { + return hCatDropAndCreateTable; + } + + public void setDropAndCreateHCatalogTable(boolean dropAndCreate) { + this.hCatDropAndCreateTable = dropAndCreate; + } + public void setHCatStorageStanza(String stanza) { this.hCatStorageStanza = stanza; } http://git-wip-us.apache.org/repos/asf/sqoop/blob/2eecf680/src/java/org/apache/sqoop/mapreduce/hcat/SqoopHCatUtilities.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/mapreduce/hcat/SqoopHCatUtilities.java b/src/java/org/apache/sqoop/mapreduce/hcat/SqoopHCatUtilities.java index 9d5a9ee..ebe7209 100644 --- a/src/java/org/apache/sqoop/mapreduce/hcat/SqoopHCatUtilities.java +++ b/src/java/org/apache/sqoop/mapreduce/hcat/SqoopHCatUtilities.java @@ -66,6 +66,7 @@ import org.apache.sqoop.config.ConfigurationConstants; import org.apache.sqoop.config.ConfigurationHelper; import org.apache.sqoop.hive.HiveTypes; import org.apache.sqoop.manager.ConnManager; +import org.apache.sqoop.tool.BaseSqoopTool; import org.apache.sqoop.util.Executor; import org.apache.sqoop.util.LoggingAsyncSink; import org.apache.sqoop.util.SubprocessSecurityManager; @@ -337,7 +338,11 @@ public final class SqoopHCatUtilities { if (options.doCreateHCatalogTable()) { LOG.info("Creating HCatalog table " + hCatQualifiedTableName + " for import"); - createHCatTable(); + createHCatTable(false); + } else if (options.doDropAndCreateHCatalogTable()) { + LOG.info("Dropping and Creating HCatalog table " + + hCatQualifiedTableName + " for import"); + createHCatTable(true); } // For serializing the schema to conf HCatInputFormat hif = HCatInputFormat.setInput(hCatJob, hCatDatabaseName, @@ -555,8 +560,13 @@ public final class SqoopHCatUtilities { return sb; } - private void createHCatTable() throws IOException { + private void createHCatTable(boolean dropIfExists) throws IOException { StringBuilder sb = new StringBuilder(); + if (dropIfExists) { + sb.append("drop table "). + append(escHCatObj(hCatDatabaseName)).append('.'). + append(escHCatObj(hCatTableName)).append(";\n"); + } sb.append("create table "). append(escHCatObj(hCatDatabaseName)).append('.'); sb.append(escHCatObj(hCatTableName)).append(" (\n\t"); http://git-wip-us.apache.org/repos/asf/sqoop/blob/2eecf680/src/java/org/apache/sqoop/tool/BaseSqoopTool.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/tool/BaseSqoopTool.java b/src/java/org/apache/sqoop/tool/BaseSqoopTool.java index 9c810a9..50dd67d 100644 --- a/src/java/org/apache/sqoop/tool/BaseSqoopTool.java +++ b/src/java/org/apache/sqoop/tool/BaseSqoopTool.java @@ -122,6 +122,8 @@ public abstract class BaseSqoopTool extends com.cloudera.sqoop.tool.SqoopTool { public static final String HCATALOG_DATABASE_ARG = "hcatalog-database"; public static final String CREATE_HCATALOG_TABLE_ARG = "create-hcatalog-table"; + public static final String DROP_AND_CREATE_HCATALOG_TABLE = + "drop-and-create-hcatalog-table"; public static final String HCATALOG_STORAGE_STANZA_ARG = "hcatalog-storage-stanza"; public static final String HCATALOG_HOME_ARG = "hcatalog-home"; @@ -606,6 +608,10 @@ public abstract class BaseSqoopTool extends com.cloudera.sqoop.tool.SqoopTool { .withLongOpt(CREATE_HCATALOG_TABLE_ARG) .create()); hCatOptions.addOption(OptionBuilder + .withDescription("Drop and Create HCatalog before import") + .withLongOpt(DROP_AND_CREATE_HCATALOG_TABLE) + .create()); + hCatOptions.addOption(OptionBuilder .hasArg() .withDescription("HCatalog storage stanza for table creation") .withLongOpt(HCATALOG_STORAGE_STANZA_ARG) @@ -1140,6 +1146,10 @@ public abstract class BaseSqoopTool extends com.cloudera.sqoop.tool.SqoopTool { out.setCreateHCatalogTable(true); } + if (in.hasOption(DROP_AND_CREATE_HCATALOG_TABLE)) { + out.setDropAndCreateHCatalogTable(true); + } + if (in.hasOption(HCATALOG_HOME_ARG)) { out.setHCatHome(in.getOptionValue(HCATALOG_HOME_ARG)); } @@ -1621,6 +1631,12 @@ public abstract class BaseSqoopTool extends com.cloudera.sqoop.tool.SqoopTool { + "these options should be omitted"); } } + if (options.doCreateHCatalogTable() && + options.doDropAndCreateHCatalogTable()) { + throw new InvalidOptionsException("Options --create-hcatalog-table" + + " and --drop-and-create-hcatalog-table are mutually exclusive." + + " Use any one of them"); + } } protected void validateHBaseOptions(SqoopOptions options) http://git-wip-us.apache.org/repos/asf/sqoop/blob/2eecf680/src/test/org/apache/sqoop/hcat/HCatalogImportTest.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/hcat/HCatalogImportTest.java b/src/test/org/apache/sqoop/hcat/HCatalogImportTest.java index 8db06d1..67b7a78 100644 --- a/src/test/org/apache/sqoop/hcat/HCatalogImportTest.java +++ b/src/test/org/apache/sqoop/hcat/HCatalogImportTest.java @@ -854,6 +854,44 @@ public class HCatalogImportTest extends ImportJobTestCase { + " create-hcatalog-table with pre-existing table test", e); } } + + public void testDropAndCreateWithPreExistingTable() throws Exception { + final int TOTAL_RECORDS = 1 * 10; + String table = getTableName().toUpperCase(); + ColumnGenerator[] cols = new ColumnGenerator[] { + HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(0), + "varchar(20)", Types.VARCHAR, HCatFieldSchema.Type.STRING, 0, 0, + new HiveVarchar("1", 20), "1", KeyType.STATIC_KEY), + HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(1), + "varchar(20)", Types.VARCHAR, HCatFieldSchema.Type.STRING, 0, 0, + new HiveVarchar("2", 20), "2", KeyType.DYNAMIC_KEY), }; + List<String> addlArgsArray = new ArrayList<String>(); + addlArgsArray.add("--drop-and-create-hcatalog-table"); + setExtraArgs(addlArgsArray); + // Precreate table + utils.createHCatTable(CreateMode.CREATE, TOTAL_RECORDS, table, cols); + runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, + null, true, false); + } + + public void testDropAndCreateWithoutPreExistingTable() throws Exception { + final int TOTAL_RECORDS = 1 * 10; + String table = getTableName().toUpperCase(); + ColumnGenerator[] cols = new ColumnGenerator[] { + HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(0), + "varchar(20)", Types.VARCHAR, HCatFieldSchema.Type.STRING, 0, 0, + new HiveVarchar("1", 20), "1", KeyType.STATIC_KEY), + HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(1), + "varchar(20)", Types.VARCHAR, HCatFieldSchema.Type.STRING, 0, 0, + new HiveVarchar("2", 20), "2", KeyType.DYNAMIC_KEY), + }; + List<String> addlArgsArray = new ArrayList<String>(); + addlArgsArray.add("--drop-and-create-hcatalog-table"); + setExtraArgs(addlArgsArray); + runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, + null, true, false); + } + public void testTableWithNonIdentColChars() throws Exception { final int TOTAL_RECORDS = 1 * 10; String table = getTableName().toUpperCase(); http://git-wip-us.apache.org/repos/asf/sqoop/blob/2eecf680/src/test/org/apache/sqoop/hcat/TestHCatalogBasic.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/hcat/TestHCatalogBasic.java b/src/test/org/apache/sqoop/hcat/TestHCatalogBasic.java index d33d456..5cd4c26 100644 --- a/src/test/org/apache/sqoop/hcat/TestHCatalogBasic.java +++ b/src/test/org/apache/sqoop/hcat/TestHCatalogBasic.java @@ -211,6 +211,41 @@ public class TestHCatalogBasic extends TestCase { SqoopOptions opts = parseImportArgs(args); } + public void testHCatImportWithDropAndCreateTable() throws Exception { + String[] args = { + "--connect", + "jdbc:db:url", + "--table", + "dbtable", + "--hcatalog-table", + "table", + "--drop-and-create-hcatalog-table", + }; + SqoopOptions opts = parseImportArgs(args); + importTool.validateOptions(opts); + } + + public void testHCatImportWithCreateTableAndDropAndCreateTable() + throws Exception { + String[] args = { + "--connect", + "jdbc:db:url", + "--table", + "dbtable", + "--hcatalog-table", + "table", + "--create-hcatalog-table", + "--drop-and-create-hcatalog-table", + }; + SqoopOptions opts = parseImportArgs(args); + try { + importTool.validateOptions(opts); + fail("Expected InvalidOptionsException"); + } catch (SqoopOptions.InvalidOptionsException ioe) { + // expected. + } + } + public void testHCatImportWithStorageStanza() throws Exception { String[] args = { "--hcatalog-table",
