Updated Branches: refs/heads/trunk 8d3454dc6 -> 6523ebb17
SQOOP-1249: Sqoop HCatalog Import fails with -queries because of validation issues (Venkat Ranganathan via Jarek Jarcec Cecho) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/6523ebb1 Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/6523ebb1 Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/6523ebb1 Branch: refs/heads/trunk Commit: 6523ebb176a8780eb88ba4f6556a7904c1e622ff Parents: 8d3454d Author: Jarek Jarcec Cecho <[email protected]> Authored: Sat Dec 7 10:27:36 2013 -0800 Committer: Jarek Jarcec Cecho <[email protected]> Committed: Sat Dec 7 10:27:36 2013 -0800 ---------------------------------------------------------------------- src/java/org/apache/sqoop/tool/ImportTool.java | 3 +- .../apache/sqoop/hcat/HCatalogImportTest.java | 89 ++++++++++++++++++-- 2 files changed, 82 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/6523ebb1/src/java/org/apache/sqoop/tool/ImportTool.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/tool/ImportTool.java b/src/java/org/apache/sqoop/tool/ImportTool.java index 50826e9..59279b6 100644 --- a/src/java/org/apache/sqoop/tool/ImportTool.java +++ b/src/java/org/apache/sqoop/tool/ImportTool.java @@ -894,9 +894,10 @@ public class ImportTool extends com.cloudera.sqoop.tool.BaseSqoopTool { } else if (options.getSqlQuery() != null && options.getTargetDir() == null && options.getHBaseTable() == null + && options.getHCatTableName() == null && options.getAccumuloTable() == null) { throw new InvalidOptionsException( - "Must specify destination with --target-dir." + "Must specify destination with --target-dir. " + HELP_STR); } else if (options.getSqlQuery() != null && options.doHiveImport() && options.getHiveTableName() == null) { http://git-wip-us.apache.org/repos/asf/sqoop/blob/6523ebb1/src/test/org/apache/sqoop/hcat/HCatalogImportTest.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/hcat/HCatalogImportTest.java b/src/test/org/apache/sqoop/hcat/HCatalogImportTest.java index 13850fc..ab08013 100644 --- a/src/test/org/apache/sqoop/hcat/HCatalogImportTest.java +++ b/src/test/org/apache/sqoop/hcat/HCatalogImportTest.java @@ -173,6 +173,48 @@ public class HCatalogImportTest extends ImportJobTestCase { return args.toArray(new String[0]); } + protected String[] getQueryArgv(boolean includeHadoopFlags, String[] colNames, + Configuration conf) { + + String columnsString = ""; + String splitByCol = null; + if (colNames != null) { + splitByCol = colNames[0]; + columnsString = colNames[0]; + for (int c = 1; c < colNames.length; ++c) { + columnsString += "," + colNames[c]; + } + } + ArrayList<String> args = new ArrayList<String>(); + + if (includeHadoopFlags) { + CommonArgs.addHadoopFlags(args); + } + args.addAll(getConfigParams()); + args.add("--query"); + StringBuilder query = new StringBuilder("select "); + if (colNames != null) { + query.append(columnsString); + } else { + query.append('*'); + } + query.append(' '); + query.append("from ").append(getTableName()); + query.append(" where $CONDITIONS"); + args.add(query.toString()); + if (colNames != null) { + args.add("--split-by"); + args.add(splitByCol); + } + args.add("--hcatalog-table"); + args.add(getTableName()); + args.add("--connect"); + args.add(getConnectString()); + args.addAll(getExtraArgs(conf)); + + return args.toArray(new String[0]); + } + private void validateHCatRecords(final List<HCatRecord> recs, final HCatSchema schema, int expectedCount, ColumnGenerator... cols) throws IOException { @@ -242,12 +284,20 @@ public class HCatalogImportTest extends ImportJobTestCase { protected void runHCatImport(List<String> addlArgsArray, int totalRecords, String table, ColumnGenerator[] cols, String[] cNames) throws Exception { - runHCatImport(addlArgsArray, totalRecords, table, cols, cNames, false); + runHCatImport(addlArgsArray, totalRecords, table, cols, + cNames, false, false); + } + + protected void runHCatQueryImport(List<String> addlArgsArray, + int totalRecords, String table, ColumnGenerator[] cols, + String[] cNames) throws Exception { + runHCatImport(addlArgsArray, totalRecords, table, cols, + cNames, false, true); } protected void runHCatImport(List<String> addlArgsArray, int totalRecords, String table, ColumnGenerator[] cols, - String[] cNames, boolean dontCreate) throws Exception { + String[] cNames, boolean dontCreate, boolean isQuery) throws Exception { CreateMode mode = CreateMode.CREATE; if (dontCreate) { mode = CreateMode.NO_CREATION; @@ -255,8 +305,6 @@ public class HCatalogImportTest extends ImportJobTestCase { HCatSchema tblSchema = utils.createHCatTable(mode, totalRecords, table, cols); utils.createSqlTable(getConnection(), false, totalRecords, table, cols); - Map<String, String> addlArgsMap = utils.getAddlTestArgs(); - String[] argv = {}; addlArgsArray.add("-m"); addlArgsArray.add("1"); addlArgsArray.add("--hcatalog-table"); @@ -272,7 +320,12 @@ public class HCatalogImportTest extends ImportJobTestCase { colNames[2 + i] = cols[i].getName().toUpperCase(); } } - String[] importArgs = getArgv(true, colNames, new Configuration()); + String[] importArgs; + if (isQuery) { + importArgs = getQueryArgv(true, colNames, new Configuration()); + } else { + importArgs = getArgv(true, colNames, new Configuration()); + } LOG.debug("Import args = " + Arrays.toString(importArgs)); SqoopHCatUtilities.instance().setConfigured(false); runImport(new ImportTool(), importArgs); @@ -569,7 +622,8 @@ public class HCatalogImportTest extends ImportJobTestCase { List<String> addlArgsArray = new ArrayList<String>(); addlArgsArray.add("--create-hcatalog-table"); setExtraArgs(addlArgsArray); - runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null, true); + runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, + null, true, false); } public void testTableCreationWithPartition() throws Exception { @@ -590,7 +644,7 @@ public class HCatalogImportTest extends ImportJobTestCase { addlArgsArray.add("2"); addlArgsArray.add("--create-hcatalog-table"); setExtraArgs(addlArgsArray); - runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null, true); + runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null, true, false); } public void testTableCreationWithStorageStanza() throws Exception { @@ -613,7 +667,7 @@ public class HCatalogImportTest extends ImportJobTestCase { addlArgsArray.add("--hcatalog-storage-stanza"); addlArgsArray.add(HCatalogTestUtils.STORED_AS_TEXT); setExtraArgs(addlArgsArray); - runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null, true); + runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null, true, false); } public void testHiveDropDelims() throws Exception { @@ -667,6 +721,22 @@ public class HCatalogImportTest extends ImportJobTestCase { runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null); } + public void testQueryImport() throws Exception { + final int TOTAL_RECORDS = 1 * 10; + String table = getTableName().toUpperCase(); + ColumnGenerator[] cols = new ColumnGenerator[] { + HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(0), + "varchar(20)", Types.VARCHAR, HCatFieldSchema.Type.STRING, "1", + "1", KeyType.NOT_A_KEY), + HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(1), + "varchar(20)", Types.VARCHAR, HCatFieldSchema.Type.STRING, "2", + "2", KeyType.DYNAMIC_KEY), }; + List<String> addlArgsArray = new ArrayList<String>(); + setExtraArgs(addlArgsArray); + + runHCatQueryImport(addlArgsArray, TOTAL_RECORDS, table, cols, null); + } + public void testCreateTableWithPreExistingTable() throws Exception { final int TOTAL_RECORDS = 1 * 10; String table = getTableName().toUpperCase(); @@ -683,7 +753,8 @@ public class HCatalogImportTest extends ImportJobTestCase { try { // Precreate table utils.createHCatTable(CreateMode.CREATE, TOTAL_RECORDS, table, cols); - runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null, true); + runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, + null, true, false); fail("HCatalog job with --create-hcatalog-table and pre-existing" + " table should fail"); } catch (Exception e) {
