Repository: sqoop Updated Branches: refs/heads/trunk fdcbaf5c5 -> ae9ab0fa3
SQOOP-2582. Query import won't work for parquet (Jarcec via Hari) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/ae9ab0fa Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/ae9ab0fa Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/ae9ab0fa Branch: refs/heads/trunk Commit: ae9ab0fa37efb405b349e2c91050479ef4355526 Parents: fdcbaf5 Author: Hari Shreedharan <[email protected]> Authored: Thu Sep 24 14:40:33 2015 -0700 Committer: Hari Shreedharan <[email protected]> Committed: Thu Sep 24 14:40:33 2015 -0700 ---------------------------------------------------------------------- src/java/org/apache/sqoop/tool/CodeGenTool.java | 2 +- .../com/cloudera/sqoop/TestParquetImport.java | 43 ++++++++++++++++++++ 2 files changed, 44 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/ae9ab0fa/src/java/org/apache/sqoop/tool/CodeGenTool.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/tool/CodeGenTool.java b/src/java/org/apache/sqoop/tool/CodeGenTool.java index 22ab030..b3107a2 100644 --- a/src/java/org/apache/sqoop/tool/CodeGenTool.java +++ b/src/java/org/apache/sqoop/tool/CodeGenTool.java @@ -91,7 +91,7 @@ public class CodeGenTool extends com.cloudera.sqoop.tool.BaseSqoopTool { } LOG.info("Beginning code generation"); - if (options.getFileLayout() == SqoopOptions.FileLayout.ParquetFile) { + if (tableName != null && options.getFileLayout() == SqoopOptions.FileLayout.ParquetFile) { String className = options.getClassName() != null ? options.getClassName() : tableName; if (className.equalsIgnoreCase(tableName)) { http://git-wip-us.apache.org/repos/asf/sqoop/blob/ae9ab0fa/src/test/com/cloudera/sqoop/TestParquetImport.java ---------------------------------------------------------------------- diff --git a/src/test/com/cloudera/sqoop/TestParquetImport.java b/src/test/com/cloudera/sqoop/TestParquetImport.java index ae2e617..43995c1 100644 --- a/src/test/com/cloudera/sqoop/TestParquetImport.java +++ b/src/test/com/cloudera/sqoop/TestParquetImport.java @@ -78,6 +78,31 @@ public class TestParquetImport extends ImportJobTestCase { return args.toArray(new String[args.size()]); } + protected String[] getOutputQueryArgv(boolean includeHadoopFlags, String[] extraArgs) { + ArrayList<String> args = new ArrayList<String>(); + + if (includeHadoopFlags) { + CommonArgs.addHadoopFlags(args); + } + + args.add("--query"); + args.add("SELECT * FROM " + getTableName() + " WHERE $CONDITIONS"); + args.add("--connect"); + args.add(HsqldbTestServer.getUrl()); + args.add("--target-dir"); + args.add(getWarehouseDir() + "/" + getTableName()); + args.add("--m"); + args.add("1"); + args.add("--split-by"); + args.add("INTFIELD1"); + args.add("--as-parquetfile"); + if (extraArgs != null) { + args.addAll(Arrays.asList(extraArgs)); + } + + return args.toArray(new String[args.size()]); + } + public void testSnappyCompression() throws IOException { runParquetImportTest("snappy"); } @@ -198,6 +223,24 @@ public class TestParquetImport extends ImportJobTestCase { } } + public void testQueryImport() throws IOException, SQLException { + String [] types = { "INT" }; + String [] vals = { "1" }; + createTableWithColTypes(types, vals); + + runImport(getOutputQueryArgv(true, null)); + + DatasetReader<GenericRecord> reader = getReader(); + try { + assertTrue(reader.hasNext()); + GenericRecord record1 = reader.next(); + assertEquals(1, record1.get("DATA_COL0")); + assertFalse(reader.hasNext()); + } finally { + reader.close(); + } + } + public void testIncrementalParquetImport() throws IOException, SQLException { String [] types = { "INT" }; String [] vals = { "1" };
