Repository: incubator-drill Updated Branches: refs/heads/master 27a9c98a5 -> 4198a17a8
DRILL-660: Fix errors when querying a hive table with no data. Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/ce15e931 Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/ce15e931 Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/ce15e931 Branch: refs/heads/master Commit: ce15e931ff0aa1969fa7c75d2ce184ca59458004 Parents: dd650cc Author: vkorukanti <[email protected]> Authored: Tue Jun 10 11:51:41 2014 -0700 Committer: vkorukanti <[email protected]> Committed: Tue Jun 10 22:41:12 2014 -0700 ---------------------------------------------------------------------- .../drill/exec/store/hive/HiveRecordReader.java | 43 ++++++++++++-------- .../exec/store/hive/HiveScanBatchCreator.java | 7 ++++ .../exec/store/hive/HiveTestDataGenerator.java | 2 + .../apache/drill/jdbc/test/TestJdbcQuery.java | 13 +++--- .../apache/drill/jdbc/test/TestMetadataDDL.java | 2 + 5 files changed, 42 insertions(+), 25 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ce15e931/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveRecordReader.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveRecordReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveRecordReader.java index 4361262..edd79e6 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveRecordReader.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveRecordReader.java @@ -108,6 +108,7 @@ public class HiveRecordReader implements RecordReader { protected List<ValueVector> pVectors = Lists.newArrayList(); protected Object redoRecord; List<Object> partitionValues = Lists.newArrayList(); + protected boolean empty; protected static final int TARGET_RECORD_COUNT = 4000; @@ -117,6 +118,7 @@ public class HiveRecordReader implements RecordReader { this.inputSplit = inputSplit; this.context = context; this.columns = columns; + this.empty = (inputSplit == null && partition == null); init(); } @@ -144,11 +146,9 @@ public class HiveRecordReader implements RecordReader { } job.setInputFormat(format.getClass()); - if (partition != null) { - List<FieldSchema> partitionKeys = table.getPartitionKeys(); - for (FieldSchema field : partitionKeys) { - partitionNames.add(field.getName()); - } + List<FieldSchema> partitionKeys = table.getPartitionKeys(); + for (FieldSchema field : partitionKeys) { + partitionNames.add(field.getName()); } try { @@ -168,7 +168,7 @@ public class HiveRecordReader implements RecordReader { for (SchemaPath field : columns) { String columnName = field.getRootSegment().getPath(); //TODO? if (!tableColumns.contains(columnName)) { - if (partition != null && partitionNames.contains(columnName)) { + if (partitionNames.contains(columnName)) { selectedPartitionNames.add(columnName); } else { throw new ExecutionSetupException(String.format("Column %s does not exist", columnName)); @@ -195,11 +195,11 @@ public class HiveRecordReader implements RecordReader { selectedPartitionNames = partitionNames; } - if (partition != null) { - for (int i = 0; i < table.getPartitionKeys().size(); i++) { - FieldSchema field = table.getPartitionKeys().get(i); - if (selectedPartitionNames.contains(field.getName())) { - selectedPartitionTypes.add(field.getType()); + for (int i = 0; i < table.getPartitionKeys().size(); i++) { + FieldSchema field = table.getPartitionKeys().get(i); + if (selectedPartitionNames.contains(field.getName())) { + selectedPartitionTypes.add(field.getType()); + if (partition != null) { partitionValues.add(convertPartitionType(field.getType(), partition.getValues().get(i))); } } @@ -207,13 +207,16 @@ public class HiveRecordReader implements RecordReader { } catch (SerDeException e) { throw new ExecutionSetupException(e); } - try { - reader = format.getRecordReader(inputSplit, job, Reporter.NULL); - } catch (IOException e) { - throw new ExecutionSetupException("Failed to get Recordreader", e); + + if (!empty) { + try { + reader = format.getRecordReader(inputSplit, job, Reporter.NULL); + } catch (IOException e) { + throw new ExecutionSetupException("Failed to get Recordreader", e); + } + key = reader.createKey(); + value = reader.createValue(); } - key = reader.createKey(); - value = reader.createValue(); } @Override @@ -228,7 +231,7 @@ public class HiveRecordReader implements RecordReader { } for (int i = 0; i < selectedPartitionNames.size(); i++) { String type = selectedPartitionTypes.get(i); - MaterializedField field = MaterializedField.create(SchemaPath.getSimplePath(columnNames.get(i)), Types.getMajorTypeFromName(type)); + MaterializedField field = MaterializedField.create(SchemaPath.getSimplePath(selectedPartitionNames.get(i)), Types.getMajorTypeFromName(type)); Class vvClass = TypeHelper.getValueVectorClass(field.getType().getMinorType(), field.getDataMode()); pVectors.add(output.addField(field, vvClass)); } @@ -439,6 +442,10 @@ public class HiveRecordReader implements RecordReader { @Override public int next() { + if (empty) { + return 0; + } + for (ValueVector vv : vectors) { VectorAllocator.getAllocator(vv, 50).alloc(TARGET_RECORD_COUNT); } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ce15e931/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveScanBatchCreator.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveScanBatchCreator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveScanBatchCreator.java index a0837bc..8914db2 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveScanBatchCreator.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveScanBatchCreator.java @@ -66,6 +66,13 @@ public class HiveScanBatchCreator implements BatchCreator<HiveSubScan> { } } } + + // If there are no readers created (which is possible when the table is empty), create an empty RecordReader to + // output the schema + if (readers.size() == 0) { + readers.add(new HiveRecordReader(table, null, null, config.getColumns(), context)); + } + return new ScanBatch(config, context, readers.iterator()); } } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ce15e931/exec/java-exec/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java index f1565d9..5a511c0 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java @@ -85,6 +85,8 @@ public class HiveTestDataGenerator { "ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' STORED AS TEXTFILE"); executeQuery(String.format("LOAD DATA LOCAL INPATH '%s' OVERWRITE INTO TABLE default.foodate", testDateDataFile)); + // create a table with no data + executeQuery("CREATE TABLE IF NOT EXISTS default.empty_table(a INT, b STRING)"); ss.close(); } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ce15e931/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java ---------------------------------------------------------------------- diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java index f486cc9..1b83148 100644 --- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java +++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java @@ -50,7 +50,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; - public class TestJdbcQuery extends JdbcTest{ +public class TestJdbcQuery extends JdbcTest{ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestJdbcQuery.class); @@ -77,12 +77,6 @@ import static org.junit.Assert.fail; } @Test - @Ignore - public void testHiveRead() throws Exception{ - testQuery("select * from hive.kv"); - } - - @Test public void testHiveReadWithDb() throws Exception{ testQuery("select * from hive.`default`.kv"); testQuery("select key from hive.`default`.kv group by key"); @@ -95,6 +89,11 @@ import static org.junit.Assert.fail; } @Test + public void testQueryEmptyHiveTable() throws Exception { + testQuery("SELECT * FROM hive.`default`.empty_table"); + } + + @Test @Ignore public void testJsonQuery() throws Exception{ testQuery("select * from cp.`employee.json`"); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ce15e931/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestMetadataDDL.java ---------------------------------------------------------------------- diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestMetadataDDL.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestMetadataDDL.java index 95af9f8..5299bb5 100644 --- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestMetadataDDL.java +++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestMetadataDDL.java @@ -49,6 +49,7 @@ public class TestMetadataDDL extends TestJdbcQuery { JdbcAssert.withFull("hive.default") .sql("SHOW TABLES") .returns( + "TABLE_SCHEMA=hive.default; TABLE_NAME=empty_table\n" + "TABLE_SCHEMA=hive.default; TABLE_NAME=kv\n" + "TABLE_SCHEMA=hive.default; TABLE_NAME=foodate\n" ); @@ -69,6 +70,7 @@ public class TestMetadataDDL extends TestJdbcQuery { JdbcAssert.withFull("dfs.tmp") .sql("SHOW TABLES IN hive.`default`") .returns( + "TABLE_SCHEMA=hive.default; TABLE_NAME=empty_table\n" + "TABLE_SCHEMA=hive.default; TABLE_NAME=kv\n" + "TABLE_SCHEMA=hive.default; TABLE_NAME=foodate\n"); }
