DRILL-1053: Check file if it exists on filesystem before adding it to FileInputFormat
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/718f4a27 Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/718f4a27 Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/718f4a27 Branch: refs/heads/master Commit: 718f4a27bd6d89b221b77491a57646dce3719879 Parents: 8a22910 Author: vkorukanti <venki.koruka...@gmail.com> Authored: Fri Jun 20 19:06:27 2014 -0700 Committer: Jacques Nadeau <jacq...@apache.org> Committed: Wed Jun 25 16:16:05 2014 -0700 ---------------------------------------------------------------------- .../apache/drill/exec/store/hive/HiveScan.java | 29 ++++++++++++-------- .../exec/store/hive/HiveTestDataGenerator.java | 5 ++++ 2 files changed, 23 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/718f4a27/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveScan.java ---------------------------------------------------------------------- diff --git a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveScan.java b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveScan.java index 2f217d9..8c02eb3 100644 --- a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveScan.java +++ b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveScan.java @@ -39,7 +39,7 @@ import org.apache.drill.exec.physical.base.ScanStats.GroupScanProperty; import org.apache.drill.exec.proto.CoordinationProtos; import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint; import org.apache.drill.exec.store.StoragePluginRegistry; -import org.apache.drill.exec.store.schedule.CompleteFileWork; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.metastore.MetaStoreUtils; import org.apache.hadoop.hive.metastore.api.Partition; @@ -140,10 +140,13 @@ public class HiveScan extends AbstractGroupScan { InputFormat<?, ?> format = (InputFormat<?, ?>) Class.forName(table.getSd().getInputFormat()).getConstructor().newInstance(); job.setInputFormat(format.getClass()); Path path = new Path(table.getSd().getLocation()); - FileInputFormat.addInputPath(job, path); - format = job.getInputFormat(); - for (InputSplit split : format.getSplits(job, 1)) { - inputSplits.add(split); + FileSystem fs = FileSystem.get(job); + if (fs.exists(path)) { + FileInputFormat.addInputPath(job, path); + format = job.getInputFormat(); + for (InputSplit split : format.getSplits(job, 1)) { + inputSplits.add(split); + } } for (InputSplit split : inputSplits) { partitionMap.put(split, null); @@ -157,12 +160,16 @@ public class HiveScan extends AbstractGroupScan { } InputFormat<?, ?> format = (InputFormat<?, ?>) Class.forName(partition.getSd().getInputFormat()).getConstructor().newInstance(); job.setInputFormat(format.getClass()); - FileInputFormat.addInputPath(job, new Path(partition.getSd().getLocation())); - format = job.getInputFormat(); - InputSplit[] splits = format.getSplits(job,1); - for (InputSplit split : splits) { - inputSplits.add(split); - partitionMap.put(split, partition); + Path path = new Path(partition.getSd().getLocation()); + FileSystem fs = FileSystem.get(job); + if (fs.exists(path)) { + FileInputFormat.addInputPath(job, path); + format = job.getInputFormat(); + InputSplit[] splits = format.getSplits(job, 1); + for (InputSplit split : splits) { + inputSplits.add(split); + partitionMap.put(split, partition); + } } } } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/718f4a27/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java ---------------------------------------------------------------------- diff --git a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java index e051abb..3fe36f5 100644 --- a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java +++ b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java @@ -87,6 +87,11 @@ public class HiveTestDataGenerator { // create a table with no data executeQuery("CREATE TABLE IF NOT EXISTS default.empty_table(a INT, b STRING)"); + // delete the table location of empty table + File emptyTableLocation = new File(WH_DIR + "/empty_table"); + if (emptyTableLocation.exists()) { + FileUtils.forceDelete(emptyTableLocation); + } // create a Hive table that has columns with data types which are supported for reading in Drill. testDataFile = generateAllTypesDataFile();
