Repository: sqoop Updated Branches: refs/heads/trunk 967f0eb0f -> 02e36db2b
SQOOP-1281: Support of glob paths during export (Clément MAHTIEU via Jarek Jarcec Cecho) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/02e36db2 Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/02e36db2 Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/02e36db2 Branch: refs/heads/trunk Commit: 02e36db2b8deee01ae08a493369097b6812a164e Parents: 967f0eb Author: Jarek Jarcec Cecho <[email protected]> Authored: Tue Oct 6 08:51:20 2015 -0700 Committer: Jarek Jarcec Cecho <[email protected]> Committed: Tue Oct 6 08:51:20 2015 -0700 ---------------------------------------------------------------------- .../apache/sqoop/mapreduce/ExportJobBase.java | 16 ++++++++++++---- src/test/com/cloudera/sqoop/TestAvroExport.java | 20 ++++++++++++++++++++ 2 files changed, 32 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/02e36db2/src/java/org/apache/sqoop/mapreduce/ExportJobBase.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/mapreduce/ExportJobBase.java b/src/java/org/apache/sqoop/mapreduce/ExportJobBase.java index f9fa7f3..068b8f7 100644 --- a/src/java/org/apache/sqoop/mapreduce/ExportJobBase.java +++ b/src/java/org/apache/sqoop/mapreduce/ExportJobBase.java @@ -127,18 +127,26 @@ public class ExportJobBase extends JobBase { FileSystem fs = p.getFileSystem(conf); try { - FileStatus stat = fs.getFileStatus(p); + FileStatus[] fileStatuses = fs.globStatus(p); - if (null == stat) { + if (null == fileStatuses) { // Couldn't get the item. LOG.warn("Input path " + p + " does not exist"); return FileType.UNKNOWN; } + if (fileStatuses.length == 0) { + LOG.warn("Input path " + p + " does not match any file"); + return FileType.UNKNOWN; + } + + FileStatus stat = fileStatuses[0]; + if (stat.isDir()) { - FileStatus [] subitems = fs.listStatus(p); + Path dir = stat.getPath(); + FileStatus [] subitems = fs.listStatus(dir); if (subitems == null || subitems.length == 0) { - LOG.warn("Input path " + p + " contains no files"); + LOG.warn("Input path " + dir + " contains no files"); return FileType.UNKNOWN; // empty dir. } http://git-wip-us.apache.org/repos/asf/sqoop/blob/02e36db2/src/test/com/cloudera/sqoop/TestAvroExport.java ---------------------------------------------------------------------- diff --git a/src/test/com/cloudera/sqoop/TestAvroExport.java b/src/test/com/cloudera/sqoop/TestAvroExport.java index 1d8f5df..5303048 100644 --- a/src/test/com/cloudera/sqoop/TestAvroExport.java +++ b/src/test/com/cloudera/sqoop/TestAvroExport.java @@ -333,6 +333,26 @@ public class TestAvroExport extends ExportJobTestCase { } } + public void testPathPatternInExportDir() throws IOException, SQLException { + final int TOTAL_RECORDS = 10; + + ColumnGenerator[] gens = new ColumnGenerator[] { + colGenerator(true, Schema.create(Schema.Type.BOOLEAN), true, "BIT"), + }; + + createAvroFile(0, TOTAL_RECORDS, gens); + createTable(gens); + + // Converts path to an unary set while preserving the leading '/' + String pathPattern = new StringBuilder(getTablePath().toString()) + .insert(1, "{") + .append("}") + .toString(); + + runExport(getArgv(true, 10, 10, "--export-dir", pathPattern)); + verifyExport(TOTAL_RECORDS); + } + public void testNullableField() throws IOException, SQLException { String[] argv = {}; final int TOTAL_RECORDS = 1 * 10;
