Repository: sqoop Updated Branches: refs/heads/sqoop2 c408ac5a4 -> 934a00976
SQOOP-2156: Sqoop2: HdfsUtils.getOutputMapreduceFiles should ignore hidden files (Jarek Jarcec Cecho via Abraham Elmahrek) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/934a0097 Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/934a0097 Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/934a0097 Branch: refs/heads/sqoop2 Commit: 934a009762719a2459aa8f85e0c300645c5b2d20 Parents: c408ac5 Author: Abraham Elmahrek <[email protected]> Authored: Mon Mar 2 18:27:28 2015 -0800 Committer: Abraham Elmahrek <[email protected]> Committed: Mon Mar 2 18:27:28 2015 -0800 ---------------------------------------------------------------------- .../java/org/apache/sqoop/test/utils/HdfsUtils.java | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/934a0097/test/src/main/java/org/apache/sqoop/test/utils/HdfsUtils.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/sqoop/test/utils/HdfsUtils.java b/test/src/main/java/org/apache/sqoop/test/utils/HdfsUtils.java index 5a55b8b..79700a6 100644 --- a/test/src/main/java/org/apache/sqoop/test/utils/HdfsUtils.java +++ b/test/src/main/java/org/apache/sqoop/test/utils/HdfsUtils.java @@ -20,6 +20,7 @@ package org.apache.sqoop.test.utils; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.PathFilter; import org.apache.log4j.Logger; import java.io.BufferedWriter; @@ -37,6 +38,14 @@ public class HdfsUtils { private static final char PATH_SEPARATOR = '/'; + public static PathFilter filterHiddenFiles = new PathFilter() { + @Override + public boolean accept(Path path) { + String fileName = path.getName(); + return !fileName.startsWith("_") && !fileName.startsWith("."); + } + }; + /** * Get list of mapreduce output files from given directory. * @@ -45,9 +54,9 @@ public class HdfsUtils { * @throws IOException * @throws FileNotFoundException */ - public static Path [] getOutputMapreduceFiles(FileSystem fs, String directory) throws FileNotFoundException, IOException { + public static Path [] getOutputMapreduceFiles(FileSystem fs, String directory) throws IOException { LinkedList<Path> files = new LinkedList<Path>(); - for (FileStatus fileStatus : fs.listStatus(new Path(directory))) { + for (FileStatus fileStatus : fs.listStatus(new Path(directory), filterHiddenFiles)) { LOG.debug("Found mapreduce output file: " + fileStatus.getPath() + " with size " + fileStatus.getLen()); files.add(fileStatus.getPath()); }
