Repository: falcon Updated Branches: refs/heads/master 334669d84 -> 1d7ca192b
FALCON-1094 getAllFilesRecursivelyHDFS without recursive call. Contributed by Ruslan Ostafiychuk Project: http://git-wip-us.apache.org/repos/asf/falcon/repo Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/1d7ca192 Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/1d7ca192 Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/1d7ca192 Branch: refs/heads/master Commit: 1d7ca192b8ba88078e1716f4dc0ab81707c38144 Parents: 334669d Author: Ruslan Ostafiychuk <[email protected]> Authored: Fri Mar 13 15:59:24 2015 +0200 Committer: Ruslan Ostafiychuk <[email protected]> Committed: Fri Mar 13 15:59:24 2015 +0200 ---------------------------------------------------------------------- falcon-regression/CHANGES.txt | 2 ++ .../falcon/regression/core/util/HadoopUtil.java | 22 ++++++++------------ 2 files changed, 11 insertions(+), 13 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/falcon/blob/1d7ca192/falcon-regression/CHANGES.txt ---------------------------------------------------------------------- diff --git a/falcon-regression/CHANGES.txt b/falcon-regression/CHANGES.txt index 4b38ad6..3e32d78 100644 --- a/falcon-regression/CHANGES.txt +++ b/falcon-regression/CHANGES.txt @@ -60,6 +60,8 @@ Trunk (Unreleased) via Samarth Gupta) IMPROVEMENTS + FALCON-1094 getAllFilesRecursivelyHDFS without recursive call (Ruslan Ostafiychuk) + FALCON-1093 Tag all new tests added to falcon-regression (Paul Isaychuk via Ruslan Ostafiychuk) FALCON-1089 ProcessInstanceStatusTest improvement (Paul Isaychuk via Ruslan Ostafiychuk) http://git-wip-us.apache.org/repos/asf/falcon/blob/1d7ca192/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/HadoopUtil.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/HadoopUtil.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/HadoopUtil.java index 7122274..e2c3ae2 100644 --- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/HadoopUtil.java +++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/HadoopUtil.java @@ -24,7 +24,9 @@ import org.apache.falcon.regression.core.helpers.ColoHelper; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.LocatedFileStatus; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.RemoteIterator; import org.apache.log4j.Logger; import java.io.File; @@ -114,23 +116,17 @@ public final class HadoopUtil { public static List<Path> getAllFilesRecursivelyHDFS( FileSystem fs, Path location) throws IOException { List<Path> returnList = new ArrayList<Path>(); - FileStatus[] stats; + RemoteIterator<LocatedFileStatus> remoteIterator; try { - stats = fs.listStatus(location); + remoteIterator = fs.listFiles(location, true); } catch (FileNotFoundException e) { - e.printStackTrace(); - return new ArrayList<Path>(); - } - if (stats == null) { + LOGGER.info("Path '" + location + "' is not found on " + fs.getUri()); return returnList; } - for (FileStatus stat : stats) { - if (!isDir(stat)) { - if (!stat.getPath().toUri().toString().contains("_SUCCESS")) { - returnList.add(stat.getPath()); - } - } else { - returnList.addAll(getAllFilesRecursivelyHDFS(fs, stat.getPath())); + while(remoteIterator.hasNext()) { + Path path = remoteIterator.next().getPath(); + if (!path.toUri().toString().contains("_SUCCESS")) { + returnList.add(path); } } return returnList;
