Updated Branches: refs/heads/master 045557973 -> 37a73f103
CRUNCH-234: Fix non-zero size on empty intermediate outputs in hadoop2 Project: http://git-wip-us.apache.org/repos/asf/crunch/repo Commit: http://git-wip-us.apache.org/repos/asf/crunch/commit/37a73f10 Tree: http://git-wip-us.apache.org/repos/asf/crunch/tree/37a73f10 Diff: http://git-wip-us.apache.org/repos/asf/crunch/diff/37a73f10 Branch: refs/heads/master Commit: 37a73f1039ddb205874acb58177772cef86e6485 Parents: 0455579 Author: Josh Wills <[email protected]> Authored: Mon Jul 8 09:51:42 2013 -0700 Committer: Josh Wills <[email protected]> Committed: Mon Jul 8 09:51:42 2013 -0700 ---------------------------------------------------------------------- .../main/java/org/apache/crunch/io/SourceTargetHelper.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/crunch/blob/37a73f10/crunch-core/src/main/java/org/apache/crunch/io/SourceTargetHelper.java ---------------------------------------------------------------------- diff --git a/crunch-core/src/main/java/org/apache/crunch/io/SourceTargetHelper.java b/crunch-core/src/main/java/org/apache/crunch/io/SourceTargetHelper.java index 66764cf..7b9bea0 100644 --- a/crunch-core/src/main/java/org/apache/crunch/io/SourceTargetHelper.java +++ b/crunch-core/src/main/java/org/apache/crunch/io/SourceTargetHelper.java @@ -41,7 +41,13 @@ public class SourceTargetHelper { } long size = 0; for (FileStatus status : stati) { - size += fs.getContentSummary(status.getPath()).getLength(); + if (status.isDir()) { + for (FileStatus st : fs.listStatus(status.getPath())) { + size += st.getLen(); + } + } else { + size += status.getLen(); + } } return size; }
