HADOOP-12154. FileSystem#getUsed() returns the file length only from root '/' (Contributed by J.Andreina)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c9ae815f Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c9ae815f Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c9ae815f Branch: refs/heads/YARN-2928 Commit: c9ae815f6a67e8f3150b4dd4f209a63d7da97b89 Parents: a943699 Author: Vinayakumar B <[email protected]> Authored: Tue Jun 30 15:25:20 2015 +0530 Committer: Zhijie Shen <[email protected]> Committed: Mon Jul 6 11:31:55 2015 -0700 ---------------------------------------------------------------------- hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++ .../java/org/apache/hadoop/fs/FileSystem.java | 6 ++--- .../org/apache/hadoop/hdfs/TestDFSShell.java | 26 ++++++++++++++++++++ 3 files changed, 32 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c9ae815f/hadoop-common-project/hadoop-common/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 50192ae..2cf9509 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -899,6 +899,9 @@ Release 2.8.0 - UNRELEASED HADOOP-12089. StorageException complaining " no lease ID" when updating FolderLastModifiedTime in WASB. (Duo Xu via cnauroth) + HADOOP-12154. FileSystem#getUsed() returns the file length only from root '/' + (J.Andreina via vinayakumarb) + Release 2.7.2 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/c9ae815f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java index 1d7bc87..c98074a 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java @@ -2085,9 +2085,9 @@ public abstract class FileSystem extends Configured implements Closeable { /** Return the total size of all files in the filesystem.*/ public long getUsed() throws IOException{ long used = 0; - FileStatus[] files = listStatus(new Path("/")); - for(FileStatus file:files){ - used += file.getLen(); + RemoteIterator<LocatedFileStatus> files = listFiles(new Path("/"), true); + while (files.hasNext()) { + used += files.next().getLen(); } return used; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/c9ae815f/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java index 2df31c4..1386124 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java @@ -913,6 +913,32 @@ public class TestDFSShell { cluster.shutdown(); } } + + @Test(timeout = 30000) + public void testTotalSizeOfAllFiles() throws Exception { + Configuration conf = new HdfsConfiguration(); + MiniDFSCluster cluster = null; + try { + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build(); + FileSystem fs = cluster.getFileSystem(); + // create file under root + FSDataOutputStream File1 = fs.create(new Path("/File1")); + File1.write("hi".getBytes()); + File1.close(); + // create file under sub-folder + FSDataOutputStream File2 = fs.create(new Path("/Folder1/File2")); + File2.write("hi".getBytes()); + File2.close(); + // getUsed() should return total length of all the files in Filesystem + assertEquals(4, fs.getUsed()); + } finally { + if (cluster != null) { + cluster.shutdown(); + cluster = null; + } + } + } + private static void runCount(String path, long dirs, long files, FsShell shell ) throws IOException { ByteArrayOutputStream bytes = new ByteArrayOutputStream();
