Repository: hadoop Updated Branches: refs/heads/HDFS-8966 08aafacd0 -> 27009cf32
HDFS-8545. Refactor FS#getUsed() to use ContentSummary and add an API to fetch the total file length from a specific path (Contributed by J.Andreina) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7d2d16f4 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7d2d16f4 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7d2d16f4 Branch: refs/heads/HDFS-8966 Commit: 7d2d16f4ee87ae56dc20016a91c109dd5130f7d4 Parents: b0c818b Author: Vinayakumar B <[email protected]> Authored: Thu Oct 29 11:28:17 2015 +0530 Committer: Vinayakumar B <[email protected]> Committed: Thu Oct 29 11:28:17 2015 +0530 ---------------------------------------------------------------------- .../java/org/apache/hadoop/fs/FileSystem.java | 19 ++++++------- .../org/apache/hadoop/fs/FilterFileSystem.java | 8 +++++- .../org/apache/hadoop/fs/HarFileSystem.java | 6 +++++ hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../hadoop/hdfs/TestDistributedFileSystem.java | 28 ++++++++++++++++++++ 5 files changed, 54 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/7d2d16f4/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java index bd8aa2a..d3eb0ad 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java @@ -2079,16 +2079,17 @@ public abstract class FileSystem extends Configured implements Closeable { CACHE.remove(this.key, this); } - /** Return the total size of all files in the filesystem.*/ - public long getUsed() throws IOException{ - long used = 0; - RemoteIterator<LocatedFileStatus> files = listFiles(new Path("/"), true); - while (files.hasNext()) { - used += files.next().getLen(); - } - return used; + /** Return the total size of all files in the filesystem. */ + public long getUsed() throws IOException { + Path path = new Path("/"); + return getUsed(path); } - + + /** Return the total size of all files from a specified path. */ + public long getUsed(Path path) throws IOException { + return getContentSummary(path).getLength(); + } + /** * Get the block size for a particular file. * @param f the filename http://git-wip-us.apache.org/repos/asf/hadoop/blob/7d2d16f4/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java index f862c74..00f6778 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java @@ -389,7 +389,13 @@ public class FilterFileSystem extends FileSystem { public long getUsed() throws IOException{ return fs.getUsed(); } - + + /** Return the total size of all files from a specified path.*/ + @Override + public long getUsed(Path path) throws IOException { + return fs.getUsed(path); + } + @Override public long getDefaultBlockSize() { return fs.getDefaultBlockSize(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/7d2d16f4/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java index 868b8dc..ea5e6a3 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java @@ -1237,6 +1237,12 @@ public class HarFileSystem extends FileSystem { return fs.getUsed(); } + /** Return the total size of all files from a specified path.*/ + @Override + public long getUsed(Path path) throws IOException { + return fs.getUsed(path); + } + @SuppressWarnings("deprecation") @Override public long getDefaultBlockSize() { http://git-wip-us.apache.org/repos/asf/hadoop/blob/7d2d16f4/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index e79d660..f8f7452 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -1602,6 +1602,9 @@ Release 2.8.0 - UNRELEASED HDFS-9295. Add a thorough test of the full KMS code path. (Daniel Templeton via zhz) + HDFS-8545. Refactor FS#getUsed() to use ContentSummary and add an API to fetch + the total file length from a specific path (J.Andreina via vinayakumarb) + OPTIMIZATIONS HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than http://git-wip-us.apache.org/repos/asf/hadoop/blob/7d2d16f4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java index 79da7b8..0543e9d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java @@ -1048,4 +1048,32 @@ public class TestDistributedFileSystem { cluster.shutdown(); } } + + @Test(timeout = 30000) + public void testTotalDfsUsed() throws Exception { + Configuration conf = new HdfsConfiguration(); + MiniDFSCluster cluster = null; + try { + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build(); + FileSystem fs = cluster.getFileSystem(); + // create file under root + FSDataOutputStream File1 = fs.create(new Path("/File1")); + File1.write("hi".getBytes()); + File1.close(); + // create file under sub-folder + FSDataOutputStream File2 = fs.create(new Path("/Folder1/File2")); + File2.write("hi".getBytes()); + File2.close(); + // getUsed(Path) should return total len of all the files from a path + assertEquals(2, fs.getUsed(new Path("/Folder1"))); + //getUsed() should return total length of all files in filesystem + assertEquals(4, fs.getUsed()); + } finally { + if (cluster != null) { + cluster.shutdown(); + cluster = null; + } + } + } + }
