Repository: hadoop Updated Branches: refs/heads/branch-2 5cdc4ba9f -> d19e29bd6
HDFS-8986. Add option to -du to calculate directory space usage excluding snapshots. Contributed by Xiao Chen. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d19e29bd Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d19e29bd Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d19e29bd Branch: refs/heads/branch-2 Commit: d19e29bd6840810617911df3d79f96f125499577 Parents: 5cdc4ba Author: Wei-Chiu Chuang <weic...@apache.org> Authored: Wed Aug 24 08:49:02 2016 -0700 Committer: Wei-Chiu Chuang <weic...@apache.org> Committed: Wed Aug 24 08:49:54 2016 -0700 ---------------------------------------------------------------------- .../org/apache/hadoop/fs/ContentSummary.java | 127 +++++++++-- .../java/org/apache/hadoop/fs/shell/Count.java | 20 +- .../org/apache/hadoop/fs/shell/FsUsage.java | 39 ++-- .../src/site/markdown/FileSystemShell.md | 11 +- .../org/apache/hadoop/fs/shell/TestCount.java | 5 +- .../src/test/resources/testConf.xml | 12 +- .../hadoop/hdfs/protocolPB/PBHelperClient.java | 8 + .../src/main/proto/hdfs.proto | 4 + .../ContentSummaryComputationContext.java | 6 + .../hadoop/hdfs/server/namenode/INode.java | 9 +- .../hdfs/server/namenode/INodeDirectory.java | 4 + .../org/apache/hadoop/hdfs/TestDFSShell.java | 223 ++++++++++++++++++- 12 files changed, 423 insertions(+), 45 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/d19e29bd/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java index 3dedbcc..3e75951 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java @@ -34,6 +34,11 @@ public class ContentSummary extends QuotaUsage implements Writable{ private long length; private long fileCount; private long directoryCount; + // These fields are to track the snapshot-related portion of the values. + private long snapshotLength; + private long snapshotFileCount; + private long snapshotDirectoryCount; + private long snapshotSpaceConsumed; /** We don't use generics. Instead override spaceConsumed and other methods in order to keep backward compatibility. */ @@ -56,6 +61,26 @@ public class ContentSummary extends QuotaUsage implements Writable{ return this; } + public Builder snapshotLength(long snapshotLength) { + this.snapshotLength = snapshotLength; + return this; + } + + public Builder snapshotFileCount(long snapshotFileCount) { + this.snapshotFileCount = snapshotFileCount; + return this; + } + + public Builder snapshotDirectoryCount(long snapshotDirectoryCount) { + this.snapshotDirectoryCount = snapshotDirectoryCount; + return this; + } + + public Builder snapshotSpaceConsumed(long snapshotSpaceConsumed) { + this.snapshotSpaceConsumed = snapshotSpaceConsumed; + return this; + } + @Override public Builder quota(long quota){ super.quota(quota); @@ -107,6 +132,10 @@ public class ContentSummary extends QuotaUsage implements Writable{ private long length; private long fileCount; private long directoryCount; + private long snapshotLength; + private long snapshotFileCount; + private long snapshotDirectoryCount; + private long snapshotSpaceConsumed; } /** Constructor deprecated by ContentSummary.Builder*/ @@ -142,17 +171,37 @@ public class ContentSummary extends QuotaUsage implements Writable{ this.length = builder.length; this.fileCount = builder.fileCount; this.directoryCount = builder.directoryCount; + this.snapshotLength = builder.snapshotLength; + this.snapshotFileCount = builder.snapshotFileCount; + this.snapshotDirectoryCount = builder.snapshotDirectoryCount; + this.snapshotSpaceConsumed = builder.snapshotSpaceConsumed; } /** @return the length */ public long getLength() {return length;} + public long getSnapshotLength() { + return snapshotLength; + } + /** @return the directory count */ public long getDirectoryCount() {return directoryCount;} + public long getSnapshotDirectoryCount() { + return snapshotDirectoryCount; + } + /** @return the file count */ public long getFileCount() {return fileCount;} + public long getSnapshotFileCount() { + return snapshotFileCount; + } + + public long getSnapshotSpaceConsumed() { + return snapshotSpaceConsumed; + } + @Override @InterfaceAudience.Private public void write(DataOutput out) throws IOException { @@ -180,9 +229,14 @@ public class ContentSummary extends QuotaUsage implements Writable{ if (this == to) { return true; } else if (to instanceof ContentSummary) { - return getLength() == ((ContentSummary) to).getLength() && - getFileCount() == ((ContentSummary) to).getFileCount() && - getDirectoryCount() == ((ContentSummary) to).getDirectoryCount() && + ContentSummary right = (ContentSummary) to; + return getLength() == right.getLength() && + getFileCount() == right.getFileCount() && + getDirectoryCount() == right.getDirectoryCount() && + getSnapshotLength() == right.getSnapshotLength() && + getSnapshotFileCount() == right.getSnapshotFileCount() && + getSnapshotDirectoryCount() == right.getSnapshotDirectoryCount() && + getSnapshotSpaceConsumed() == right.getSnapshotSpaceConsumed() && super.equals(to); } else { return super.equals(to); @@ -191,7 +245,9 @@ public class ContentSummary extends QuotaUsage implements Writable{ @Override public int hashCode() { - long result = getLength() ^ getFileCount() ^ getDirectoryCount(); + long result = getLength() ^ getFileCount() ^ getDirectoryCount() + ^ getSnapshotLength() ^ getSnapshotFileCount() + ^ getSnapshotDirectoryCount() ^ getSnapshotSpaceConsumed(); return ((int) result) ^ super.hashCode(); } @@ -255,15 +311,14 @@ public class ContentSummary extends QuotaUsage implements Writable{ * @param qOption a flag indicating if quota needs to be printed or not * @return the string representation of the object */ + @Override public String toString(boolean qOption) { return toString(qOption, false); } /** Return the string representation of the object in the output format. - * if qOption is false, output directory count, file count, and content size; - * if qOption is true, output quota and remaining quota as well. - * if hOption is false file sizes are returned in bytes - * if hOption is true file sizes are returned in human readable + * For description of the options, + * @see #toString(boolean, boolean, boolean, boolean, List) * * @param qOption a flag indicating if quota needs to be printed or not * @param hOption a flag indicating if human readable output if to be used @@ -273,10 +328,24 @@ public class ContentSummary extends QuotaUsage implements Writable{ return toString(qOption, hOption, false, null); } + /** Return the string representation of the object in the output format. + * For description of the options, + * @see #toString(boolean, boolean, boolean, boolean, List) + * + * @param qOption a flag indicating if quota needs to be printed or not + * @param hOption a flag indicating if human readable output is to be used + * @param xOption a flag indicating if calculation from snapshots is to be + * included in the output + * @return the string representation of the object + */ + public String toString(boolean qOption, boolean hOption, boolean xOption) { + return toString(qOption, hOption, false, xOption, null); + } + /** * Return the string representation of the object in the output format. - * if tOption is true, display the quota by storage types, - * Otherwise, same logic with #toString(boolean,boolean) + * For description of the options, + * @see #toString(boolean, boolean, boolean, boolean, List) * * @param qOption a flag indicating if quota needs to be printed or not * @param hOption a flag indicating if human readable output if to be used @@ -286,6 +355,29 @@ public class ContentSummary extends QuotaUsage implements Writable{ */ public String toString(boolean qOption, boolean hOption, boolean tOption, List<StorageType> types) { + return toString(qOption, hOption, tOption, false, types); + } + + /** Return the string representation of the object in the output format. + * if qOption is false, output directory count, file count, and content size; + * if qOption is true, output quota and remaining quota as well. + * if hOption is false, file sizes are returned in bytes + * if hOption is true, file sizes are returned in human readable + * if tOption is true, display the quota by storage types + * if tOption is false, same logic with #toString(boolean,boolean) + * if xOption is false, output includes the calculation from snapshots + * if xOption is true, output excludes the calculation from snapshots + * + * @param qOption a flag indicating if quota needs to be printed or not + * @param hOption a flag indicating if human readable output is to be used + * @param tOption a flag indicating if display quota by storage types + * @param xOption a flag indicating if calculation from snapshots is to be + * included in the output + * @param types Storage types to display + * @return the string representation of the object + */ + public String toString(boolean qOption, boolean hOption, boolean tOption, + boolean xOption, List<StorageType> types) { String prefix = ""; if (tOption) { @@ -296,10 +388,17 @@ public class ContentSummary extends QuotaUsage implements Writable{ prefix = getQuotaUsage(hOption); } - return prefix + String.format(SUMMARY_FORMAT, - formatSize(directoryCount, hOption), - formatSize(fileCount, hOption), - formatSize(length, hOption)); + if (xOption) { + return prefix + String.format(SUMMARY_FORMAT, + formatSize(directoryCount - snapshotDirectoryCount, hOption), + formatSize(fileCount - snapshotFileCount, hOption), + formatSize(length - snapshotLength, hOption)); + } else { + return prefix + String.format(SUMMARY_FORMAT, + formatSize(directoryCount, hOption), + formatSize(fileCount, hOption), + formatSize(length, hOption)); + } } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/d19e29bd/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java index 2b5a4cc..16f52e3 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java @@ -51,6 +51,8 @@ public class Count extends FsCommand { private static final String OPTION_HUMAN = "h"; private static final String OPTION_HEADER = "v"; private static final String OPTION_TYPE = "t"; + // exclude snapshots from calculation. Only work on default columns. + private static final String OPTION_EXCLUDE_SNAPSHOT = "x"; //return the quota, namespace count and disk space usage. private static final String OPTION_QUOTA_AND_USAGE = "u"; @@ -58,7 +60,8 @@ public class Count extends FsCommand { public static final String USAGE = "[-" + OPTION_QUOTA + "] [-" + OPTION_HUMAN + "] [-" + OPTION_HEADER + "] [-" + OPTION_TYPE + " [<storage type>]] [-" + - OPTION_QUOTA_AND_USAGE + "] <path> ..."; + OPTION_QUOTA_AND_USAGE + "] [-" + OPTION_EXCLUDE_SNAPSHOT + + "] <path> ..."; public static final String DESCRIPTION = "Count the number of directories, files and bytes under the paths\n" + "that match the specified file pattern. The output columns are:\n" + @@ -72,6 +75,8 @@ public class Count extends FsCommand { "The -" + OPTION_HUMAN + " option shows file sizes in human readable format.\n" + "The -" + OPTION_HEADER + " option displays a header line.\n" + + "The -" + OPTION_EXCLUDE_SNAPSHOT + " option excludes snapshots " + + "from being calculated. \n" + "The -" + OPTION_TYPE + " option displays quota by storage types.\n" + "It must be used with -" + OPTION_QUOTA + " option.\n" + "If a comma-separated list of storage types is given after the -" + @@ -87,6 +92,7 @@ public class Count extends FsCommand { private boolean showQuotabyType; private List<StorageType> storageTypes = null; private boolean showQuotasAndUsageOnly; + private boolean excludeSnapshots; /** Constructor */ public Count() {} @@ -106,7 +112,8 @@ public class Count extends FsCommand { @Override protected void processOptions(LinkedList<String> args) { CommandFormat cf = new CommandFormat(1, Integer.MAX_VALUE, - OPTION_QUOTA, OPTION_HUMAN, OPTION_HEADER, OPTION_QUOTA_AND_USAGE); + OPTION_QUOTA, OPTION_HUMAN, OPTION_HEADER, OPTION_QUOTA_AND_USAGE, + OPTION_EXCLUDE_SNAPSHOT); cf.addOptionWithValue(OPTION_TYPE); cf.parse(args); if (args.isEmpty()) { // default path is the current working directory @@ -115,6 +122,7 @@ public class Count extends FsCommand { showQuotas = cf.getOpt(OPTION_QUOTA); humanReadable = cf.getOpt(OPTION_HUMAN); showQuotasAndUsageOnly = cf.getOpt(OPTION_QUOTA_AND_USAGE); + excludeSnapshots = cf.getOpt(OPTION_EXCLUDE_SNAPSHOT); if (showQuotas || showQuotasAndUsageOnly) { String types = cf.getOptValue(OPTION_TYPE); @@ -125,6 +133,11 @@ public class Count extends FsCommand { } else { showQuotabyType = false; } + if (excludeSnapshots) { + out.println(OPTION_QUOTA + " or " + OPTION_QUOTA_AND_USAGE + " option " + + "is given, the -x option is ignored."); + excludeSnapshots = false; + } } if (cf.getOpt(OPTION_HEADER)) { @@ -163,7 +176,8 @@ public class Count extends FsCommand { storageTypes) + src); } else { ContentSummary summary = src.fs.getContentSummary(src.path); - out.println(summary.toString(showQuotas, isHumanReadable()) + src); + out.println(summary. + toString(showQuotas, isHumanReadable(), excludeSnapshots) + src); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/d19e29bd/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsUsage.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsUsage.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsUsage.java index 43a950e..00e755e 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsUsage.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsUsage.java @@ -26,6 +26,7 @@ import java.util.List; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.FsStatus; import org.apache.hadoop.fs.Path; import org.apache.hadoop.util.StringUtils; @@ -106,27 +107,30 @@ class FsUsage extends FsCommand { /** show disk usage */ public static class Du extends FsUsage { public static final String NAME = "du"; - public static final String USAGE = "[-s] [-h] <path> ..."; + public static final String USAGE = "[-s] [-h] [-x] <path> ..."; public static final String DESCRIPTION = - "Show the amount of space, in bytes, used by the files that " + - "match the specified file pattern. The following flags are optional:\n" + - "-s: Rather than showing the size of each individual file that" + - " matches the pattern, shows the total (summary) size.\n" + - "-h: Formats the sizes of files in a human-readable fashion" + - " rather than a number of bytes.\n\n" + - "Note that, even without the -s option, this only shows size summaries " + - "one level deep into a directory.\n\n" + - "The output is in the form \n" + - "\tsize\tname(full path)\n"; + "Show the amount of space, in bytes, used by the files that match " + + "the specified file pattern. The following flags are optional:\n" + + "-s: Rather than showing the size of each individual file that" + + " matches the pattern, shows the total (summary) size.\n" + + "-h: Formats the sizes of files in a human-readable fashion" + + " rather than a number of bytes.\n" + + "-x: Excludes snapshots from being counted.\n\n" + + "Note that, even without the -s option, this only shows size " + + "summaries one level deep into a directory.\n\n" + + "The output is in the form \n" + + "\tsize\tname(full path)\n"; protected boolean summary = false; - + private boolean excludeSnapshots = false; + @Override protected void processOptions(LinkedList<String> args) throws IOException { - CommandFormat cf = new CommandFormat(0, Integer.MAX_VALUE, "h", "s"); + CommandFormat cf = new CommandFormat(0, Integer.MAX_VALUE, "h", "s", "x"); cf.parse(args); humanReadable = cf.getOpt("h"); summary = cf.getOpt("s"); + excludeSnapshots = cf.getOpt("x"); if (args.isEmpty()) args.add(Path.CUR_DIR); } @@ -144,11 +148,10 @@ class FsUsage extends FsCommand { @Override protected void processPath(PathData item) throws IOException { - long length; - if (item.stat.isDirectory()) { - length = item.fs.getContentSummary(item.path).getLength(); - } else { - length = item.stat.getLen(); + ContentSummary contentSummary = item.fs.getContentSummary(item.path); + long length = contentSummary.getLength(); + if (excludeSnapshots) { + length -= contentSummary.getSnapshotLength(); } usagesTable.addRow(formatSize(length), item); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/d19e29bd/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md b/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md index 2096df3..0ee3d56 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md @@ -132,10 +132,12 @@ Similar to get command, except that the destination is restricted to a local fil count ----- -Usage: `hadoop fs -count [-q] [-h] [-v] [-t [<storage type>]] [-u] <paths> ` +Usage: `hadoop fs -count [-q] [-h] [-v] [-x] [-t [<storage type>]] [-u] <paths> ` Count the number of directories, files and bytes under the paths that match the specified file pattern. Get the quota and the usage. The output columns with -count are: DIR\_COUNT, FILE\_COUNT, CONTENT\_SIZE, PATHNAME +The -u and -q options control what columns the output contains. -q means show quotas, -u limits the output to show quotas and usage only. + The output columns with -count -q are: QUOTA, REMAINING\_QUOTA, SPACE\_QUOTA, REMAINING\_SPACE\_QUOTA, DIR\_COUNT, FILE\_COUNT, CONTENT\_SIZE, PATHNAME The output columns with -count -u are: QUOTA, REMAINING\_QUOTA, SPACE\_QUOTA, REMAINING\_SPACE\_QUOTA @@ -146,6 +148,8 @@ The -h option shows sizes in human readable format. The -v option displays a header line. +The -x option excludes snapshots from the result calculation. Without the -x option (default), the result is always calculated from all INodes, including all snapshots under the given path. The -x option is ignored if -u or -q option is given. + Example: * `hadoop fs -count hdfs://nn1.example.com/file1 hdfs://nn2.example.com/file2` @@ -211,14 +215,15 @@ Example: du ---- -Usage: `hadoop fs -du [-s] [-h] URI [URI ...]` +Usage: `hadoop fs -du [-s] [-h] [-x] URI [URI ...]` Displays sizes of files and directories contained in the given directory or the length of a file in case its just a file. Options: -* The -s option will result in an aggregate summary of file lengths being displayed, rather than the individual files. +* The -s option will result in an aggregate summary of file lengths being displayed, rather than the individual files. Without the -s option, calculation is done by going 1-level deep from the given path. * The -h option will format file sizes in a "human-readable" fashion (e.g 64.0m instead of 67108864) +* The -x option will exclude snapshots from the result calculation. Without the -x option (default), the result is always calculated from all INodes, including all snapshots under the given path. The du returns three columns with the following format: http://git-wip-us.apache.org/repos/asf/hadoop/blob/d19e29bd/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCount.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCount.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCount.java index f8f08d6..a40027e 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCount.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCount.java @@ -447,7 +447,7 @@ public class TestCount { Count count = new Count(); String actual = count.getUsage(); String expected = - "-count [-q] [-h] [-v] [-t [<storage type>]] [-u] <path> ..."; + "-count [-q] [-h] [-v] [-t [<storage type>]] [-u] [-x] <path> ..."; assertEquals("Count.getUsage", expected, actual); } @@ -465,6 +465,7 @@ public class TestCount { + " DIR_COUNT FILE_COUNT CONTENT_SIZE PATHNAME\n" + "The -h option shows file sizes in human readable format.\n" + "The -v option displays a header line.\n" + + "The -x option excludes snapshots from being calculated. \n" + "The -t option displays quota by storage types.\n" + "It must be used with -q option.\n" + "If a comma-separated list of storage types is given after the -t option, \n" @@ -520,7 +521,7 @@ public class TestCount { } @Override - public String toString(boolean qOption, boolean hOption) { + public String toString(boolean qOption, boolean hOption, boolean xOption) { if (qOption) { if (hOption) { return (HUMAN + WITH_QUOTAS); http://git-wip-us.apache.org/repos/asf/hadoop/blob/d19e29bd/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml b/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml index d497dc9..2e13dce 100644 --- a/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml +++ b/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml @@ -200,7 +200,7 @@ <comparators> <comparator> <type>RegexpComparator</type> - <expected-output>^-du \[-s\] \[-h\] <path> \.\.\. :\s*</expected-output> + <expected-output>^-du \[-s\] \[-h\] \[-x\] <path> \.\.\. :\s*</expected-output> </comparator> <comparator> <type>RegexpComparator</type> @@ -228,6 +228,10 @@ </comparator> <comparator> <type>RegexpComparator</type> + <expected-output>^\s*-x\s*Excludes snapshots from being counted.\s*</expected-output> + </comparator> + <comparator> + <type>RegexpComparator</type> <expected-output>^\s*Note that, even without the -s option, this only shows size summaries one level\s*</expected-output> </comparator> <comparator> @@ -274,7 +278,7 @@ <comparators> <comparator> <type>RegexpComparator</type> - <expected-output>^-count \[-q\] \[-h\] \[-v\] \[-t \[<storage type>\]\] \[-u\] <path> \.\.\. :( )*</expected-output> + <expected-output>^-count \[-q\] \[-h\] \[-v\] \[-t \[<storage type>\]\] \[-u\] \[-x\] <path> \.\.\. :( )*</expected-output> </comparator> <comparator> <type>RegexpComparator</type> @@ -308,6 +312,10 @@ <type>RegexpComparator</type> <expected-output>^( |\t)*The -v option displays a header line.( )*</expected-output> </comparator> + <comparator> + <type>RegexpComparator</type> + <expected-output>^( |\t)*The -x option excludes snapshots from being calculated.( )*</expected-output> + </comparator> </comparators> </test> http://git-wip-us.apache.org/repos/asf/hadoop/blob/d19e29bd/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java index 145386c..7b0e471 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java @@ -1379,6 +1379,10 @@ public class PBHelperClient { builder.length(cs.getLength()). fileCount(cs.getFileCount()). directoryCount(cs.getDirectoryCount()). + snapshotLength(cs.getSnapshotLength()). + snapshotFileCount(cs.getSnapshotFileCount()). + snapshotDirectoryCount(cs.getSnapshotDirectoryCount()). + snapshotSpaceConsumed(cs.getSnapshotSpaceConsumed()). quota(cs.getQuota()). spaceConsumed(cs.getSpaceConsumed()). spaceQuota(cs.getSpaceQuota()); @@ -1973,6 +1977,10 @@ public class PBHelperClient { builder.setLength(cs.getLength()). setFileCount(cs.getFileCount()). setDirectoryCount(cs.getDirectoryCount()). + setSnapshotLength(cs.getSnapshotLength()). + setSnapshotFileCount(cs.getSnapshotFileCount()). + setSnapshotDirectoryCount(cs.getSnapshotDirectoryCount()). + setSnapshotSpaceConsumed(cs.getSnapshotSpaceConsumed()). setQuota(cs.getQuota()). setSpaceConsumed(cs.getSpaceConsumed()). setSpaceQuota(cs.getSpaceQuota()); http://git-wip-us.apache.org/repos/asf/hadoop/blob/d19e29bd/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto index d781fc4..59804a3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto @@ -138,6 +138,10 @@ message ContentSummaryProto { required uint64 spaceConsumed = 5; required uint64 spaceQuota = 6; optional StorageTypeQuotaInfosProto typeQuotaInfos = 7; + optional uint64 snapshotLength = 8; + optional uint64 snapshotFileCount = 9; + optional uint64 snapshotDirectoryCount = 10; + optional uint64 snapshotSpaceConsumed = 11; } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/d19e29bd/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ContentSummaryComputationContext.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ContentSummaryComputationContext.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ContentSummaryComputationContext.java index 5739835..6df9e75 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ContentSummaryComputationContext.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ContentSummaryComputationContext.java @@ -29,6 +29,7 @@ public class ContentSummaryComputationContext { private FSNamesystem fsn = null; private BlockStoragePolicySuite bsps = null; private ContentCounts counts = null; + private ContentCounts snapshotCounts = null; private long nextCountLimit = 0; private long limitPerRun = 0; private long yieldCount = 0; @@ -51,6 +52,7 @@ public class ContentSummaryComputationContext { this.limitPerRun = limitPerRun; this.nextCountLimit = limitPerRun; this.counts = new ContentCounts.Builder().build(); + this.snapshotCounts = new ContentCounts.Builder().build(); this.sleepMilliSec = sleepMicroSec/1000; this.sleepNanoSec = (int)((sleepMicroSec%1000)*1000); } @@ -125,6 +127,10 @@ public class ContentSummaryComputationContext { return counts; } + public ContentCounts getSnapshotCounts() { + return snapshotCounts; + } + public BlockStoragePolicySuite getBlockStoragePolicySuite() { Preconditions.checkState((bsps != null || fsn != null), "BlockStoragePolicySuite must be either initialized or available via" + http://git-wip-us.apache.org/repos/asf/hadoop/blob/d19e29bd/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java index 0348c1f..c6258a1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java @@ -428,8 +428,9 @@ public abstract class INode implements INodeAttributes, Diff.Element<byte[]> { */ public final ContentSummary computeAndConvertContentSummary(int snapshotId, ContentSummaryComputationContext summary) { - ContentCounts counts = computeContentSummary(snapshotId, summary) - .getCounts(); + computeContentSummary(snapshotId, summary); + final ContentCounts counts = summary.getCounts(); + final ContentCounts snapshotCounts = summary.getSnapshotCounts(); final QuotaCounts q = getQuotaCounts(); return new ContentSummary.Builder(). length(counts.getLength()). @@ -440,6 +441,10 @@ public abstract class INode implements INodeAttributes, Diff.Element<byte[]> { spaceQuota(q.getStorageSpace()). typeConsumed(counts.getTypeSpaces()). typeQuota(q.getTypeSpaces().asArray()). + snapshotLength(snapshotCounts.getLength()). + snapshotFileCount(snapshotCounts.getFileCount()). + snapshotDirectoryCount(snapshotCounts.getDirectoryCount()). + snapshotSpaceConsumed(snapshotCounts.getStoragespace()). build(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/d19e29bd/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java index d00c136..9a8f9b2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java @@ -634,6 +634,10 @@ public class INodeDirectory extends INodeWithAdditionalFields // computation should include all the deleted files/directories sf.computeContentSummary4Snapshot(summary.getBlockStoragePolicySuite(), summary.getCounts()); + // Also compute ContentSummary for snapshotCounts (So we can extract it + // later from the ContentSummary of all). + sf.computeContentSummary4Snapshot(summary.getBlockStoragePolicySuite(), + summary.getSnapshotCounts()); } final DirectoryWithQuotaFeature q = getDirectoryWithQuotaFeature(); if (q != null && snapshotId == Snapshot.CURRENT_STATE_ID) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/d19e29bd/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java index 945d3df..213db09 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java @@ -105,6 +105,11 @@ public class TestDFSShell { return p; } + static void rmr(FileSystem fs, Path p) throws IOException { + assertTrue(fs.delete(p, true)); + assertFalse(fs.exists(p)); + } + static File createLocalFile(File f) throws IOException { assertTrue(!f.exists()); PrintWriter out = new PrintWriter(f); @@ -214,6 +219,7 @@ public class TestDFSShell { shell.setConf(conf); try { + cluster.waitActive(); Path myPath = new Path("/test/dir"); assertTrue(fs.mkdirs(myPath)); assertTrue(fs.exists(myPath)); @@ -239,7 +245,7 @@ public class TestDFSShell { assertTrue(val == 0); String returnString = out.toString(); out.reset(); - // Check if size matchs as expected + // Check if size matches as expected assertThat(returnString, containsString(myFileLength.toString())); assertThat(returnString, containsString(myFile2Length.toString())); @@ -274,6 +280,221 @@ public class TestDFSShell { } + @Test (timeout = 180000) + public void testDuSnapshots() throws IOException { + final int replication = 2; + final Configuration conf = new HdfsConfiguration(); + final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) + .numDataNodes(replication).build(); + final DistributedFileSystem dfs = cluster.getFileSystem(); + final PrintStream psBackup = System.out; + final ByteArrayOutputStream out = new ByteArrayOutputStream(); + final PrintStream psOut = new PrintStream(out); + final FsShell shell = new FsShell(); + shell.setConf(conf); + + try { + System.setOut(psOut); + cluster.waitActive(); + final Path parent = new Path("/test"); + final Path dir = new Path(parent, "dir"); + mkdir(dfs, dir); + final Path file = new Path(dir, "file"); + writeFile(dfs, file); + final Path file2 = new Path(dir, "file2"); + writeFile(dfs, file2); + final Long fileLength = dfs.getFileStatus(file).getLen(); + final Long file2Length = dfs.getFileStatus(file2).getLen(); + + /* + * Construct dir as follows: + * /test/dir/file <- this will later be deleted after snapshot taken. + * /test/dir/newfile <- this will be created after snapshot taken. + * /test/dir/file2 + * Snapshot enabled on /test + */ + + // test -du on /test/dir + int ret = -1; + try { + ret = shell.run(new String[] {"-du", dir.toString()}); + } catch (Exception e) { + System.err.println("Exception raised from DFSShell.run " + + e.getLocalizedMessage()); + } + assertEquals(0, ret); + String returnString = out.toString(); + LOG.info("-du return is:\n" + returnString); + // Check if size matches as expected + assertTrue(returnString.contains(fileLength.toString())); + assertTrue(returnString.contains(file2Length.toString())); + out.reset(); + + // take a snapshot, then remove file and add newFile + final String snapshotName = "ss1"; + final Path snapshotPath = new Path(parent, ".snapshot/" + snapshotName); + dfs.allowSnapshot(parent); + assertThat(dfs.createSnapshot(parent, snapshotName), is(snapshotPath)); + rmr(dfs, file); + final Path newFile = new Path(dir, "newfile"); + writeFile(dfs, newFile); + final Long newFileLength = dfs.getFileStatus(newFile).getLen(); + + // test -du -s on /test + ret = -1; + try { + ret = shell.run(new String[] {"-du", "-s", parent.toString()}); + } catch (Exception e) { + System.err.println("Exception raised from DFSShell.run " + + e.getLocalizedMessage()); + } + assertEquals(0, ret); + returnString = out.toString(); + LOG.info("-du -s return is:\n" + returnString); + Long combinedLength = fileLength + file2Length + newFileLength; + assertTrue(returnString.contains(combinedLength.toString())); + out.reset(); + + // test -du on /test + ret = -1; + try { + ret = shell.run(new String[] {"-du", parent.toString()}); + } catch (Exception e) { + System.err.println("Exception raised from DFSShell.run " + + e.getLocalizedMessage()); + } + assertEquals(0, ret); + returnString = out.toString(); + LOG.info("-du return is:\n" + returnString); + assertTrue(returnString.contains(combinedLength.toString())); + out.reset(); + + // test -du -s -x on /test + ret = -1; + try { + ret = shell.run(new String[] {"-du", "-s", "-x", parent.toString()}); + } catch (Exception e) { + System.err.println("Exception raised from DFSShell.run " + + e.getLocalizedMessage()); + } + assertEquals(0, ret); + returnString = out.toString(); + LOG.info("-du -s -x return is:\n" + returnString); + Long exludeSnapshotLength = file2Length + newFileLength; + assertTrue(returnString.contains(exludeSnapshotLength.toString())); + out.reset(); + + // test -du -x on /test + ret = -1; + try { + ret = shell.run(new String[] {"-du", "-x", parent.toString()}); + } catch (Exception e) { + System.err.println("Exception raised from DFSShell.run " + + e.getLocalizedMessage()); + } + assertEquals(0, ret); + returnString = out.toString(); + LOG.info("-du -x return is:\n" + returnString); + assertTrue(returnString.contains(exludeSnapshotLength.toString())); + out.reset(); + } finally { + System.setOut(psBackup); + cluster.shutdown(); + } + } + + @Test (timeout = 180000) + public void testCountSnapshots() throws IOException { + final int replication = 2; + final Configuration conf = new HdfsConfiguration(); + final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) + .numDataNodes(replication).build(); + final DistributedFileSystem dfs = cluster.getFileSystem(); + final PrintStream psBackup = System.out; + final ByteArrayOutputStream out = new ByteArrayOutputStream(); + final PrintStream psOut = new PrintStream(out); + System.setOut(psOut); + final FsShell shell = new FsShell(); + shell.setConf(conf); + + try { + cluster.waitActive(); + final Path parent = new Path("/test"); + final Path dir = new Path(parent, "dir"); + mkdir(dfs, dir); + final Path file = new Path(dir, "file"); + writeFile(dfs, file); + final Path file2 = new Path(dir, "file2"); + writeFile(dfs, file2); + final long fileLength = dfs.getFileStatus(file).getLen(); + final long file2Length = dfs.getFileStatus(file2).getLen(); + final Path dir2 = new Path(parent, "dir2"); + mkdir(dfs, dir2); + + /* + * Construct dir as follows: + * /test/dir/file <- this will later be deleted after snapshot taken. + * /test/dir/newfile <- this will be created after snapshot taken. + * /test/dir/file2 + * /test/dir2 <- this will later be deleted after snapshot taken. + * Snapshot enabled on /test + */ + + // take a snapshot + // then create /test/dir/newfile and remove /test/dir/file, /test/dir2 + final String snapshotName = "s1"; + final Path snapshotPath = new Path(parent, ".snapshot/" + snapshotName); + dfs.allowSnapshot(parent); + assertThat(dfs.createSnapshot(parent, snapshotName), is(snapshotPath)); + rmr(dfs, file); + rmr(dfs, dir2); + final Path newFile = new Path(dir, "new file"); + writeFile(dfs, newFile); + final Long newFileLength = dfs.getFileStatus(newFile).getLen(); + + // test -count on /test. Include header for easier debugging. + int val = -1; + try { + val = shell.run(new String[] {"-count", "-v", parent.toString() }); + } catch (Exception e) { + System.err.println("Exception raised from DFSShell.run " + + e.getLocalizedMessage()); + } + assertEquals(0, val); + String returnString = out.toString(); + LOG.info("-count return is:\n" + returnString); + Scanner in = new Scanner(returnString); + in.nextLine(); + assertEquals(3, in.nextLong()); //DIR_COUNT + assertEquals(3, in.nextLong()); //FILE_COUNT + assertEquals(fileLength + file2Length + newFileLength, + in.nextLong()); //CONTENT_SIZE + out.reset(); + + // test -count -x on /test. Include header for easier debugging. + val = -1; + try { + val = + shell.run(new String[] {"-count", "-x", "-v", parent.toString()}); + } catch (Exception e) { + System.err.println("Exception raised from DFSShell.run " + + e.getLocalizedMessage()); + } + assertEquals(0, val); + returnString = out.toString(); + LOG.info("-count -x return is:\n" + returnString); + in = new Scanner(returnString); + in.nextLine(); + assertEquals(2, in.nextLong()); //DIR_COUNT + assertEquals(2, in.nextLong()); //FILE_COUNT + assertEquals(file2Length + newFileLength, in.nextLong()); //CONTENT_SIZE + out.reset(); + } finally { + System.setOut(psBackup); + cluster.shutdown(); + } + } + @Test (timeout = 30000) public void testPut() throws IOException { Configuration conf = new HdfsConfiguration(); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
