Repository: hadoop Updated Branches: refs/heads/trunk 4e7ad4f0a -> fb06c0083
HADOOP-7713. dfs -count -q should label output column (Jonathan Allen via aw) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fb06c008 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fb06c008 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fb06c008 Branch: refs/heads/trunk Commit: fb06c0083799fa5ca514447ba6b63ce564272805 Parents: 4e7ad4f Author: Allen Wittenauer <[email protected]> Authored: Thu Feb 5 07:44:49 2015 -0800 Committer: Allen Wittenauer <[email protected]> Committed: Thu Feb 5 07:44:49 2015 -0800 ---------------------------------------------------------------------- hadoop-common-project/hadoop-common/CHANGES.txt | 3 + .../org/apache/hadoop/fs/ContentSummary.java | 57 ++++++--- .../java/org/apache/hadoop/fs/shell/Count.java | 32 ++++-- .../src/site/apt/FileSystemShell.apt.vm | 12 +- .../apache/hadoop/fs/TestContentSummary.java | 6 +- .../org/apache/hadoop/fs/shell/TestCount.java | 115 +++++++++++++++++-- .../src/test/resources/testConf.xml | 16 ++- .../src/test/resources/testHDFSConf.xml | 42 +++++++ 8 files changed, 237 insertions(+), 46 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/fb06c008/hadoop-common-project/hadoop-common/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 9f136f5..00cc91e 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -154,6 +154,9 @@ Trunk (Unreleased) HADOOP-10976. moving the source code of hadoop-tools docs to the directory under hadoop-tools (Masatake Iwasaki via aw) + HADOOP-7713. dfs -count -q should label output column (Jonathan Allen + via aw) + BUG FIXES HADOOP-11473. test-patch says "-1 overall" even when all checks are +1 http://git-wip-us.apache.org/repos/asf/hadoop/blob/fb06c008/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java index 5d01637..6276dda 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java @@ -97,28 +97,35 @@ public class ContentSummary implements Writable{ this.spaceConsumed = in.readLong(); this.spaceQuota = in.readLong(); } - - /** + + /** * Output format: * <----12----> <----12----> <-------18-------> - * DIR_COUNT FILE_COUNT CONTENT_SIZE FILE_NAME + * DIR_COUNT FILE_COUNT CONTENT_SIZE */ - private static final String STRING_FORMAT = "%12s %12s %18s "; - /** + private static final String SUMMARY_FORMAT = "%12s %12s %18s "; + /** * Output format: - * <----12----> <----15----> <----15----> <----15----> <----12----> <----12----> <-------18-------> - * QUOTA REMAINING_QUATA SPACE_QUOTA SPACE_QUOTA_REM DIR_COUNT FILE_COUNT CONTENT_SIZE FILE_NAME + * <----12----> <------15-----> <------15-----> <------15-----> + * QUOTA REM_QUOTA SPACE_QUOTA REM_SPACE_QUOTA + * <----12----> <----12----> <-------18-------> + * DIR_COUNT FILE_COUNT CONTENT_SIZE */ - private static final String QUOTA_STRING_FORMAT = "%12s %15s "; - private static final String SPACE_QUOTA_STRING_FORMAT = "%15s %15s "; - + private static final String QUOTA_SUMMARY_FORMAT = "%12s %15s "; + private static final String SPACE_QUOTA_SUMMARY_FORMAT = "%15s %15s "; + + private static final String[] HEADER_FIELDS = new String[] { "DIR_COUNT", + "FILE_COUNT", "CONTENT_SIZE"}; + private static final String[] QUOTA_HEADER_FIELDS = new String[] { "QUOTA", + "REM_QUOTA", "SPACE_QUOTA", "REM_SPACE_QUOTA" }; + /** The header string */ private static final String HEADER = String.format( - STRING_FORMAT.replace('d', 's'), "directories", "files", "bytes"); + SUMMARY_FORMAT, (Object[]) HEADER_FIELDS); private static final String QUOTA_HEADER = String.format( - QUOTA_STRING_FORMAT + SPACE_QUOTA_STRING_FORMAT, - "name quota", "rem name quota", "space quota", "rem space quota") + + QUOTA_SUMMARY_FORMAT + SPACE_QUOTA_SUMMARY_FORMAT, + (Object[]) QUOTA_HEADER_FIELDS) + HEADER; /** Return the header of the output. @@ -131,7 +138,25 @@ public class ContentSummary implements Writable{ public static String getHeader(boolean qOption) { return qOption ? QUOTA_HEADER : HEADER; } - + + /** + * Returns the names of the fields from the summary header. + * + * @return names of fields as displayed in the header + */ + public static String[] getHeaderFields() { + return HEADER_FIELDS; + } + + /** + * Returns the names of the fields used in the quota summary. + * + * @return names of quota fields as displayed in the header + */ + public static String[] getQuotaHeaderFields() { + return QUOTA_HEADER_FIELDS; + } + @Override public String toString() { return toString(true); @@ -175,11 +200,11 @@ public class ContentSummary implements Writable{ spaceQuotaRem = formatSize(spaceQuota - spaceConsumed, hOption); } - prefix = String.format(QUOTA_STRING_FORMAT + SPACE_QUOTA_STRING_FORMAT, + prefix = String.format(QUOTA_SUMMARY_FORMAT + SPACE_QUOTA_SUMMARY_FORMAT, quotaStr, quotaRem, spaceQuotaStr, spaceQuotaRem); } - return prefix + String.format(STRING_FORMAT, + return prefix + String.format(SUMMARY_FORMAT, formatSize(directoryCount, hOption), formatSize(fileCount, hOption), formatSize(length, hOption)); http://git-wip-us.apache.org/repos/asf/hadoop/blob/fb06c008/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java index ff7a10f..dd7d168 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.util.Arrays; import java.util.LinkedList; +import org.apache.commons.lang.StringUtils; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; @@ -44,18 +45,26 @@ public class Count extends FsCommand { private static final String OPTION_QUOTA = "q"; private static final String OPTION_HUMAN = "h"; + private static final String OPTION_HEADER = "v"; public static final String NAME = "count"; public static final String USAGE = - "[-" + OPTION_QUOTA + "] [-" + OPTION_HUMAN + "] <path> ..."; - public static final String DESCRIPTION = + "[-" + OPTION_QUOTA + "] [-" + OPTION_HUMAN + "] [-" + OPTION_HEADER + + "] <path> ..."; + public static final String DESCRIPTION = "Count the number of directories, files and bytes under the paths\n" + - "that match the specified file pattern. The output columns are:\n" + - "DIR_COUNT FILE_COUNT CONTENT_SIZE FILE_NAME or\n" + - "QUOTA REMAINING_QUOTA SPACE_QUOTA REMAINING_SPACE_QUOTA \n" + - " DIR_COUNT FILE_COUNT CONTENT_SIZE FILE_NAME\n" + - "The -h option shows file sizes in human readable format."; - + "that match the specified file pattern. The output columns are:\n" + + StringUtils.join(ContentSummary.getHeaderFields(), ' ') + + " PATHNAME\n" + + "or, with the -" + OPTION_QUOTA + " option:\n" + + StringUtils.join(ContentSummary.getQuotaHeaderFields(), ' ') + "\n" + + " " + + StringUtils.join(ContentSummary.getHeaderFields(), ' ') + + " PATHNAME\n" + + "The -" + OPTION_HUMAN + + " option shows file sizes in human readable format.\n" + + "The -" + OPTION_HEADER + " option displays a header line."; + private boolean showQuotas; private boolean humanReadable; @@ -65,7 +74,7 @@ public class Count extends FsCommand { /** Constructor * @deprecated invoke via {@link FsShell} * @param cmd the count command - * @param pos the starting index of the arguments + * @param pos the starting index of the arguments * @param conf configuration */ @Deprecated @@ -77,13 +86,16 @@ public class Count extends FsCommand { @Override protected void processOptions(LinkedList<String> args) { CommandFormat cf = new CommandFormat(1, Integer.MAX_VALUE, - OPTION_QUOTA, OPTION_HUMAN); + OPTION_QUOTA, OPTION_HUMAN, OPTION_HEADER); cf.parse(args); if (args.isEmpty()) { // default path is the current working directory args.add("."); } showQuotas = cf.getOpt(OPTION_QUOTA); humanReadable = cf.getOpt(OPTION_HUMAN); + if (cf.getOpt(OPTION_HEADER)) { + out.println(ContentSummary.getHeader(showQuotas) + "PATHNAME"); + } } @Override http://git-wip-us.apache.org/repos/asf/hadoop/blob/fb06c008/hadoop-common-project/hadoop-common/src/site/apt/FileSystemShell.apt.vm ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/site/apt/FileSystemShell.apt.vm b/hadoop-common-project/hadoop-common/src/site/apt/FileSystemShell.apt.vm index 5dfbd20..3fd56fc 100644 --- a/hadoop-common-project/hadoop-common/src/site/apt/FileSystemShell.apt.vm +++ b/hadoop-common-project/hadoop-common/src/site/apt/FileSystemShell.apt.vm @@ -154,17 +154,19 @@ bin/hadoop fs <args> * count - Usage: <<<hadoop fs -count [-q] [-h] <paths> >>> + Usage: <<<hadoop fs -count [-q] [-h] [-v] <paths> >>> Count the number of directories, files and bytes under the paths that match the specified file pattern. The output columns with -count are: DIR_COUNT, - FILE_COUNT, CONTENT_SIZE FILE_NAME + FILE_COUNT, CONTENT_SIZE PATHNAME The output columns with -count -q are: QUOTA, REMAINING_QUATA, SPACE_QUOTA, - REMAINING_SPACE_QUOTA, DIR_COUNT, FILE_COUNT, CONTENT_SIZE, FILE_NAME + REMAINING_SPACE_QUOTA, DIR_COUNT, FILE_COUNT, CONTENT_SIZE, PATHNAME The -h option shows sizes in human readable format. + The -v option displays a header line. + Example: * <<<hadoop fs -count hdfs://nn1.example.com/file1 hdfs://nn2.example.com/file2>>> @@ -173,6 +175,8 @@ bin/hadoop fs <args> * <<<hadoop fs -count -q -h hdfs://nn1.example.com/file1>>> + * <<<hdfs dfs -count -q -h -v hdfs://nn1.example.com/file1>>> + Exit Code: Returns 0 on success and -1 on error. @@ -743,4 +747,4 @@ permissions userid groupid modification_date modification_time dirname Usage: <<<hadoop fs -usage command>>> - Return the help for an individual command. \ No newline at end of file + Return the help for an individual command. http://git-wip-us.apache.org/repos/asf/hadoop/blob/fb06c008/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestContentSummary.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestContentSummary.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestContentSummary.java index 9c8a8a4..5db0de3 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestContentSummary.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestContentSummary.java @@ -137,15 +137,15 @@ public class TestContentSummary { // check the header with quotas @Test public void testGetHeaderWithQuota() { - String header = " name quota rem name quota space quota " - + "rem space quota directories files bytes "; + String header = " QUOTA REM_QUOTA SPACE_QUOTA " + + "REM_SPACE_QUOTA DIR_COUNT FILE_COUNT CONTENT_SIZE "; assertEquals(header, ContentSummary.getHeader(true)); } // check the header without quotas @Test public void testGetHeaderNoQuota() { - String header = " directories files bytes "; + String header = " DIR_COUNT FILE_COUNT CONTENT_SIZE "; assertEquals(header, ContentSummary.getHeader(false)); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/fb06c008/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCount.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCount.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCount.java index 6c753c6..1f2f2d4 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCount.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCount.java @@ -31,6 +31,7 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FilterFileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.shell.CommandFormat.NotEnoughArgumentsException; import org.junit.Test; import org.junit.Before; import org.junit.BeforeClass; @@ -47,7 +48,6 @@ public class TestCount { private static Configuration conf; private static FileSystem mockFs; private static FileStatus fileStat; - private static ContentSummary mockCs; @BeforeClass public static void setup() { @@ -55,7 +55,6 @@ public class TestCount { conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class); mockFs = mock(FileSystem.class); fileStat = mock(FileStatus.class); - mockCs = mock(ContentSummary.class); when(fileStat.isFile()).thenReturn(true); } @@ -87,6 +86,85 @@ public class TestCount { assertTrue(count.isHumanReadable()); } + // check no options is handled correctly + @Test + public void processOptionsNoOptions() { + LinkedList<String> options = new LinkedList<String>(); + options.add("dummy"); + Count count = new Count(); + count.processOptions(options); + assertFalse(count.isShowQuotas()); + } + + // check -q is handled correctly + @Test + public void processOptionsShowQuotas() { + LinkedList<String> options = new LinkedList<String>(); + options.add("-q"); + options.add("dummy"); + Count count = new Count(); + count.processOptions(options); + assertTrue(count.isShowQuotas()); + } + + // check missing arguments is handled correctly + @Test + public void processOptionsMissingArgs() { + LinkedList<String> options = new LinkedList<String>(); + Count count = new Count(); + try { + count.processOptions(options); + fail("Count.processOptions - NotEnoughArgumentsException not thrown"); + } catch (NotEnoughArgumentsException e) { + } + assertFalse(count.isShowQuotas()); + } + + // check the correct header is produced with no quotas (-v) + @Test + public void processOptionsHeaderNoQuotas() { + LinkedList<String> options = new LinkedList<String>(); + options.add("-v"); + options.add("dummy"); + + PrintStream out = mock(PrintStream.class); + + Count count = new Count(); + count.out = out; + + count.processOptions(options); + + String noQuotasHeader = + // <----12----> <----12----> <-------18-------> + " DIR_COUNT FILE_COUNT CONTENT_SIZE PATHNAME"; + verify(out).println(noQuotasHeader); + verifyNoMoreInteractions(out); + } + + // check the correct header is produced with quotas (-q -v) + @Test + public void processOptionsHeaderWithQuotas() { + LinkedList<String> options = new LinkedList<String>(); + options.add("-q"); + options.add("-v"); + options.add("dummy"); + + PrintStream out = mock(PrintStream.class); + + Count count = new Count(); + count.out = out; + + count.processOptions(options); + + String withQuotasHeader = + // <----12----> <-----15------> <-----15------> <-----15------> + " QUOTA REM_QUOTA SPACE_QUOTA REM_SPACE_QUOTA " + + // <----12----> <----12----> <-------18-------> + " DIR_COUNT FILE_COUNT CONTENT_SIZE PATHNAME"; + verify(out).println(withQuotasHeader); + verifyNoMoreInteractions(out); + } + // check quotas are reported correctly @Test public void processPathShowQuotas() throws Exception { @@ -211,29 +289,48 @@ public class TestCount { public void getUsage() { Count count = new Count(); String actual = count.getUsage(); - String expected = "-count [-q] [-h] <path> ..."; + String expected = "-count [-q] [-h] [-v] <path> ..."; assertEquals("Count.getUsage", expected, actual); } + // check the correct description is returned + @Test + public void getDescription() { + Count count = new Count(); + String actual = count.getDescription(); + String expected = + "Count the number of directories, files and bytes under the paths\n" + + "that match the specified file pattern. The output columns are:\n" + + "DIR_COUNT FILE_COUNT CONTENT_SIZE PATHNAME\n" + + "or, with the -q option:\n" + + "QUOTA REM_QUOTA SPACE_QUOTA REM_SPACE_QUOTA\n" + + " DIR_COUNT FILE_COUNT CONTENT_SIZE PATHNAME\n" + + "The -h option shows file sizes in human readable format.\n" + + "The -v option displays a header line."; + + assertEquals("Count.getDescription", expected, actual); + } + // mock content system static class MockContentSummary extends ContentSummary { - - public MockContentSummary() {} + + public MockContentSummary() { + } @Override public String toString(boolean qOption, boolean hOption) { if (qOption) { if (hOption) { - return(HUMAN + WITH_QUOTAS); + return (HUMAN + WITH_QUOTAS); } else { - return(BYTES + WITH_QUOTAS); + return (BYTES + WITH_QUOTAS); } } else { if (hOption) { - return(HUMAN + NO_QUOTAS); + return (HUMAN + NO_QUOTAS); } else { - return(BYTES + NO_QUOTAS); + return (BYTES + NO_QUOTAS); } } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/fb06c008/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml b/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml index 5c667e1..f962813 100644 --- a/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml +++ b/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml @@ -238,7 +238,7 @@ <comparators> <comparator> <type>RegexpComparator</type> - <expected-output>^-count \[-q\] \[-h\] <path> \.\.\. :( )*</expected-output> + <expected-output>^-count \[-q\] \[-h\] \[-v\] <path> \.\.\. :( )*</expected-output> </comparator> <comparator> <type>RegexpComparator</type> @@ -250,20 +250,28 @@ </comparator> <comparator> <type>RegexpComparator</type> - <expected-output>^( |\t)*DIR_COUNT FILE_COUNT CONTENT_SIZE FILE_NAME or( )*</expected-output> + <expected-output>^( |\t)*DIR_COUNT FILE_COUNT CONTENT_SIZE PATHNAME( )*</expected-output> </comparator> <comparator> <type>RegexpComparator</type> - <expected-output>^( |\t)*QUOTA REMAINING_QUOTA SPACE_QUOTA REMAINING_SPACE_QUOTA( )*</expected-output> + <expected-output>^( |\t)*or, with the -q option:( )*</expected-output> </comparator> <comparator> <type>RegexpComparator</type> - <expected-output>^( |\t)*DIR_COUNT FILE_COUNT CONTENT_SIZE FILE_NAME( )*</expected-output> + <expected-output>^( |\t)*QUOTA REM_QUOTA SPACE_QUOTA REM_SPACE_QUOTA( )*</expected-output> + </comparator> + <comparator> + <type>RegexpComparator</type> + <expected-output>^( |\t)*DIR_COUNT FILE_COUNT CONTENT_SIZE PATHNAME( )*</expected-output> </comparator> <comparator> <type>RegexpComparator</type> <expected-output>^( |\t)*The -h option shows file sizes in human readable format.( )*</expected-output> </comparator> + <comparator> + <type>RegexpComparator</type> + <expected-output>^( |\t)*The -v option displays a header line.( )*</expected-output> + </comparator> </comparators> </test> http://git-wip-us.apache.org/repos/asf/hadoop/blob/fb06c008/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml index c86b06d..a45f2f2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml @@ -8699,6 +8699,48 @@ </comparators> </test> + <test> <!-- TESTED --> + <description>count: file using absolute path showing header record</description> + <test-commands> + <command>-fs NAMENODE -touchz /file1</command> + <command>-fs NAMENODE -count -v /file1</command> + </test-commands> + <cleanup-commands> + <command>-fs NAMENODE -rm /file1</command> + </cleanup-commands> + <comparators> + <comparator> + <type>RegexpComparator</type> + <expected-output>( |\t)*DIR_COUNT FILE_COUNT CONTENT_SIZE PATHNAME</expected-output> + </comparator> + <comparator> + <type>RegexpComparator</type> + <expected-output>( |\t)*0( |\t)*1( |\t)*0 /file1</expected-output> + </comparator> + </comparators> + </test> + + <test> <!-- TESTED --> + <description>count: file using absolute path with -q option and showing header record</description> + <test-commands> + <command>-fs NAMENODE -touchz /file1</command> + <command>-fs NAMENODE -count -q -v /file1</command> + </test-commands> + <cleanup-commands> + <command>-fs NAMENODE -rm /file1</command> + </cleanup-commands> + <comparators> + <comparator> + <type>RegexpComparator</type> + <expected-output>( |\t)*QUOTA REM_QUOTA SPACE_QUOTA REM_SPACE_QUOTA DIR_COUNT FILE_COUNT CONTENT_SIZE PATHNAME</expected-output> + </comparator> + <comparator> + <type>RegexpComparator</type> + <expected-output>( |\t)*none( |\t)*inf( |\t)*none( |\t)*inf( |\t)*0( |\t)*1( |\t)*0 /file1</expected-output> + </comparator> + </comparators> + </test> + <!-- Tests for chmod --> <test> <!-- TESTED --> <description>chmod: change permission(octal mode) of file in absolute path</description>
