This is an automated email from the ASF dual-hosted git repository.
tasanuma pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new b89d875 HADOOP-16954. Add -S option in "Count" command to show only
Snapshot Counts. Contributed by hemanthboyina.
b89d875 is described below
commit b89d875f7b1db4a98d37f13040eecc5afdf1a485
Author: Takanobu Asanuma <[email protected]>
AuthorDate: Mon Apr 6 11:03:10 2020 +0900
HADOOP-16954. Add -S option in "Count" command to show only Snapshot
Counts. Contributed by hemanthboyina.
---
.../java/org/apache/hadoop/fs/ContentSummary.java | 34 ++++++++++++++++++--
.../java/org/apache/hadoop/fs/shell/Count.java | 17 ++++++++--
.../src/site/markdown/FileSystemShell.md | 5 ++-
.../org/apache/hadoop/fs/TestContentSummary.java | 36 ++++++++++++++++++++++
.../java/org/apache/hadoop/fs/shell/TestCount.java | 25 +++++++++++++--
.../hadoop-common/src/test/resources/testConf.xml | 2 +-
6 files changed, 110 insertions(+), 9 deletions(-)
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java
index cdbd10f..20e205a 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java
@@ -281,6 +281,21 @@ public class ContentSummary extends QuotaUsage implements
Writable{
private static final String ALL_HEADER = QUOTA_HEADER + SUMMARY_HEADER;
+ /**
+ * Output format:<-------18-------> <----------24---------->
+ * <----------24---------->. <-------------28------------> SNAPSHOT_LENGTH
+ * SNAPSHOT_FILE_COUNT SNAPSHOT_DIR_COUNT SNAPSHOT_SPACE_CONSUMED
+ */
+ private static final String SNAPSHOT_FORMAT = "%18s %24s %24s %28s ";
+
+ private static final String[] SNAPSHOT_HEADER_FIELDS =
+ new String[] {"SNAPSHOT_LENGTH", "SNAPSHOT_FILE_COUNT",
+ "SNAPSHOT_DIR_COUNT", "SNAPSHOT_SPACE_CONSUMED"};
+
+ /** The header string. */
+ private static final String SNAPSHOT_HEADER =
+ String.format(SNAPSHOT_FORMAT, (Object[]) SNAPSHOT_HEADER_FIELDS);
+
/** Return the header of the output.
* if qOption is false, output directory count, file count, and content size;
@@ -293,7 +308,9 @@ public class ContentSummary extends QuotaUsage implements
Writable{
return qOption ? ALL_HEADER : SUMMARY_HEADER;
}
-
+ public static String getSnapshotHeader() {
+ return SNAPSHOT_HEADER;
+ }
/**
* Returns the names of the fields from the summary header.
@@ -416,7 +433,7 @@ public class ContentSummary extends QuotaUsage implements
Writable{
}
/**
- * Formats a size to be human readable or in bytes
+ * Formats a size to be human readable or in bytes.
* @param size value to be formatted
* @param humanReadable flag indicating human readable or not
* @return String representation of the size
@@ -426,4 +443,17 @@ public class ContentSummary extends QuotaUsage implements
Writable{
? StringUtils.TraditionalBinaryPrefix.long2String(size, "", 1)
: String.valueOf(size);
}
+
+ /**
+ * Return the string representation of the snapshot counts in the output
+ * format.
+ * @param hOption flag indicating human readable or not
+ * @return String representation of the snapshot counts
+ */
+ public String toSnapshot(boolean hOption) {
+ return String.format(SNAPSHOT_FORMAT, formatSize(snapshotLength, hOption),
+ formatSize(snapshotFileCount, hOption),
+ formatSize(snapshotDirectoryCount, hOption),
+ formatSize(snapshotSpaceConsumed, hOption));
+ }
}
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java
index 22d8be5..ab7e195 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java
@@ -56,13 +56,14 @@ public class Count extends FsCommand {
//return the quota, namespace count and disk space usage.
private static final String OPTION_QUOTA_AND_USAGE = "u";
private static final String OPTION_ECPOLICY = "e";
+ private static final String OPTION_SNAPSHOT_COUNT = "s";
public static final String NAME = "count";
public static final String USAGE =
"[-" + OPTION_QUOTA + "] [-" + OPTION_HUMAN + "] [-" + OPTION_HEADER
+ "] [-" + OPTION_TYPE + " [<storage type>]] [-" +
OPTION_QUOTA_AND_USAGE + "] [-" + OPTION_EXCLUDE_SNAPSHOT
- + "] [-" + OPTION_ECPOLICY
+ + "] [-" + OPTION_ECPOLICY + "] [-" + OPTION_SNAPSHOT_COUNT
+ "] <path> ...";
public static final String DESCRIPTION =
"Count the number of directories, files and bytes under the paths\n" +
@@ -93,7 +94,8 @@ public class Count extends FsCommand {
"the storage types.\n" +
"The -" + OPTION_QUOTA_AND_USAGE + " option shows the quota and \n" +
"the usage against the quota without the detailed content summary."+
- "The -"+ OPTION_ECPOLICY +" option shows the erasure coding policy.";
+ "The -" + OPTION_ECPOLICY + " option shows the erasure coding
policy."
+ + "The -" + OPTION_SNAPSHOT_COUNT + " option shows snapshot counts.";
private boolean showQuotas;
private boolean humanReadable;
@@ -102,6 +104,7 @@ public class Count extends FsCommand {
private boolean showQuotasAndUsageOnly;
private boolean excludeSnapshots;
private boolean displayECPolicy;
+ private boolean showSnapshot;
/** Constructor */
public Count() {}
@@ -123,7 +126,7 @@ public class Count extends FsCommand {
CommandFormat cf = new CommandFormat(1, Integer.MAX_VALUE,
OPTION_QUOTA, OPTION_HUMAN, OPTION_HEADER, OPTION_QUOTA_AND_USAGE,
OPTION_EXCLUDE_SNAPSHOT,
- OPTION_ECPOLICY);
+ OPTION_ECPOLICY, OPTION_SNAPSHOT_COUNT);
cf.addOptionWithValue(OPTION_TYPE);
cf.parse(args);
if (args.isEmpty()) { // default path is the current working directory
@@ -134,6 +137,7 @@ public class Count extends FsCommand {
showQuotasAndUsageOnly = cf.getOpt(OPTION_QUOTA_AND_USAGE);
excludeSnapshots = cf.getOpt(OPTION_EXCLUDE_SNAPSHOT);
displayECPolicy = cf.getOpt(OPTION_ECPOLICY);
+ showSnapshot = cf.getOpt(OPTION_SNAPSHOT_COUNT);
if (showQuotas || showQuotasAndUsageOnly) {
String types = cf.getOptValue(OPTION_TYPE);
@@ -165,6 +169,9 @@ public class Count extends FsCommand {
if(displayECPolicy){
headString.append("ERASURECODING_POLICY ");
}
+ if (showSnapshot) {
+ headString.append(ContentSummary.getSnapshotHeader());
+ }
headString.append("PATHNAME");
out.println(headString.toString());
}
@@ -205,6 +212,10 @@ public class Count extends FsCommand {
outputString.append(summary.getErasureCodingPolicy())
.append(" ");
}
+ if (showSnapshot) {
+ ContentSummary summary = src.fs.getContentSummary(src.path);
+ outputString.append(summary.toSnapshot(isHumanReadable()));
+ }
outputString.append(src);
out.println(outputString.toString());
}
diff --git
a/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
b/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
index 7df2cce..39b2ff7 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
@@ -145,7 +145,7 @@ Similar to get command, except that the destination is
restricted to a local fil
count
-----
-Usage: `hadoop fs -count [-q] [-h] [-v] [-x] [-t [<storage type>]] [-u] [-e]
<paths> `
+Usage: `hadoop fs -count [-q] [-h] [-v] [-x] [-t [<storage type>]] [-u] [-e]
[-s] <paths> `
Count the number of directories, files and bytes under the paths that match
the specified file pattern. Get the quota and the usage. The output columns
with -count are: DIR\_COUNT, FILE\_COUNT, CONTENT\_SIZE, PATHNAME
@@ -169,6 +169,8 @@ The output columns with -count -e are: DIR\_COUNT,
FILE\_COUNT, CONTENT_SIZE, ER
The ERASURECODING\_POLICY is name of the policy for the file. If a erasure
coding policy is setted on that file, it will return name of the policy. If no
erasure coding policy is setted, it will return \"Replicated\" which means it
use replication storage strategy.
+The -s option shows the snapshot counts for each directory.
+
Example:
* `hadoop fs -count hdfs://nn1.example.com/file1 hdfs://nn2.example.com/file2`
@@ -179,6 +181,7 @@ Example:
* `hadoop fs -count -u -h hdfs://nn1.example.com/file1`
* `hadoop fs -count -u -h -v hdfs://nn1.example.com/file1`
* `hadoop fs -count -e hdfs://nn1.example.com/file1`
+* `hadoop fs -count -s hdfs://nn1.example.com/file1`
Exit Code:
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestContentSummary.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestContentSummary.java
index 7cc7ae4..98f9f20 100644
---
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestContentSummary.java
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestContentSummary.java
@@ -253,4 +253,40 @@ public class TestContentSummary {
String expected = " 32.6 K 211.9 M 8.0 E ";
assertEquals(expected, contentSummary.toString(false, true));
}
+
+ // check the toSnapshot method with human readable.
+ @Test
+ public void testToSnapshotHumanReadable() {
+ long snapshotLength = Long.MAX_VALUE;
+ long snapshotFileCount = 222222222;
+ long snapshotDirectoryCount = 33333;
+ long snapshotSpaceConsumed = 222256578;
+
+ ContentSummary contentSummary = new ContentSummary.Builder()
+ .snapshotLength(snapshotLength).snapshotFileCount(snapshotFileCount)
+ .snapshotDirectoryCount(snapshotDirectoryCount)
+ .snapshotSpaceConsumed(snapshotSpaceConsumed).build();
+ String expected =
+ " 8.0 E 211.9 M 32.6 K "
+ + " 212.0 M ";
+ assertEquals(expected, contentSummary.toSnapshot(true));
+ }
+
+ // check the toSnapshot method with human readable disabled.
+ @Test
+ public void testToSnapshotNotHumanReadable() {
+ long snapshotLength = 1111;
+ long snapshotFileCount = 2222;
+ long snapshotDirectoryCount = 3333;
+ long snapshotSpaceConsumed = 4444;
+
+ ContentSummary contentSummary = new ContentSummary.Builder()
+ .snapshotLength(snapshotLength).snapshotFileCount(snapshotFileCount)
+ .snapshotDirectoryCount(snapshotDirectoryCount)
+ .snapshotSpaceConsumed(snapshotSpaceConsumed).build();
+ String expected =
+ " 1111 2222 3333 "
+ + " 4444 ";
+ assertEquals(expected, contentSummary.toSnapshot(false));
+ }
}
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCount.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCount.java
index b5adfcf..f101fed 100644
---
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCount.java
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCount.java
@@ -412,6 +412,25 @@ public class TestCount {
}
@Test
+ public void processPathWithSnapshotHeader() throws Exception {
+ Path path = new Path("mockfs:/test");
+ when(mockFs.getFileStatus(eq(path))).thenReturn(fileStat);
+ PrintStream out = mock(PrintStream.class);
+ Count count = new Count();
+ count.out = out;
+ LinkedList<String> options = new LinkedList<String>();
+ options.add("-s");
+ options.add("-v");
+ options.add("dummy");
+ count.processOptions(options);
+ String withSnapshotHeader = " DIR_COUNT FILE_COUNT CONTENT_SIZE "
+ + " SNAPSHOT_LENGTH SNAPSHOT_FILE_COUNT "
+ + " SNAPSHOT_DIR_COUNT SNAPSHOT_SPACE_CONSUMED PATHNAME";
+ verify(out).println(withSnapshotHeader);
+ verifyNoMoreInteractions(out);
+ }
+
+ @Test
public void getCommandName() {
Count count = new Count();
String actual = count.getCommandName();
@@ -448,7 +467,8 @@ public class TestCount {
Count count = new Count();
String actual = count.getUsage();
String expected =
- "-count [-q] [-h] [-v] [-t [<storage type>]] [-u] [-x] [-e] <path>
...";
+ "-count [-q] [-h] [-v] [-t [<storage type>]]"
+ + " [-u] [-x] [-e] [-s] <path> ...";
assertEquals("Count.getUsage", expected, actual);
}
@@ -480,7 +500,8 @@ public class TestCount {
+ "storage types.\n"
+ "The -u option shows the quota and \n"
+ "the usage against the quota without the detailed content summary."
- + "The -e option shows the erasure coding policy.";
+ + "The -e option shows the erasure coding policy."
+ + "The -s option shows snapshot counts.";
assertEquals("Count.getDescription", expected, actual);
}
diff --git
a/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
b/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
index 392d391..530791e 100644
--- a/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
+++ b/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
@@ -282,7 +282,7 @@
<comparators>
<comparator>
<type>RegexpComparator</type>
- <expected-output>^-count \[-q\] \[-h\] \[-v\] \[-t \[<storage
type>\]\] \[-u\] \[-x\] \[-e\] <path> \.\.\. :( )*</expected-output>
+ <expected-output>^-count \[-q\] \[-h\] \[-v\] \[-t \[<storage
type>\]\] \[-u\] \[-x\] \[-e\] \[-s\] <path> \.\.\. :(
)*</expected-output>
</comparator>
<comparator>
<type>RegexpComparator</type>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
