Repository: hadoop Updated Branches: refs/heads/branch-2 e7e2b8158 -> 87f440739
HDFS-12502. nntop should support a category based on FilesInGetListingOps. (cherry picked from commit 60bfee270ed3a653c44c0bc92396167b5022df6e) (cherry picked from commit aecf6c50d351e7aed2008a682677a1edb104585d) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/87f44073 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/87f44073 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/87f44073 Branch: refs/heads/branch-2 Commit: 87f440739604f97261da9624afb36920a6a4c28f Parents: e7e2b81 Author: Zhe Zhang <z...@apache.org> Authored: Wed Oct 18 23:51:24 2017 -0700 Committer: Zhe Zhang <z...@apache.org> Committed: Thu Oct 19 00:05:34 2017 -0700 ---------------------------------------------------------------------- .../hdfs/server/namenode/FSNamesystem.java | 5 ++++ .../server/namenode/top/metrics/TopMetrics.java | 30 +++++++++++++++----- .../server/namenode/metrics/TestTopMetrics.java | 11 +++++-- 3 files changed, 36 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/87f44073/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index e21da7f..eded007 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -3619,6 +3619,11 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, readUnlock(operationName); } logAuditEvent(true, operationName, src); + if (topConf.isEnabled && isAuditEnabled() && isExternalInvocation() + && dl != null && Server.getRemoteUser() != null) { + topMetrics.reportFilesInGetListing(Server.getRemoteUser().toString(), + dl.getPartialListing().length); + } return dl; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/87f44073/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/metrics/TopMetrics.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/metrics/TopMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/metrics/TopMetrics.java index 2719c88..3d8dd19 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/metrics/TopMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/metrics/TopMetrics.java @@ -70,6 +70,14 @@ public class TopMetrics implements MetricsSource { public static final Logger LOG = LoggerFactory.getLogger(TopMetrics.class); public static final String TOPMETRICS_METRICS_SOURCE_NAME = "NNTopUserOpCounts"; + /** + * In addition to counts of different RPC calls, NNTop also reports top + * users listing large directories (measured by the number of files involved + * in listing operations from the user). This is important because the CPU + * and GC overhead of a listing operation grows linearly with the number of + * files involved. This category in NNTop is {@link #FILES_IN_GETLISTING}. + */ + public static final String FILES_IN_GETLISTING = "filesInGetListing"; private final boolean isMetricsSourceEnabled; private static void logConf(Configuration conf) { @@ -123,22 +131,30 @@ public class TopMetrics implements MetricsSource { public void report(boolean succeeded, String userName, InetAddress addr, String cmd, String src, String dst, FileStatus status) { // currently nntop only makes use of the username and the command - report(userName, cmd); + report(userName, cmd, 1); } - public void report(String userName, String cmd) { + public void reportFilesInGetListing(String userName, int numFiles) { + report(userName, FILES_IN_GETLISTING, numFiles); + } + + public void report(String userName, String cmd, int delta) { long currTime = Time.monotonicNow(); - report(currTime, userName, cmd); + report(currTime, userName, cmd, delta); } - public void report(long currTime, String userName, String cmd) { + public void report(long currTime, String userName, String cmd, int delta) { LOG.debug("a metric is reported: cmd: {} user: {}", cmd, userName); userName = UserGroupInformation.trimLoginMethod(userName); for (RollingWindowManager rollingWindowManager : rollingWindowManagers .values()) { - rollingWindowManager.recordMetric(currTime, cmd, userName, 1); - rollingWindowManager.recordMetric(currTime, - TopConf.ALL_CMDS, userName, 1); + rollingWindowManager.recordMetric(currTime, cmd, userName, delta); + // Increase the number of all RPC calls by the user, unless the report + // is for the number of files in a listing operation. + if (!cmd.equals(FILES_IN_GETLISTING)) { + rollingWindowManager.recordMetric(currTime, + TopConf.ALL_CMDS, userName, delta); + } } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/87f44073/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestTopMetrics.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestTopMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestTopMetrics.java index 4d3a4f0..cda4b14 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestTopMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestTopMetrics.java @@ -26,6 +26,7 @@ import org.apache.hadoop.metrics2.lib.Interns; import org.junit.Test; import static org.apache.hadoop.hdfs.server.namenode.top.metrics.TopMetrics.TOPMETRICS_METRICS_SOURCE_NAME; +import static org.apache.hadoop.hdfs.server.namenode.top.metrics.TopMetrics.FILES_IN_GETLISTING; import static org.apache.hadoop.test.MetricsAsserts.getMetrics; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -41,9 +42,10 @@ public class TestTopMetrics { TopMetrics topMetrics = new TopMetrics(conf, topConf.nntopReportingPeriodsMs); // Dummy command - topMetrics.report("test", "listStatus"); - topMetrics.report("test", "listStatus"); - topMetrics.report("test", "listStatus"); + topMetrics.report("test", "listStatus", 1); + topMetrics.report("test", "listStatus", 1); + topMetrics.report("test", "listStatus", 1); + topMetrics.report("test", FILES_IN_GETLISTING, 1000); MetricsRecordBuilder rb = getMetrics(topMetrics); MetricsCollector mc = rb.parent(); @@ -59,5 +61,8 @@ public class TestTopMetrics { verify(rb, times(3)).addCounter(Interns.info("op=listStatus." + "user=test.count", "Total operations performed by user"), 3L); + + verify(rb, times(3)).addCounter(Interns.info("op=" + FILES_IN_GETLISTING + + ".user=test.count", "Total operations performed by user"), 1000L); } } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
