Repository: hadoop Updated Branches: refs/heads/branch-2.7 53a261a7e -> 3b45c278a
Revert "HDFS-12502. nntop should support a category based on FilesInGetListingOps." This reverts commit 53a261a7ed19acf583bd8ab64c4cd0189b9dd842. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3b45c278 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3b45c278 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3b45c278 Branch: refs/heads/branch-2.7 Commit: 3b45c278a584bccf7a278246a6f93bca916cc7b9 Parents: 53a261a Author: Zhe Zhang <[email protected]> Authored: Tue Oct 24 15:03:34 2017 -0700 Committer: Zhe Zhang <[email protected]> Committed: Tue Oct 24 15:03:34 2017 -0700 ---------------------------------------------------------------------- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 -- .../hdfs/server/namenode/FSNamesystem.java | 5 ---- .../server/namenode/top/metrics/TopMetrics.java | 30 +++++--------------- .../server/namenode/metrics/TestTopMetrics.java | 11 ++----- 4 files changed, 10 insertions(+), 39 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/3b45c278/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 9b2ac6f..717626e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -19,9 +19,6 @@ Release 2.7.5 - UNRELEASED (Ajay Kumar via Arpit Agarwal, zhz) HDFS-10984. Expose nntop output as metrics. (Siddharth Wagle via xyao, zhz) - - HDFS-12502. nntop should support a category based on FilesInGetListingOps. - (zhz) OPTIMIZATIONS http://git-wip-us.apache.org/repos/asf/hadoop/blob/3b45c278/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 395a50d..6fe7e3d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -4444,11 +4444,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, readUnlock(operationName); } logAuditEvent(true, operationName, src); - if (topConf.isEnabled && isAuditEnabled() && isExternalInvocation() - && dl != null && Server.getRemoteUser() != null) { - topMetrics.reportFilesInGetListing(Server.getRemoteUser().toString(), - dl.getPartialListing().length); - } return dl; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/3b45c278/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/metrics/TopMetrics.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/metrics/TopMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/metrics/TopMetrics.java index 3d8dd19..2719c88 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/metrics/TopMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/metrics/TopMetrics.java @@ -70,14 +70,6 @@ public class TopMetrics implements MetricsSource { public static final Logger LOG = LoggerFactory.getLogger(TopMetrics.class); public static final String TOPMETRICS_METRICS_SOURCE_NAME = "NNTopUserOpCounts"; - /** - * In addition to counts of different RPC calls, NNTop also reports top - * users listing large directories (measured by the number of files involved - * in listing operations from the user). This is important because the CPU - * and GC overhead of a listing operation grows linearly with the number of - * files involved. This category in NNTop is {@link #FILES_IN_GETLISTING}. - */ - public static final String FILES_IN_GETLISTING = "filesInGetListing"; private final boolean isMetricsSourceEnabled; private static void logConf(Configuration conf) { @@ -131,30 +123,22 @@ public class TopMetrics implements MetricsSource { public void report(boolean succeeded, String userName, InetAddress addr, String cmd, String src, String dst, FileStatus status) { // currently nntop only makes use of the username and the command - report(userName, cmd, 1); + report(userName, cmd); } - public void reportFilesInGetListing(String userName, int numFiles) { - report(userName, FILES_IN_GETLISTING, numFiles); - } - - public void report(String userName, String cmd, int delta) { + public void report(String userName, String cmd) { long currTime = Time.monotonicNow(); - report(currTime, userName, cmd, delta); + report(currTime, userName, cmd); } - public void report(long currTime, String userName, String cmd, int delta) { + public void report(long currTime, String userName, String cmd) { LOG.debug("a metric is reported: cmd: {} user: {}", cmd, userName); userName = UserGroupInformation.trimLoginMethod(userName); for (RollingWindowManager rollingWindowManager : rollingWindowManagers .values()) { - rollingWindowManager.recordMetric(currTime, cmd, userName, delta); - // Increase the number of all RPC calls by the user, unless the report - // is for the number of files in a listing operation. - if (!cmd.equals(FILES_IN_GETLISTING)) { - rollingWindowManager.recordMetric(currTime, - TopConf.ALL_CMDS, userName, delta); - } + rollingWindowManager.recordMetric(currTime, cmd, userName, 1); + rollingWindowManager.recordMetric(currTime, + TopConf.ALL_CMDS, userName, 1); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/3b45c278/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestTopMetrics.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestTopMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestTopMetrics.java index cda4b14..4d3a4f0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestTopMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestTopMetrics.java @@ -26,7 +26,6 @@ import org.apache.hadoop.metrics2.lib.Interns; import org.junit.Test; import static org.apache.hadoop.hdfs.server.namenode.top.metrics.TopMetrics.TOPMETRICS_METRICS_SOURCE_NAME; -import static org.apache.hadoop.hdfs.server.namenode.top.metrics.TopMetrics.FILES_IN_GETLISTING; import static org.apache.hadoop.test.MetricsAsserts.getMetrics; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -42,10 +41,9 @@ public class TestTopMetrics { TopMetrics topMetrics = new TopMetrics(conf, topConf.nntopReportingPeriodsMs); // Dummy command - topMetrics.report("test", "listStatus", 1); - topMetrics.report("test", "listStatus", 1); - topMetrics.report("test", "listStatus", 1); - topMetrics.report("test", FILES_IN_GETLISTING, 1000); + topMetrics.report("test", "listStatus"); + topMetrics.report("test", "listStatus"); + topMetrics.report("test", "listStatus"); MetricsRecordBuilder rb = getMetrics(topMetrics); MetricsCollector mc = rb.parent(); @@ -61,8 +59,5 @@ public class TestTopMetrics { verify(rb, times(3)).addCounter(Interns.info("op=listStatus." + "user=test.count", "Total operations performed by user"), 3L); - - verify(rb, times(3)).addCounter(Interns.info("op=" + FILES_IN_GETLISTING + - ".user=test.count", "Total operations performed by user"), 1000L); } } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
