HDFS-8911. NameNode Metric : Add Editlog counters as a JMX metric. (Contributed by Anu Engineer)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9c3571ea Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9c3571ea Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9c3571ea Branch: refs/heads/YARN-1197 Commit: 9c3571ea607f0953487464844ed0d46fdb3e9f90 Parents: 2da5aaa Author: Arpit Agarwal <a...@apache.org> Authored: Wed Aug 19 09:42:35 2015 -0700 Committer: Arpit Agarwal <a...@apache.org> Committed: Wed Aug 19 09:42:35 2015 -0700 ---------------------------------------------------------------------- .../hadoop-common/src/site/markdown/Metrics.md | 2 ++ hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ .../hadoop/hdfs/server/namenode/FSEditLog.java | 7 ++++ .../hdfs/server/namenode/FSNamesystem.java | 19 +++++++++++ .../namenode/metrics/FSNamesystemMBean.java | 10 ++++++ .../server/namenode/TestFSNamesystemMBean.java | 34 +++++++++++++++++++- 6 files changed, 74 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/9c3571ea/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md index 7bf69db..de706ad 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md @@ -237,6 +237,8 @@ Each metrics record contains tags such as HAState and Hostname as additional inf | `HAState` | (HA-only) Current state of the NameNode: initializing or active or standby or stopping state | | `FSState` | Current state of the file system: Safemode or Operational | | `LockQueueLength` | Number of threads waiting to acquire FSNameSystem lock | +| `TotalSyncCount` | Total number of sync operations performed by edit log | +| `TotalSyncTimes` | Total number of milliseconds spent by various edit logs in sync operation| JournalNode ----------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/9c3571ea/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index f43f795..7e6370a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -811,6 +811,9 @@ Release 2.8.0 - UNRELEASED HDFS-8826. In Balancer, add an option to specify the source node list so that balancer only selects blocks to move from those nodes. (szetszwo) + HDFS-8911. NameNode Metric : Add Editlog counters as a JMX metric. + (Anu Engineer via Arpit Agarwal) + OPTIMIZATIONS HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than http://git-wip-us.apache.org/repos/asf/hadoop/blob/9c3571ea/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java index 939e841..b1960d9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java @@ -1691,4 +1691,11 @@ public class FSEditLog implements LogsPurgeable { } } + /** + + * Return total number of syncs happened on this edit log. + + * @return long - count + + */ + public long getTotalSyncCount() { + return editLogStream.getNumSync(); + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/9c3571ea/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 4c71cd5..a7dc54e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -7277,5 +7277,24 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, } } + /** + * Return total number of Sync Operations on FSEditLog. + */ + @Override + @Metric({"TotalSyncCount", + "Total number of sync operations performed on edit logs"}) + public long getTotalSyncCount() { + return fsImage.editLog.getTotalSyncCount(); + } + + /** + * Return total time spent doing sync operations on FSEditLog. + */ + @Override + @Metric({"TotalSyncTimes", + "Total time spend in sync operation on various edit logs"}) + public String getTotalSyncTimes() { + return fsImage.editLog.getJournalSet().getSyncTimes(); + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/9c3571ea/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/FSNamesystemMBean.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/FSNamesystemMBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/FSNamesystemMBean.java index 0fa4841..b314f7f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/FSNamesystemMBean.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/FSNamesystemMBean.java @@ -198,4 +198,14 @@ public interface FSNamesystemMBean { * @return int - Number of Threads waiting to acquire FSNameSystemLock */ int getFsLockQueueLength(); + + /** + * Return total number of Sync Operations on FSEditLog. + */ + long getTotalSyncCount(); + + /** + * Return total time spent doing sync operations on FSEditLog. + */ + String getTotalSyncTimes(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/9c3571ea/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystemMBean.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystemMBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystemMBean.java index fb3179a..3c0d54a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystemMBean.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystemMBean.java @@ -30,6 +30,8 @@ import javax.management.MBeanInfo; import javax.management.MBeanServer; import javax.management.ObjectName; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.junit.Test; @@ -159,4 +161,34 @@ public class TestFSNamesystemMBean { } } } -} \ No newline at end of file + + @Test(timeout = 120000) + public void testFsEditLogMetrics() throws Exception { + final Configuration conf = new Configuration(); + MiniDFSCluster cluster = null; + try { + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build(); + cluster.waitActive(); + MBeanServer mbs = ManagementFactory.getPlatformMBeanServer(); + ObjectName mxbeanNameFs = + new ObjectName("Hadoop:service=NameNode,name=FSNamesystemState"); + + FileSystem fs = cluster.getFileSystem(); + final int NUM_OPS = 10; + for (int i = 0; i < NUM_OPS; i++) { + final Path path = new Path(String.format("/user%d", i)); + fs.mkdirs(path); + } + + long syncCount = (long) mbs.getAttribute(mxbeanNameFs, "TotalSyncCount"); + String syncTimes = + (String) mbs.getAttribute(mxbeanNameFs, "TotalSyncTimes"); + assertTrue(syncCount > 0); + assertNotNull(syncTimes); + } finally { + if (cluster != null) { + cluster.shutdown(); + } + } + } +}
