Author: atm
Date: Fri Jun 20 02:39:36 2014
New Revision: 1604076
URL: http://svn.apache.org/r1604076
Log:
HDFS-6403. Add metrics for log warnings reported by JVM pauses. Contributed by
Yongjun Zhang.
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeMetrics.java
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1604076&r1=1604075&r2=1604076&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
(original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
Fri Jun 20 02:39:36 2014
@@ -199,6 +199,9 @@ Release 2.5.0 - UNRELEASED
HDFS-6480. Move waitForReady() from FSDirectory to FSNamesystem. (wheat9)
+ HDFS-6403. Add metrics for log warnings reported by JVM pauses. (Yongjun
+ Zhang via atm)
+
OPTIMIZATIONS
HDFS-6214. Webhdfs has poor throughput for files >2GB (daryn)
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java?rev=1604076&r1=1604075&r2=1604076&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
Fri Jun 20 02:39:36 2014
@@ -785,7 +785,8 @@ public class DataNode extends Configured
initIpcServer(conf);
metrics = DataNodeMetrics.create(conf, getDisplayName());
-
+ metrics.getJvmMetrics().setPauseMonitor(pauseMonitor);
+
blockPoolManager = new BlockPoolManager(this);
blockPoolManager.refreshNamenodes(conf);
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java?rev=1604076&r1=1604075&r2=1604076&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java
Fri Jun 20 02:39:36 2014
@@ -90,13 +90,15 @@ public class DataNodeMetrics {
final MutableQuantiles[] sendDataPacketBlockedOnNetworkNanosQuantiles;
@Metric MutableRate sendDataPacketTransferNanos;
final MutableQuantiles[] sendDataPacketTransferNanosQuantiles;
-
final MetricsRegistry registry = new MetricsRegistry("datanode");
final String name;
-
- public DataNodeMetrics(String name, String sessionId, int[] intervals) {
+ JvmMetrics jvmMetrics = null;
+
+ public DataNodeMetrics(String name, String sessionId, int[] intervals,
+ final JvmMetrics jvmMetrics) {
this.name = name;
+ this.jvmMetrics = jvmMetrics;
registry.tag(SessionId, sessionId);
final int len = intervals.length;
@@ -131,7 +133,7 @@ public class DataNodeMetrics {
public static DataNodeMetrics create(Configuration conf, String dnName) {
String sessionId = conf.get(DFSConfigKeys.DFS_METRICS_SESSION_ID_KEY);
MetricsSystem ms = DefaultMetricsSystem.instance();
- JvmMetrics.create("DataNode", sessionId, ms);
+ JvmMetrics jm = JvmMetrics.create("DataNode", sessionId, ms);
String name = "DataNodeActivity-"+ (dnName.isEmpty()
? "UndefinedDataNodeName"+ DFSUtil.getRandom().nextInt()
: dnName.replace(':', '-'));
@@ -141,11 +143,15 @@ public class DataNodeMetrics {
conf.getInts(DFSConfigKeys.DFS_METRICS_PERCENTILES_INTERVALS_KEY);
return ms.register(name, null, new DataNodeMetrics(name, sessionId,
- intervals));
+ intervals, jm));
}
public String name() { return name; }
+ public JvmMetrics getJvmMetrics() {
+ return jvmMetrics;
+ }
+
public void addHeartbeat(long latency) {
heartbeats.add(latency);
}
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java?rev=1604076&r1=1604075&r2=1604076&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
Fri Jun 20 02:39:36 2014
@@ -597,7 +597,8 @@ public class NameNode implements NameNod
pauseMonitor = new JvmPauseMonitor(conf);
pauseMonitor.start();
-
+ metrics.getJvmMetrics().setPauseMonitor(pauseMonitor);
+
startCommonServices(conf);
}
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeMetrics.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeMetrics.java?rev=1604076&r1=1604075&r2=1604076&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeMetrics.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeMetrics.java
Fri Jun 20 02:39:36 2014
@@ -98,7 +98,11 @@ public class NameNodeMetrics {
@Metric("GetImageServlet putImage")
MutableRate putImage;
- NameNodeMetrics(String processName, String sessionId, int[] intervals) {
+ JvmMetrics jvmMetrics = null;
+
+ NameNodeMetrics(String processName, String sessionId, int[] intervals,
+ final JvmMetrics jvmMetrics) {
+ this.jvmMetrics = jvmMetrics;
registry.tag(ProcessName, processName).tag(SessionId, sessionId);
final int len = intervals.length;
@@ -124,14 +128,19 @@ public class NameNodeMetrics {
String sessionId = conf.get(DFSConfigKeys.DFS_METRICS_SESSION_ID_KEY);
String processName = r.toString();
MetricsSystem ms = DefaultMetricsSystem.instance();
- JvmMetrics.create(processName, sessionId, ms);
+ JvmMetrics jm = JvmMetrics.create(processName, sessionId, ms);
// Percentile measurement is off by default, by watching no intervals
int[] intervals =
conf.getInts(DFSConfigKeys.DFS_METRICS_PERCENTILES_INTERVALS_KEY);
- return ms.register(new NameNodeMetrics(processName, sessionId, intervals));
+ return ms.register(new NameNodeMetrics(processName, sessionId,
+ intervals, jm));
}
+ public JvmMetrics getJvmMetrics() {
+ return jvmMetrics;
+ }
+
public void shutdown() {
DefaultMetricsSystem.shutdown();
}
