Repository: hadoop Updated Branches: refs/heads/branch-2 d29cf296c -> 5e99b7cd2 refs/heads/branch-2.7 a5a933a9a -> 0fe5f5b5b refs/heads/trunk 344d7cb6d -> fb34f4572
HDFS-7491. Add incremental blockreport latency to DN metrics. Contributed by Ming Ma. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fb34f457 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fb34f457 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fb34f457 Branch: refs/heads/trunk Commit: fb34f45727e63ea55377fe90241328025307d818 Parents: 344d7cb Author: cnauroth <[email protected]> Authored: Wed Mar 11 14:11:19 2015 -0700 Committer: cnauroth <[email protected]> Committed: Wed Mar 11 14:11:19 2015 -0700 ---------------------------------------------------------------------- .../hadoop-common/src/site/markdown/Metrics.md | 2 ++ hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java | 2 ++ .../hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java | 5 +++++ .../apache/hadoop/hdfs/server/datanode/TestDataNodeMetrics.java | 2 ++ 5 files changed, 14 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/fb34f457/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md index 0e0fc09..4a10a00 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md @@ -303,6 +303,8 @@ Each metrics record contains tags such as SessionId and Hostname as additional i | `HeartbeatsAvgTime` | Average heartbeat time in milliseconds | | `BlockReportsNumOps` | Total number of block report operations | | `BlockReportsAvgTime` | Average time of block report operations in milliseconds | +| `IncrementalBlockReportsNumOps` | Total number of incremental block report operations | +| `IncrementalBlockReportsAvgTime` | Average time of incremental block report operations in milliseconds | | `CacheReportsNumOps` | Total number of cache report operations | | `CacheReportsAvgTime` | Average time of cache report operations in milliseconds | | `PacketAckRoundTripTimeNanosNumOps` | Total number of ack round trip | http://git-wip-us.apache.org/repos/asf/hadoop/blob/fb34f457/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 0ba34a4..07213dd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -740,6 +740,9 @@ Release 2.7.0 - UNRELEASED HDFS-6806. HDFS Rolling upgrade document should mention the versions available. (J.Andreina via aajisaka) + HDFS-7491. Add incremental blockreport latency to DN metrics. + (Ming Ma via cnauroth) + OPTIMIZATIONS HDFS-7454. Reduce memory footprint for AclEntries in NameNode. http://git-wip-us.apache.org/repos/asf/hadoop/blob/fb34f457/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java index ff1ad78..3ba2f54 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java @@ -291,12 +291,14 @@ class BPServiceActor implements Runnable { // Send incremental block reports to the Namenode outside the lock boolean success = false; + final long startTime = Time.monotonicNow(); try { bpNamenode.blockReceivedAndDeleted(bpRegistration, bpos.getBlockPoolId(), reports.toArray(new StorageReceivedDeletedBlocks[reports.size()])); success = true; } finally { + dn.getMetrics().addIncrementalBlockReport(Time.monotonicNow()-startTime); if (!success) { synchronized (pendingIncrementalBRperStorage) { for (StorageReceivedDeletedBlocks report : reports) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/fb34f457/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java index 0fbc2ee..2e8eb22 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java @@ -107,6 +107,7 @@ public class DataNodeMetrics { @Metric MutableRate replaceBlockOp; @Metric MutableRate heartbeats; @Metric MutableRate blockReports; + @Metric MutableRate incrementalBlockReports; @Metric MutableRate cacheReports; @Metric MutableRate packetAckRoundTripTimeNanos; final MutableQuantiles[] packetAckRoundTripTimeNanosQuantiles; @@ -201,6 +202,10 @@ public class DataNodeMetrics { blockReports.add(latency); } + public void addIncrementalBlockReport(long latency) { + incrementalBlockReports.add(latency); + } + public void addCacheReport(long latency) { cacheReports.add(latency); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/fb34f457/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetrics.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetrics.java index 8a2bacf..5d27fe6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetrics.java @@ -72,6 +72,8 @@ public class TestDataNodeMetrics { DataNode datanode = datanodes.get(0); MetricsRecordBuilder rb = getMetrics(datanode.getMetrics().name()); assertCounter("BytesWritten", LONG_FILE_LEN, rb); + assertTrue("Expected non-zero number of incremental block reports", + getLongCounter("IncrementalBlockReportsNumOps", rb) > 0); } finally { if (cluster != null) {cluster.shutdown();} }
