HDFS-12320. Add quantiles for transactions batched in Journal sync. Contributed by Hanisha Koneru.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/423ebcaf Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/423ebcaf Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/423ebcaf Branch: refs/heads/YARN-5734 Commit: 423ebcaf51099268a1c219c42ce339f86d4da111 Parents: 36fb90c Author: Anu Engineer <[email protected]> Authored: Sat Sep 23 10:34:13 2017 -0700 Committer: Anu Engineer <[email protected]> Committed: Sat Sep 23 10:34:13 2017 -0700 ---------------------------------------------------------------------- .../hadoop-common/src/site/markdown/Metrics.md | 18 ++++++++++++++++++ .../server/namenode/metrics/NameNodeMetrics.java | 10 ++++++++++ 2 files changed, 28 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/423ebcaf/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md index 367d9e0..98d4dfa 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md @@ -268,6 +268,24 @@ The server-side metrics for a journal from the JournalNode's perspective. Each m | `Syncs3600s90thPercentileLatencyMicros` | The 90th percentile of sync latency in microseconds (1 hour granularity) | | `Syncs3600s95thPercentileLatencyMicros` | The 95th percentile of sync latency in microseconds (1 hour granularity) | | `Syncs3600s99thPercentileLatencyMicros` | The 99th percentile of sync latency in microseconds (1 hour granularity) | +| `NumTransactionsBatchedInSync60sNumOps` | Number of times transactions were batched in sync operation (1 minute granularity) | +| `NumTransactionsBatchedInSync60s50thPercentileLatencyMicros` | The 50th percentile of transactions batched in sync count (1 minute granularity) | +| `NumTransactionsBatchedInSync60s75thPercentileLatencyMicros` | The 75th percentile of transactions batched in sync count (1 minute granularity) | +| `NumTransactionsBatchedInSync60s90thPercentileLatencyMicros` | The 90th percentile of transactions batched in sync count (1 minute granularity) | +| `NumTransactionsBatchedInSync60s95thPercentileLatencyMicros` | The 95th percentile of transactions batched in sync count (1 minute granularity) | +| `NumTransactionsBatchedInSync60s99thPercentileLatencyMicros` | The 99th percentile of transactions batched in sync count (1 minute granularity) | +| `NumTransactionsBatchedInSync300sNumOps` | Number of times transactions were batched in sync operation (5 minutes granularity) | +| `NumTransactionsBatchedInSync300s50thPercentileLatencyMicros` | The 50th percentile of transactions batched in sync count (5 minutes granularity) | +| `NumTransactionsBatchedInSync300s75thPercentileLatencyMicros` | The 75th percentile of transactions batched in sync count (5 minutes granularity) | +| `NumTransactionsBatchedInSync300s90thPercentileLatencyMicros` | The 90th percentile of transactions batched in sync count (5 minutes granularity) | +| `NumTransactionsBatchedInSync300s95thPercentileLatencyMicros` | The 95th percentile of transactions batched in sync count (5 minutes granularity) | +| `NumTransactionsBatchedInSync300s99thPercentileLatencyMicros` | The 99th percentile of transactions batched in sync count (5 minutes granularity) | +| `NumTransactionsBatchedInSync3600sNumOps` | Number of times transactions were batched in sync operation (1 hour granularity) | +| `NumTransactionsBatchedInSync3600s50thPercentileLatencyMicros` | The 50th percentile of transactions batched in sync count (1 hour granularity) | +| `NumTransactionsBatchedInSync3600s75thPercentileLatencyMicros` | The 75th percentile of transactions batched in sync count (1 hour granularity) | +| `NumTransactionsBatchedInSync3600s90thPercentileLatencyMicros` | The 90th percentile of transactions batched in sync count (1 hour granularity) | +| `NumTransactionsBatchedInSync3600s95thPercentileLatencyMicros` | The 95th percentile of transactions batched in sync count (1 hour granularity) | +| `NumTransactionsBatchedInSync3600s99thPercentileLatencyMicros` | The 99th percentile of transactions batched in sync count (1 hour granularity) | | `BatchesWritten` | Total number of batches written since startup | | `TxnsWritten` | Total number of transactions written since startup | | `BytesWritten` | Total number of bytes written since startup | http://git-wip-us.apache.org/repos/asf/hadoop/blob/423ebcaf/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeMetrics.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeMetrics.java index f2534e4..94c5e9e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeMetrics.java @@ -115,6 +115,8 @@ public class NameNodeMetrics { final MutableQuantiles[] syncsQuantiles; @Metric("Journal transactions batched in sync") MutableCounterLong transactionsBatchedInSync; + @Metric("Journal transactions batched in sync") + final MutableQuantiles[] numTransactionsBatchedInSync; @Metric("Number of blockReports from individual storages") MutableRate storageBlockReport; final MutableQuantiles[] storageBlockReportQuantiles; @@ -148,6 +150,7 @@ public class NameNodeMetrics { final int len = intervals.length; syncsQuantiles = new MutableQuantiles[len]; + numTransactionsBatchedInSync = new MutableQuantiles[len]; storageBlockReportQuantiles = new MutableQuantiles[len]; cacheReportQuantiles = new MutableQuantiles[len]; generateEDEKTimeQuantiles = new MutableQuantiles[len]; @@ -159,6 +162,10 @@ public class NameNodeMetrics { syncsQuantiles[i] = registry.newQuantiles( "syncs" + interval + "s", "Journal syncs", "ops", "latency", interval); + numTransactionsBatchedInSync[i] = registry.newQuantiles( + "numTransactionsBatchedInSync" + interval + "s", + "Number of Transactions batched in sync", "ops", + "count", interval); storageBlockReportQuantiles[i] = registry.newQuantiles( "storageBlockReport" + interval + "s", "Storage block report", "ops", "latency", interval); @@ -304,6 +311,9 @@ public class NameNodeMetrics { public void incrTransactionsBatchedInSync(long count) { transactionsBatchedInSync.incr(count); + for (MutableQuantiles q : numTransactionsBatchedInSync) { + q.add(count); + } } public void incSuccessfulReReplications() { --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
