Repository: incubator-impala Updated Branches: refs/heads/master 6242478d0 -> f982c3f76
IMPALA-4884: Add JVM heap and non-heap usage in metrics and UI This commit adds heap and non-heap memory usage of the embedded JVM in the memory metrics and exposes these metrics in /memz web page of the impalad and catalog web UI. Change-Id: I543d4d428d7240e0f710d67973867162f2fcabc8 Reviewed-on: http://gerrit.cloudera.org:8080/5909 Reviewed-by: Dimitris Tsirogiannis <[email protected]> Tested-by: Impala Public Jenkins Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/af469984 Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/af469984 Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/af469984 Branch: refs/heads/master Commit: af4699847090ff7d03a588aa1b2e48170c64f03f Parents: 6242478 Author: Dimitris Tsirogiannis <[email protected]> Authored: Sat Feb 4 21:14:05 2017 -0800 Committer: Impala Public Jenkins <[email protected]> Committed: Sat Feb 11 01:34:13 2017 +0000 ---------------------------------------------------------------------- be/src/util/default-path-handlers.cc | 12 +++- .../java/org/apache/impala/common/JniUtil.java | 36 ++++++++++ www/memz.tmpl | 70 ++++++++++++++++++-- 3 files changed, 110 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/af469984/be/src/util/default-path-handlers.cc ---------------------------------------------------------------------- diff --git a/be/src/util/default-path-handlers.cc b/be/src/util/default-path-handlers.cc index 91c51d3..732d3b3 100644 --- a/be/src/util/default-path-handlers.cc +++ b/be/src/util/default-path-handlers.cc @@ -148,17 +148,23 @@ void MemUsageHandler(MemTracker* mem_tracker, MetricGroup* metric_group, if (jvm_group != NULL) { Value jvm(kObjectType); jvm_group->ToJson(false, document, &jvm); + Value heap(kArrayType); + Value non_heap(kArrayType); Value total(kArrayType); for (SizeType i = 0; i < jvm["metrics"].Size(); ++i) { if (strstr(jvm["metrics"][i]["name"].GetString(), "total") != nullptr) { total.PushBack(jvm["metrics"][i], document->GetAllocator()); + } else if (strstr(jvm["metrics"][i]["name"].GetString(), "non-heap") != nullptr) { + non_heap.PushBack(jvm["metrics"][i], document->GetAllocator()); + } else if (strstr(jvm["metrics"][i]["name"].GetString(), "heap") != nullptr) { + heap.PushBack(jvm["metrics"][i], document->GetAllocator()); } } - document->AddMember("jvm", total, document->GetAllocator()); - + document->AddMember("jvm_total", total, document->GetAllocator()); + document->AddMember("jvm_heap", heap, document->GetAllocator()); + document->AddMember("jvm_non_heap", non_heap, document->GetAllocator()); } } - } http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/af469984/fe/src/main/java/org/apache/impala/common/JniUtil.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/org/apache/impala/common/JniUtil.java b/fe/src/main/java/org/apache/impala/common/JniUtil.java index 4446a55..223c8bb 100644 --- a/fe/src/main/java/org/apache/impala/common/JniUtil.java +++ b/fe/src/main/java/org/apache/impala/common/JniUtil.java @@ -22,6 +22,7 @@ import java.io.PrintWriter; import java.io.StringWriter; import java.io.Writer; import java.lang.management.ManagementFactory; +import java.lang.management.MemoryMXBean; import java.lang.management.MemoryPoolMXBean; import java.lang.management.MemoryUsage; import java.util.ArrayList; @@ -142,6 +143,41 @@ public class JniUtil { } } } + + if (request.get_all || request.getMemory_pool().equals("heap")) { + // Populate heap usage + MemoryMXBean mBean = ManagementFactory.getMemoryMXBean(); + TJvmMemoryPool heap = new TJvmMemoryPool(); + MemoryUsage heapUsage = mBean.getHeapMemoryUsage(); + heap.setCommitted(heapUsage.getCommitted()); + heap.setInit(heapUsage.getInit()); + heap.setMax(heapUsage.getMax()); + heap.setUsed(heapUsage.getUsed()); + heap.setName("heap"); + heap.setPeak_committed(0); + heap.setPeak_init(0); + heap.setPeak_max(0); + heap.setPeak_used(0); + jvmMetrics.getMemory_pools().add(heap); + } + + if (request.get_all || request.getMemory_pool().equals("non-heap")) { + // Populate non-heap usage + MemoryMXBean mBean = ManagementFactory.getMemoryMXBean(); + TJvmMemoryPool nonHeap = new TJvmMemoryPool(); + MemoryUsage nonHeapUsage = mBean.getNonHeapMemoryUsage(); + nonHeap.setCommitted(nonHeapUsage.getCommitted()); + nonHeap.setInit(nonHeapUsage.getInit()); + nonHeap.setMax(nonHeapUsage.getMax()); + nonHeap.setUsed(nonHeapUsage.getUsed()); + nonHeap.setName("non-heap"); + nonHeap.setPeak_committed(0); + nonHeap.setPeak_init(0); + nonHeap.setPeak_max(0); + nonHeap.setPeak_used(0); + jvmMetrics.getMemory_pools().add(nonHeap); + } + TSerializer serializer = new TSerializer(protocolFactory_); try { return serializer.serialize(jvmMetrics); http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/af469984/www/memz.tmpl ---------------------------------------------------------------------- diff --git a/www/memz.tmpl b/www/memz.tmpl index f54cdaa..48bd651 100644 --- a/www/memz.tmpl +++ b/www/memz.tmpl @@ -28,15 +28,15 @@ Memory consumption / limit: <strong>{{consumption}}</strong> / <strong>{{mem_lim <h3>Breakdown</h3> <pre>{{detailed}}</pre> -{{?jvm}} -<h3>JVM memory total usage</h3> +{{?jvm_heap}} +<h3>JVM heap usage</h3> <table class='table table-bordered table-hover'> <tr> <th>Name</th> <th>Value</th> <th>Description</th> </tr> - {{#jvm}} + {{#jvm_heap}} <tr> <td><tt>{{name}}</tt></td> {{! Is this a stats metric? }} @@ -54,8 +54,68 @@ Memory consumption / limit: <strong>{{consumption}}</strong> / <strong>{{mem_lim {{description}} </td> </tr> - {{/jvm}} + {{/jvm_heap}} </table> -{{/jvm}} +{{/jvm_heap}} + +{{?jvm_non_heap}} +<h3>JVM non-heap usage</h3> +<table class='table table-bordered table-hover'> + <tr> + <th>Name</th> + <th>Value</th> + <th>Description</th> + </tr> + {{#jvm_non_heap}} + <tr> + <td><tt>{{name}}</tt></td> + {{! Is this a stats metric? }} + {{?mean}} + <td> + Last (of {{count}}): <strong>{{last}}</strong>. + Min: {{min}}, max: {{max}}, avg: {{mean}}</td> + {{/mean}} + {{^mean}} + <td> + {{human_readable}} + </td> + {{/mean}} + <td> + {{description}} + </td> + </tr> + {{/jvm_non_heap}} +</table> +{{/jvm_non_heap}} + +{{?jvm_total}} +<h3>JVM total memory usage</h3> +<table class='table table-bordered table-hover'> + <tr> + <th>Name</th> + <th>Value</th> + <th>Description</th> + </tr> + {{#jvm_total}} + <tr> + <td><tt>{{name}}</tt></td> + {{! Is this a stats metric? }} + {{?mean}} + <td> + Last (of {{count}}): <strong>{{last}}</strong>. + Min: {{min}}, max: {{max}}, avg: {{mean}}</td> + {{/mean}} + {{^mean}} + <td> + {{human_readable}} + </td> + {{/mean}} + <td> + {{description}} + </td> + </tr> + {{/jvm_total}} +</table> +{{/jvm_total}} {{> www/common-footer.tmpl }}
