HIVE-13122: LLAP: simple Model/View separation for UI (Gopal V) Signed-off-by: Gopal V <[email protected]>
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/5c078940 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/5c078940 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/5c078940 Branch: refs/heads/llap Commit: 5c078940ee9d115abd3eaabb7273cc921c8e394a Parents: 4984714 Author: Gopal V <[email protected]> Authored: Fri Feb 26 02:00:44 2016 -0800 Committer: Gopal V <[email protected]> Committed: Fri Feb 26 02:00:44 2016 -0800 ---------------------------------------------------------------------- .../main/resources/hive-webapps/llap/index.html | 72 ++++- .../resources/hive-webapps/llap/js/metrics.js | 279 +++++++++++++------ 2 files changed, 265 insertions(+), 86 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/5c078940/llap-server/src/main/resources/hive-webapps/llap/index.html ---------------------------------------------------------------------- diff --git a/llap-server/src/main/resources/hive-webapps/llap/index.html b/llap-server/src/main/resources/hive-webapps/llap/index.html index 3a7575e..31a27a0 100644 --- a/llap-server/src/main/resources/hive-webapps/llap/index.html +++ b/llap-server/src/main/resources/hive-webapps/llap/index.html @@ -47,28 +47,35 @@ <div class="page-header"> <h1>LLAP Monitor</h1> </div> - <h2 id="hostname"></h2> + <h2 id="hostname"></h2> </div> <div class="row"> <section id="heap"> <h2>Heap Metrics</h2> - <table id="attributes_table" class="table table-striped table"> + <table id="heap_metrics_table" class="table table-striped table"> <thead> <tr> <th>Used(MB)</th> <th>Max(MB)</th> <th>Use Rate(%)</th> + <th>GC time (seconds) </th> </tr> </thead> <tbody id="heap_body"> + <tr> + <td> <span id="heap-used" /> </td> + <td> <span id="heap-max" /> </td> + <td> <span id="heap-rate"> </span> <span id="heap-trend" /></td> + <td> <span id="heap-gc"> </span> <span id="heap-gc-trend" /></td> + </tr> </tbody> </table> </section> <section id="cache"> <h2>Cache Metrics</h2> - <table id="attributes_table" class="table table-striped"> + <table id="cache_metrics_table" class="table table-striped"> <thead> <tr> <th>Used(MB)</th> @@ -79,13 +86,20 @@ </tr> </thead> <tbody id="cache_body"> + <tr> + <td> <span id="cache-used" /> </td> + <td> <span id="cache-max" /> </td> + <td> <span id="cache-fill-rate" /> <span id="cache-fill-trend"/></td> + <td> <span id="cache-requests" /> </td> + <td> <span id="cache-hits"> </span> <span id="cache-hits-trend"/></td> + </tr> </tbody> </table> </section> <section id="executors"> <h2>Executors</h2> - <table id="attributes_table" class="table table-striped"> + <table id="executor_metrics_table" class="table table-striped"> <thead> <tr> <th>Used</th> @@ -93,10 +107,60 @@ <th>Use Rate(%)</th> <th>Queue</th> <th>Executing+Queuing Tasks</th> + </tr> + </thead> + <tbody id="executors_body"> + <tr> + <td> <span id="executors-used" /> </td> + <td> <span id="executors-max" /> </td> + <td> <span id="executors-rate"> </span> <span id="executors-trend"/></td> + <td> <span id="executors-queue" /> </td> + <td> <span id="executors-pending" > </span> <span id="executors-pending-trend"/></td> + </tr> + </tbody> + </table> + </section> + + <section id="Fragments"> + <h2>Fragments</h2> + <table id="fragment_metrics_table" class="table table-striped"> + <thead> + <tr> + <th>Total Fragments</th> + <th>Failed Fragments</th> + <th>Preempted Fragments</th> <th>Preemption Time Lost(s)</th> </tr> </thead> + <tbody id="fragments_body"> + <tr> + <td> <span id="fragments-total" /> </td> + <td> <span id="fragments-failed" /> </td> + <td> <span id="fragments-preempted"> </span> </td> + <td> <span id="fragments-preemption-time" /> </td> + </tr> + </tbody> + </table> + </section> + + <section id="system"> + <h2>System metrics</h2> + <table id="system_metrics_table" class="table table-striped"> + <thead> + <tr> + <th>CPU (%)</th> + <th>Load Average (<span id="proc-cores"></span> cores)</th> + <th>System Used RAM (%)</th> + <th>LLAP Open File #</th> + </tr> + </thead> <tbody id="executors_body"> + <tr> + <td> <span id="proc-cpu"> </span> <span id="proc-cpu-trend"> </span> </td> + <td> <span id="system-loadavg"> </span> <span id="system-loadavg-trend"> </span> </td> + <td> <span id="system-ram"> </span> <span id="system-ram-trend"> </span> </td> + <td> <span id="proc-fds"> </span> <span id="proc-fds-trend"> </span> </td> + </tr> </tbody> </table> </section> http://git-wip-us.apache.org/repos/asf/hive/blob/5c078940/llap-server/src/main/resources/hive-webapps/llap/js/metrics.js ---------------------------------------------------------------------- diff --git a/llap-server/src/main/resources/hive-webapps/llap/js/metrics.js b/llap-server/src/main/resources/hive-webapps/llap/js/metrics.js index 2ecdd4b..b67a6e3 100644 --- a/llap-server/src/main/resources/hive-webapps/llap/js/metrics.js +++ b/llap-server/src/main/resources/hive-webapps/llap/js/metrics.js @@ -16,97 +16,212 @@ * limitations under the License. */ -$(function() { - // JMX json keys - const heapUsedKey = "MemHeapUsedM"; - const heapMaxKey = "MemHeapMaxM"; - - const cacheUsedKey = "CacheCapacityUsed"; - const cacheMaxKey = "CacheCapacityTotal"; - const cacheHitKey = "CacheHitRatio"; - const cacheRequestKey = "CacheReadRequests"; - - const numExecutorsKey = "NumExecutors"; - const executorUsedKey = "ExecutorsStatus"; - const numQueuedKey = "ExecutorNumQueuedRequests"; - const preemptionTimeLostKey = "PreemptionTimeLost"; - - function addRow(table, data, plot) { - var row = $("<tr>"); - table.append(row); - for (var i in data) { - var item = $("<td>"); - item.append(data[i][data[i].length - 1]); - row.append(item); - if (plot[i]) { - var line = $("<span>"); - line.css("padding-left", "0.5em"); - item.append(' '); - item.append(line); - line.sparkline(data[i]); +/* llap (singletons) */ +var llap = {"model" : {}, "view" : {}} + +if (!("console" in window)) { + window.console = {"log" : function() {/*nothing*/}}; +} + +( +/* anonymous closure to prevent variable leakage */ +function () { + +var trendlist = function(max) { + var list = []; + list.fill(0,0,max); + list.peek = function() { + if(this.length != 0) { return this[this.length-1] } + } + list.add = function(v) { + while (this.length < max) { + this.push(0); } - } - } - - function createTable(table, data, plot) { - table.empty(); - addRow(table, data, plot); - } - - function addLineItem(list, newItem, max) { - max = max || 30; - list.push(newItem); - if(list.length < max) { - while(list.length < max) { - list.push(0); + this.push(v); + while(this.length > max) { + this.shift(); } - } else if (list.length > max) { - while(list.length > max) { - list.shift(); + } + return list +} + +var jmxbean = function(jmx, name) { + var beans = jmx["beans"].filter(function(b, i, a) { return b.name.match(name) }); + return beans.pop(); +} + + +llap.model.JvmMetrics = new function() { + this.name = "Hadoop:service=LlapDaemon,name=JvmMetrics"; + this.heap_used = 0; + this.heap_max = 0; + this.heap_rate = trendlist(50); + this.gc_times = trendlist(50); + this.old_gc = 0; + this.push = function(jmx) { + var bean = jmxbean(jmx, this.name); + this.hostname = bean["tag.Hostname"]; + this.heap_max = bean["MemHeapMaxM"]; + this.heap_used = bean["MemHeapUsedM"]; + this.heap_rate.add((this.heap_used*100.0)/this.heap_max); + var new_gc = (bean["GcTimeMillis"]/ 1000.0); + if (this.old_gc != 0) { + this.gc_times.add((new_gc - this.old_gc) / 1000.0); } - } - } + this.old_gc = new_gc; + } + return this; +} - var heapUsed = [], heapMax = [], heapUseRate = []; - var cacheUsed = [], cacheMax = [], cacheUseRate = [], - cacheRequest = [], cacheHit = []; - var executorsUsed = [], numExecutors = [], executorUseRate = [], - queue = [], execAndQueue = [], preemptionTimeLost = []; +llap.model.LlapDaemonCacheMetrics = new function() { + this.name = "Hadoop:service=LlapDaemon,name=LlapDaemonCacheMetrics"; + this.hit_rate = trendlist(50); + this.fill_rate = trendlist(50); + this.push = function(jmx) { + var bean = jmxbean(jmx, this.name); + this.cache_max = bean["CacheCapacityTotal"]/(1024*1024); + this.cache_used = bean["CacheCapacityUsed"]/(1024*1024); + this.cache_reqs = bean["CacheReadRequests"]; + this.fill_rate.add((this.cache_used*100.0)/this.cache_max); + this.hit_rate.add(bean["CacheHitRatio"]*100.0); + } + return this; +} - $("#hostname").append(location.hostname); - setInterval(function() { - $.getJSON("/jmx", function(jmx){ - var data = jmx["beans"]; - for (var i in data) { - if (heapUsedKey in data[i]) addLineItem(heapUsed, data[i][heapUsedKey].toFixed(1)); - if (heapMaxKey in data[i]) addLineItem(heapMax, data[i][heapMaxKey].toFixed(1)); - if (cacheUsedKey in data[i]) addLineItem(cacheUsed, data[i][cacheUsedKey] / 1024 / 1024); // Exchange Byte to MB - if (cacheMaxKey in data[i]) addLineItem(cacheMax, data[i][cacheMaxKey] / 1024 / 1024); +llap.model.LlapDaemonInfo = new function() { + this.name = "Hadoop:service=LlapDaemon,name=LlapDaemonInfo"; + this.active_rate = trendlist(50); + this.push = function(jmx) { + var bean = jmxbean(jmx, this.name); + this.executors = bean["NumExecutors"]; + this.active = bean["ExecutorsStatus"]; + this.active_rate.add(this.active.length); + } +} - if (cacheHitKey in data[i]) addLineItem(cacheHit, data[i][cacheHitKey] * 100); - if (cacheRequestKey in data[i]) addLineItem(cacheRequest, data[i][cacheRequestKey]); +llap.model.LlapDaemonExecutorMetrics = new function() { + this.name = "Hadoop:service=LlapDaemon,name=LlapDaemonExecutorMetrics"; + this.queue_rate = trendlist(50); + this.push = function(jmx) { + var bean = jmxbean(jmx, this.name); + this.queue_rate.add(bean["ExecutorNumQueuedRequests"] || 0); + this.lost_time = bean["PreemptionTimeLost"]; + this.num_tasks = bean["ExecutorTotalRequestsHandled"]; + this.interrupted_tasks = bean["ExecutorTotalInterrupted"]; + this.failed_tasks = bean["ExecutorTotalExecutionFailure"]; + } + return this; +} - if (numExecutorsKey in data[i]) addLineItem(numExecutors, data[i][numExecutorsKey]); - if (executorUsedKey in data[i]) addLineItem(executorsUsed, data[i][executorUsedKey].length); - if (numQueuedKey in data[i]) addLineItem(queue, data[i][numQueuedKey]); - if (preemptionTimeLostKey in data[i]) addLineItem(preemptionTimeLost, data[i][preemptionTimeLostKey]); - } - addLineItem(heapUseRate, (heapUsed[heapUsed.length - 1] / heapMax[heapMax.length - 1] * 100).toFixed(1)); - createTable($("#heap_body"), - [heapUsed, heapMax, heapUseRate], - [false, false, true]); - addLineItem(cacheUseRate, (cacheUsed[cacheUsed.length - 1] / cacheMax[cacheMax.length - 1] * 100).toFixed(1)); - createTable($("#cache_body"), - [cacheUsed, cacheMax, cacheUseRate, cacheRequest, cacheHit], - [false, false, true, false, true]); - addLineItem(executorUseRate, - executorsUsed[executorsUsed.length - 1] / numExecutors[numExecutors.length - 1] * 100); - addLineItem(execAndQueue, executorsUsed[executorsUsed.length - 1] + queue[queue.length - 1]); - createTable($("#executors_body"), - [executorsUsed, numExecutors, executorUseRate, queue, execAndQueue, preemptionTimeLost], - [false, false, true, false, true, false]); +llap.model.OperatingSystem = new function() { + this.name = "java.lang:type=OperatingSystem"; + this.sys_cpu_rate = trendlist(50); + this.proc_cpu_rate = trendlist(50); + this.loadavg_rate = trendlist(50); + this.used_ram_rate = trendlist(50); + this.open_fds_rate = trendlist(50); + this.push = function(jmx) { + var bean = jmxbean(jmx, this.name); + this.sys_cpu_rate.add(Math.max(bean["SystemCpuLoad"]-bean["ProcessCpuLoad"],0)*100); + this.proc_cpu_rate.add(bean["ProcessCpuLoad"]*100); + this.loadavg_rate.add(bean["SystemLoadAverage"]); + this.used_ram_rate.add(100*(bean["TotalPhysicalMemorySize"] - bean["FreePhysicalMemorySize"])/bean["TotalPhysicalMemorySize"]); + this.open_fds_rate.add(bean["OpenFileDescriptorCount"]); + this.cpu_cores = bean["AvailableProcessors"]; + } +} + +llap.view.Hostname = new function () { + this.refresh = function() { + $("#hostname").text(llap.model.JvmMetrics.hostname); + } +} + +llap.view.Heap = new function () { + this.refresh = function() { + var model = llap.model.JvmMetrics; + $("#heap-used").text(model.heap_used.toFixed(2)); + $("#heap-max").text(model.heap_max.toFixed(2)); + $("#heap-rate").text(model.heap_rate.peek().toFixed(2)); + $("#heap-trend").sparkline(model.heap_rate); + $("#heap-gc").text(model.old_gc.toFixed(2)); + $("#heap-gc-trend").sparkline(model.gc_times, {type: 'bar', barColor: 'red'}); + } +} + +llap.view.Cache = new function () { + this.refresh = function() { + var model = llap.model.LlapDaemonCacheMetrics; + $("#cache-used").text(model.cache_used.toFixed(2)); + $("#cache-max").text(model.cache_max.toFixed(2)); + $("#cache-fill-rate").text(model.fill_rate.peek().toFixed(2)); + $("#cache-fill-trend").sparkline(model.fill_rate); + $("#cache-requests").text(model.cache_reqs.toFixed(0)); + $("#cache-hits").text(model.hit_rate.peek().toFixed(2)); + $("#cache-hits-trend").sparkline(model.hit_rate); + } +} + +llap.view.Executors = new function () { + this.refresh = function() { + var model = llap.model.LlapDaemonInfo; + $("#executors-used").text(model.active.length); + $("#executors-max").text(model.executors); + $("#executors-rate").text(((model.active_rate.peek() * 100.0)/model.executors).toFixed(0)); + $("#executors-trend").sparkline(model.active_rate); + + var model1 = llap.model.LlapDaemonExecutorMetrics; + $("#executors-queue").text(model1.queue_rate.peek().toFixed(0)); + $("#executors-pending").text((model1.queue_rate.peek() + model.active_rate.peek()).toFixed(0)); + $("#executors-pending-trend").sparkline(model1.queue_rate.map( + function(v,i) { return v + model.active_rate[i] } + )); + } +} + +llap.view.Tasks = new function() { + this.refresh = function() { + var model = llap.model.LlapDaemonExecutorMetrics; + $("#fragments-total").text(model.num_tasks); + $("#fragments-failed").text(model.failed_tasks); + $("#fragments-preempted").text(model.interrupted_tasks); + $("#fragments-preemption-time").text((model.lost_time / 1000.0).toFixed(3)); + } +} + +llap.view.System = new function() { + this.refresh = function() { + var model = llap.model.OperatingSystem; + $("#proc-cores").text(model.cpu_cores); + $("#proc-cpu").text(model.proc_cpu_rate.peek().toFixed(2)); + $("#proc-cpu-trend").sparkline(model.proc_cpu_rate.map( + function(v, i) { return [model.sys_cpu_rate[i],v]; } + ), {type: 'bar', stackedBarColor: ['red','green']}); + + $("#system-loadavg").text(model.loadavg_rate.peek()); + $("#system-loadavg-trend").sparkline(model.loadavg_rate); + + $("#system-ram").text(model.used_ram_rate.peek().toFixed(2)); + $("#system-ram-trend").sparkline(model.used_ram_rate); + + $("#proc-fds").text(model.open_fds_rate.peek().toFixed(0)); + $("#proc-fds-trend").sparkline(model.open_fds_rate); + } +} + +})(); + +$(function() { + var models = [llap.model.JvmMetrics, llap.model.LlapDaemonCacheMetrics, llap.model.LlapDaemonExecutorMetrics, llap.model.LlapDaemonInfo, llap.model.OperatingSystem] + + var views = [llap.view.Hostname, llap.view.Heap, llap.view.Cache, llap.view.Executors, llap.view.Tasks, llap.view.System] + + setInterval(function() { + $.getJSON("/jmx", function(jmx){ + models.forEach(function (m) { m.push(jmx); }); + views.forEach(function (v) { v.refresh(); }); }); }, 1000); // Update par sec });
