Repository: storm Updated Branches: refs/heads/1.x-branch ce3884933 -> dc20e9ce0
http://git-wip-us.apache.org/repos/asf/storm/blob/0e0bcf27/storm-core/src/ui/public/component.html ---------------------------------------------------------------------- diff --git a/storm-core/src/ui/public/component.html b/storm-core/src/ui/public/component.html index fc7b15e..bd487af 100644 --- a/storm-core/src/ui/public/component.html +++ b/storm-core/src/ui/public/component.html @@ -162,6 +162,12 @@ $(document).ready(function() { } }); + function renderSupervisorPageLink(data, type, row, meta) { + return type === 'display' ? + ("<a href='/supervisor.html?host=" + data + "'>" + data + "</a>") : + data; + } + function renderActionCheckbox(data, type, row, meta) { var host_port = row[2]+':'+$(row[3])[0].text; switch(type) { @@ -268,6 +274,7 @@ $(document).ready(function() { //id, uptime, host, port, actions, emitted, transferred, complete latency, acked, failed dtAutoPage("#executor-stats-table", { columnDefs: [ + {render: renderSupervisorPageLink, searchable: true, targets: [2]}, {render: renderActionCheckbox, searchable: false, targets: [4]}, {type: "num", targets: [5, 6, 7, 8, 9]}, {type: "time-str", targets: [1]}, @@ -303,6 +310,7 @@ $(document).ready(function() { //id, uptime, host, port, actions, emitted, transferred, capacity, execute latency, executed, process latency, acked, failed dtAutoPage("#executor-stats-table", { columnDefs: [ + {render: renderSupervisorPageLink, searchable: true, targets: [2]}, {render: renderActionCheckbox, searchable: false, targets: [4]}, {type: "num", targets: [5, 6, 7, 8, 9, 10, 11, 12]}, {type: "time-str", targets: [1]}, http://git-wip-us.apache.org/repos/asf/storm/blob/0e0bcf27/storm-core/src/ui/public/css/style.css ---------------------------------------------------------------------- diff --git a/storm-core/src/ui/public/css/style.css b/storm-core/src/ui/public/css/style.css index ddae2d5..c4c41fd 100644 --- a/storm-core/src/ui/public/css/style.css +++ b/storm-core/src/ui/public/css/style.css @@ -112,3 +112,23 @@ PRE.jsonFormatter-codeContainer { width: 1em; } +.worker-component-button { + margin-right: 2px; + margin-top: 2px; +} + +.worker-component-button .badge { + margin-left: 2px; +} + +.worker-child-row { + padding: 10px; +} + +.supervisor-page #toggle-sys { + padding: 10px; +} + +#toggle-on-components-btn .btn { + margin: 10px; +} http://git-wip-us.apache.org/repos/asf/storm/blob/0e0bcf27/storm-core/src/ui/public/js/script.js ---------------------------------------------------------------------- diff --git a/storm-core/src/ui/public/js/script.js b/storm-core/src/ui/public/js/script.js index 769442f..fcdd75a 100644 --- a/storm-core/src/ui/public/js/script.js +++ b/storm-core/src/ui/public/js/script.js @@ -271,3 +271,194 @@ function getStatic(url, cb) { success: cb }); }; + +function makeSupervisorWorkerStatsTable (response, elId, parentId) { + makeWorkerStatsTable (response, elId, parentId, "supervisor"); +}; + +function makeTopologyWorkerStatsTable (response, elId, parentId) { + makeWorkerStatsTable (response, elId, parentId, "topology"); +}; + +var formatComponents = function (row) { + if (!row) return; + var result = ''; + Object.keys(row.componentNumTasks || {}).sort().forEach (function (component){ + var numTasks = row.componentNumTasks[component]; + result += '<a class="worker-component-button btn btn-xs btn-primary" href="/component.html?id=' + + component + '&topology_id=' + row.topologyId + '">'; + result += component; + result += '<span class="badge">' + numTasks + '</span>'; + result += '</a>'; + }); + return result; +}; + +var format = function (row){ + var result = '<div class="worker-child-row">Worker components: '; + result += formatComponents (row) || 'N/A'; + result += '</div>'; + return result; +}; + +// Build a table of per-worker resources and components (when permitted) +var makeWorkerStatsTable = function (response, elId, parentId, type) { + var showCpu = response.schedulerDisplayResource; + + var columns = [ + { + data: 'host', + render: function (data, type, row){ + return type === 'display' ? + ('<a href="/supervisor.html?host=' + data + '">' + data + '</a>') : + data; + } + }, + { + data: 'port', + render: function (data, type, row) { + var logLink = row.workerLogLink; + return type === 'display' ? + ('<a href="' + logLink + '">' + data + '</a>'): + data; + } + }, + { + data: function (row, type){ + // if we are showing or filtering, using the formatted + // uptime, else use the seconds (for sorting) + return (type === 'display' || type === 'filter') ? + row.uptime : + row.uptimeSeconds; + } + }, + { data: 'executorsTotal' }, + { + data: function (row){ + return row.assignedMemOnHeap + row.assignedMemOffHeap; + } + }, + ]; + + if (showCpu) { + columns.push ({ data: 'assignedCpu' }); + } + + columns.push ({ + data: function (row, type, obj, dt) { + var components = Object.keys(row.componentNumTasks || {}); + if (components.length === 0){ + // if no components returned, it means the worker + // topology isn't one the user is authorized to see + return "N/A"; + } + + if (type == 'filter') { + return components; + } + + if (type == 'display') { + // show a button to toggle the component row + return '<button class="btn btn-xs btn-info details-control" type="button">' + + components.length + ' components</button>'; + } + + return components.length; + } + }); + + switch (type){ + case 'topology': + // the topology page has the supervisor id as the second column in the worker table + columns.splice(1, 0, { + data: 'supervisorId', + render: function (data, type, row){ + return type === 'display' ? + ('<a href="/supervisor.html?id=' + data + '">' + data + '</a>') : + data; + } + }); + break; + case 'supervisor': + // the supervisor page has the topology name as the first column in the worker table + columns.unshift ({ + data: function (row, type){ + return type === 'display' ? + ('<a href="/topology.html?id=' + row.topologyId + '">' + row.topologyName + '</a>') : + row.topologyId; + } + }); + break; + } + + var workerStatsTable = dtAutoPage(elId, { + data: response.workers, + autoWidth: false, + columns: columns, + initComplete: function (){ + // add a "Toggle Components" button + renderToggleComponents ($(elId + '_filter'), elId); + var show = $.cookies.get("showComponents") || false; + + // if the cookie is false, then we are done + if (!show) { + return; + } + + // toggle all components visibile + $(elId + ' tr').each(function (){ + var dt = $(elId).dataTable(); + showComponents(dt.api().row(this), true); + }); + } + }); + + // Add event listener for opening and closing components row + // on a per component basis + $(elId + ' tbody').on('click', 'button.details-control', function () { + var tr = $(this).closest('tr'); + var row = workerStatsTable.row(tr); + showComponents(row, !row.child.isShown()); + }); + + $(parentId + ' #toggle-on-components-btn').on('click', 'input', function (){ + toggleComponents(elId); + }); + + $(elId + ' [data-toggle="tooltip"]').tooltip(); +}; + +function renderToggleComponents(div, targetTable) { + var showComponents = $.cookies.get("showComponents") || false; + div.append("<span id='toggle-on-components-btn' class=\"tip right\" " + + "title=\"Use this to toggle visibility of worker components.\">"+ + "<input value=\"Toggle Components\" type=\"button\" class=\"btn btn-info\">" + + "</span>"); +} + +function showComponents(row, open) { + var tr = $(this).closest('tr'); + if (!open) { + // This row is already open - close it + row.child.hide(); + tr.removeClass('shown'); + } else { + // Open this row + row.child (format (row.data())).show(); + tr.addClass('shown'); + } +} + +function toggleComponents(elId) { + var show = $.cookies.get('showComponents') || false; + show = !show; + + var exDate = new Date(); + exDate.setDate(exDate.getDate() + 365); + + $.cookies.set('showComponents', show, {'path':'/', 'expiresAt':exDate.toUTCString()}); + $(elId + ' tr').each(function (){ + var dt = $(elId).dataTable(); + showComponents(dt.api().row(this), show); + }); +} http://git-wip-us.apache.org/repos/asf/storm/blob/0e0bcf27/storm-core/src/ui/public/supervisor.html ---------------------------------------------------------------------- diff --git a/storm-core/src/ui/public/supervisor.html b/storm-core/src/ui/public/supervisor.html new file mode 100644 index 0000000..afe946e --- /dev/null +++ b/storm-core/src/ui/public/supervisor.html @@ -0,0 +1,132 @@ +<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd";> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<html><head> +<meta charset="UTF-8"> +<meta name="viewport" content="width=device-width, initial-scale=1"> +<title>Storm UI</title> +<link href="/css/bootstrap-3.3.1.min.css" rel="stylesheet" type="text/css"> +<link href="/css/jquery.dataTables.1.10.4.min.css" rel="stylesheet" type="text/css"> +<link href="/css/dataTables.bootstrap.css" rel="stylesheet" type="text/css"> +<link href="/css/jsonFormatter.min.css" rel="stylesheet" type="text/css"> +<link href="/css/style.css?_ts=${packageTimestamp}" rel="stylesheet" type="text/css"> +<script src="/js/jquery-1.11.1.min.js" type="text/javascript"></script> +<script src="/js/jquery.dataTables.1.10.4.min.js" type="text/javascript"></script> +<script src="/js/jquery.cookies.2.2.0.min.js" type="text/javascript"></script> +<script src="/js/jquery.mustache.js" type="text/javascript"></script> +<script src="/js/url.min.js" type="text/javascript"></script> +<script src="/js/bootstrap-3.3.1.min.js" type="text/javascript"></script> +<script src="/js/jquery.blockUI.min.js" type="text/javascript"></script> +<script src="/js/jsonFormatter.min.js" type="text/javascript"></script> +<script src="/js/script.js?_ts=${packageTimestamp}" type="text/javascript"></script> +<script src="/js/dataTables.bootstrap.min.js" type="text/javascript"></script> +</head> +<body> +<div class="supervisor-page container-fluid"> + <div class="row"> + <div class="col-md-11"> + <h1><a href="/">Storm UI</a></h1> + </div> + <div id="ui-user" class="col-md-1"></div> + </div> + <div class="row"> + <div class="col-md-12"> + <h2>Supervisor summary</h2> + <div id="supervisor-summary"></div> + </div> + </div> + <div class="row"> + <div class="col-md-12"> + <h2 id="worker-resources-header">Worker resources</h2> + <div id="worker-resources"></div> + </div> + </div> + <div class="row"> + <div class="col-md-12"> + <span id="toggle-sys" style="display: block;" class="js-only"></span> + </div> + </div> +</div> +</body> + +<script> + +$(document).ajaxStop($.unblockUI); +$(document).ajaxStart(function(){ + $.blockUI({ message: '<img src="images/spinner.gif" /> <h3>Loading summary...</h3>'}); +}); +$(document).ready(function() { + var supervisorId = $.url("?id"); + var host = $.url("?host"); + var windowParam = $.url("?window"); + var sys = $.cookies.get("sys") || "false"; + var url = "/api/v1/supervisor?" + + (supervisorId ? "id="+supervisorId : "host="+host) + + "&sys="+sys; + if(windowParam) url += "&window=" + windowParam; + $.extend( $.fn.dataTable.defaults, { + stateSave: true, + lengthMenu: [[20,40,60,100,-1], [20, 40, 60, 100, "All"]], + pageLength: 20 + }); + + renderToggleSys($("#toggle-sys")); + + var supervisorSummary = $("#supervisor-summary"); + var workerStats = $("#worker-resources"); + + $.ajaxSetup({ + "error":function(jqXHR,textStatus,response) { + var errorJson = jQuery.parseJSON(jqXHR.responseText); + getStatic("/templates/json-error-template.html", function(template) { + $("#json-response-error").append(Mustache.render($(template).filter("#json-error-template").html(),errorJson)); + }); + } + }); + function jsError(other) { + try { + other(); + } catch (err) { + getStatic("/templates/json-error-template.html", function(template) { + $("#json-response-error").append(Mustache.render($(template).filter("#json-error-template").html(),{error: "JS Error", errorMessage: err})); + }); + } + } + + $.getJSON(url,function(response,status,jqXHR) { + getStatic("/templates/supervisor-page-template.html", function(template) { + jsError(function() { + supervisorSummary.append( + Mustache.render($(template).filter("#supervisor-summary-template").html(),response)); + + //id, host, uptime, slots, used slots + dtAutoPage("#supervisor-summary-table", { + columnDefs: [ + {type: "num", targets: [3, 4]}, + {type: "time-str", targets: [2]} + ] + }); + + $('#supervisor-summary-table [data-toggle="tooltip"]').tooltip(); + workerStats.append(Mustache.render($(template).filter("#worker-stats-template").html(),response)); + makeSupervisorWorkerStatsTable(response, '#worker-stats-table', '#worker-resources'); + }); + }); + }); +}); +</script> http://git-wip-us.apache.org/repos/asf/storm/blob/0e0bcf27/storm-core/src/ui/public/templates/index-page-template.html ---------------------------------------------------------------------- diff --git a/storm-core/src/ui/public/templates/index-page-template.html b/storm-core/src/ui/public/templates/index-page-template.html index 0574e87..c23838f 100644 --- a/storm-core/src/ui/public/templates/index-page-template.html +++ b/storm-core/src/ui/public/templates/index-page-template.html @@ -252,8 +252,8 @@ <tbody> {{#supervisors}} <tr> - <td><a href="{{logLink}}">{{host}}</a></td> - <td>{{id}}</td> + <td><a href="/supervisor.html?host={{host}}">{{host}}</a> (<a href="{{logLink}}" title="View log">log</a>)</td> + <td><a href="/supervisor.html?id={{id}}">{{id}}</a></td> <td>{{uptime}}</td> <td>{{slotsTotal}}</td> <td>{{slotsUsed}}</td> http://git-wip-us.apache.org/repos/asf/storm/blob/0e0bcf27/storm-core/src/ui/public/templates/supervisor-page-template.html ---------------------------------------------------------------------- diff --git a/storm-core/src/ui/public/templates/supervisor-page-template.html b/storm-core/src/ui/public/templates/supervisor-page-template.html new file mode 100644 index 0000000..9e6fadc --- /dev/null +++ b/storm-core/src/ui/public/templates/supervisor-page-template.html @@ -0,0 +1,145 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<script id="supervisor-summary-template" type="text/html"> + <table class="table table-striped compact" id="supervisor-summary-table"> + <thead> + <tr> + <th> + <span data-toggle="tooltip" data-placement="top" title="The hostname reported by the remote host. (Note that this hostname is not the result of a reverse lookup at the Nimbus node.)"> + Host + </span> + </th> + <th> + <span data-toggle="tooltip" data-placement="top" title="A unique identifier given to a Supervisor when it joins the cluster."> + Supervisor Id + </span> + </th> + <th> + <span data-toggle="tooltip" data-placement="top" title="The length of time a Supervisor has been registered to the cluster."> + Uptime + </span> + </th> + <th> + <span data-toggle="tooltip" data-placement="top" title="Slots are Workers (processes)."> + Slots + </span> + </th> + <th> + <span data-toggle="tooltip" data-placement="top" title="Slots are Workers (processes)."> + Used slots + </span> + </th> + <th> + <span data-toggle="tooltip" data-placement="top" title="Memory capacity of a supervisor."> + Total Mem (MB) + </span> + </th> + <th> + <span data-toggle="tooltip" data-placement="top" title="Memory that has been allocated."> + Used Mem (MB) + </span> + </th> + {{#schedulerDisplayResource}} + <th> + <span data-toggle="tooltip" data-placement="top" title="CPU capacity of a supervisor. Every 100 means one core."> + Total CPU (%) + </span> + </th> + <th> + <span data-toggle="tooltip" data-placement="top" title="CPU that has been allocated. Every 100 means one core"> + Used CPU (%) + </span> + </th> + {{/schedulerDisplayResource}} + <th> + <span data-toggle="tooltip" data-placement="top" title="Version"> + Version + </span> + </th> + </tr> + </thead> + <tbody> + {{#supervisors}} + <tr> + <td><a href="/supervisor.html?host={{host}}">{{host}} (<a href="{{logLink}}" title="View log">log</a>)</a></td> + <td><a href="/supervisor.html?id={{id}}">{{id}}</td> + <td>{{uptime}}</td> + <td>{{slotsTotal}}</td> + <td>{{slotsUsed}}</td> + <td>{{totalMem}}</td> + <td>{{usedMem}}</td> + {{#schedulerDisplayResource}} + <td>{{totalCpu}}</td> + <td>{{usedCpu}}</td> + {{/schedulerDisplayResource}} + <td>{{version}}</td> + </tr> + {{/supervisors}} + </tbody> + </table> +</script> +<script id="worker-stats-template" type="text/html"> + <table class="table table-striped compact" id="worker-stats-table"> + <thead> + <tr> + <th class="header headerSortDown"> + <span data-toggle="tooltip" data-placement="top" title="The name given to the topology by when it was submitted. Click the name to view the Topology's information."> + Topology Name + </span> + </th> + <th class="header"> + <span data-original-title="The hostname reported by the remote host. (Note that this hostname is not the result of a reverse lookup at the Nimbus node.)" data-toggle="tooltip" data-placement="top"> + Host + </span> + </th> + <th class="header"> + <span data-toggle="tooltip" data-placement="top" title="The port number used by the Worker. Click on the port number to open the logviewer page for this Worker."> + Port + </span> + </th> + <th class="header"> + <span data-toggle="tooltip" data-placement="top" title="The length of time a Worker has been alive."> + Uptime + </span> + </th> + <th class="header"> + <span data-toggle="tooltip" data-placement="top" title="The number of executors."> + Num executors + </span> + </th> + <th class="header"> + <span data-toggle="tooltip" data-placement="top" title="Assigned Total Memory by Scheduler."> + Assigned Mem (MB) + </span> + </th> + {{#schedulerDisplayResource}} + <th class="header"> + <span data-toggle="tooltip" data-placement="top" title="Assigned Total CPU by Scheduler. Every 100 means 1 core."> + Assigned CPU (%) + </span> + </th> + {{/schedulerDisplayResource}} + <th class="header"> + <span data-toggle="tooltip" data-placement="top" title="The components running in this worker and the number of tasks per component."> + Components + </span> + </th> + </tr></thead> + <tbody> + </tbody> + </table> +</script> http://git-wip-us.apache.org/repos/asf/storm/blob/0e0bcf27/storm-core/src/ui/public/templates/topology-page-template.html ---------------------------------------------------------------------- diff --git a/storm-core/src/ui/public/templates/topology-page-template.html b/storm-core/src/ui/public/templates/topology-page-template.html index 1825364..f390a21 100644 --- a/storm-core/src/ui/public/templates/topology-page-template.html +++ b/storm-core/src/ui/public/templates/topology-page-template.html @@ -29,45 +29,52 @@ </span> </th> <th> - <span data-toggle="tooltip" data-placement="above" title="The user that submitted the Topology, if authentication is enabled."> + <span data-toggle="tooltip" data-placement="top" title="The user that submitted the Topology, if authentication is enabled."> Owner </span> </th> <th> - <span data-toggle="tooltip" data-placement="above" title="The status can be one of ACTIVE, INACTIVE, KILLED, or REBALANCING."> + <span data-toggle="tooltip" data-placement="top" title="The status can be one of ACTIVE, INACTIVE, KILLED, or REBALANCING."> Status </span> </th> <th> - <span data-toggle="tooltip" data-placement="above" title="The time since the Topology was submitted."> + <span data-toggle="tooltip" data-placement="top" title="The time since the Topology was submitted."> Uptime </span> </th> <th> - <span data-toggle="tooltip" data-placement="above" title="The number of Workers (processes)."> + <span data-toggle="tooltip" data-placement="top" title="The number of Workers (processes)."> Num workers </span> </th> <th> - <span data-toggle="tooltip" data-placement="above" title="Executors are threads in a Worker process."> + <span data-toggle="tooltip" data-placement="top" title="Executors are threads in a Worker process."> Num executors </span> </th> <th> - <span data-toggle="tooltip" data-placement="above" title="A Task is an instance of a Bolt or Spout. The number of Tasks is almost always equal to the number of Executors."> + <span data-toggle="tooltip" data-placement="top" title="A Task is an instance of a Bolt or Spout. The number of Tasks is almost always equal to the number of Executors."> Num tasks </span> </th> <th> - <span cdata-toggle="tooltip" data-placement="above" title="Number of nimbus hosts on which this topology's code is replicated. "> + <span cdata-toggle="tooltip" data-placement="top" title="Number of nimbus hosts on which this topology's code is replicated. "> Replication count </span> </th> <th> - <span data-toggle="tooltip" data-placement="above" title="Assigned Total Memory by Scheduler."> + <span data-toggle="tooltip" data-placement="top" title="Assigned Total Memory by Scheduler."> Assigned Mem (MB) </span> </th> + {{#schedulerDisplayResource}} + <th> + <span data-toggle="tooltip" data-placement="top" title="Assigned Total CPU by Scheduler. Every 100 means 1 core."> + Assigned CPU (%) + </span> + </th> + {{/schedulerDisplayResource}} <th> <span data-toggle="tooltip" data-placement="left" title="This shows information from the scheduler about the latest attempt to schedule the Topology on the cluster."> Scheduler Info @@ -87,6 +94,9 @@ <td>{{tasksTotal}}</td> <td>{{replicationCount}}</td> <td>{{assignedTotalMem}}</td> + {{#schedulerDisplayResource}} + <td>{{assignedCpu}}</td> + {{/schedulerDisplayResource}} <td>{{schedulerInfo}}</td> </tr> </tbody> @@ -95,48 +105,48 @@ <script id="topology-resources-template" type="text/html"> <table id="topology-resources-table" class="table compact"> <thead> - <tr> - <th> - <span data-toggle="tooltip" data-placement="right" title="The name given to the topology by when it was submitted."> - Name - </span> - </th> - <th> - <span data-toggle="tooltip" data-placement="right" title="The unique ID given to a Topology each time it is launched."> - Id - </span> - </th> - <th> - <span data-toggle="tooltip" data-placement="above" title="Requested Total On-Heap Memory by User."> - Requested On-Heap Memory (MB) - </span> - </th> - <th> - <span data-toggle="tooltip" data-placement="above" title="Assigned Total On-Heap Memory by Scheduler."> - Assigned On-Heap Memory (MB) - </span> - </th> - <th> - <span data-toggle="tooltip" data-placement="above" title="Requested Total Off-Heap Memory by User."> - Requested Off-Heap Memory (MB) - </span> - </th> - <th> - <span data-toggle="tooltip" data-placement="above" title="Assigned Total Off-Heap Memory by Scheduler."> - Assigned Off-Heap Memory (MB) - </span> - </th> - <th> - <span data-toggle="tooltip" data-placement="above" title="Requested Total CPU by User. Every 100 means 1 core."> - Requested CPU (%) - </span> - </th> - <th> - <span data-toggle="tooltip" data-placement="left" title="Assigned Total CPU by Scheduler. Every 100 means 1 core."> - Assigned CPU (%) - </span> - </th> - </tr> + <tr> + <th> + <span data-toggle="tooltip" data-placement="right" title="The name given to the topology by when it was submitted."> + Name + </span> + </th> + <th> + <span data-toggle="tooltip" data-placement="right" title="The unique ID given to a Topology each time it is launched."> + Id + </span> + </th> + <th> + <span data-toggle="tooltip" data-placement="top" title="Requested Total On-Heap Memory by User."> + Requested On-Heap Memory (MB) + </span> + </th> + <th> + <span data-toggle="tooltip" data-placement="top" title="Assigned Total On-Heap Memory by Scheduler."> + Assigned On-Heap Memory (MB) + </span> + </th> + <th> + <span data-toggle="tooltip" data-placement="top" title="Requested Total Off-Heap Memory by User."> + Requested Off-Heap Memory (MB) + </span> + </th> + <th> + <span data-toggle="tooltip" data-placement="top" title="Assigned Total Off-Heap Memory by Scheduler."> + Assigned Off-Heap Memory (MB) + </span> + </th> + <th> + <span data-toggle="tooltip" data-placement="top" title="Requested Total CPU by User. Every 100 means 1 core."> + Requested CPU (%) + </span> + </th> + <th> + <span data-toggle="tooltip" data-placement="left" title="Assigned Total CPU by Scheduler. Every 100 means 1 core."> + Assigned CPU (%) + </span> + </th> + </tr> </thead> <tbody> <tr> @@ -163,22 +173,22 @@ </span> </th> <th> - <span data-toggle="tooltip" data-placement="above" title="The number of Tuples emitted."> + <span data-toggle="tooltip" data-placement="top" title="The number of Tuples emitted."> Emitted </span> </th> <th> - <span data-toggle="tooltip" data-placement="above" title="The number of Tuples emitted that sent to one or more bolts."> + <span data-toggle="tooltip" data-placement="top" title="The number of Tuples emitted that sent to one or more bolts."> Transferred </span> </th> <th> - <span data-toggle="tooltip" data-placement="above" title="The average time a Tuple "tree" takes to be completely processed by the Topology. A value of 0 is expected if no acking is done."> + <span data-toggle="tooltip" data-placement="top" title="The average time a Tuple "tree" takes to be completely processed by the Topology. A value of 0 is expected if no acking is done."> Complete latency (ms) </span> </th> <th> - <span data-toggle="tooltip" data-placement="above" title="The number of Tuple "trees" successfully processed. A value of 0 is expected if no acking is done."> + <span data-toggle="tooltip" data-placement="top" title="The number of Tuple "trees" successfully processed. A value of 0 is expected if no acking is done."> Acked </span> </th> @@ -215,22 +225,22 @@ </span> </th> <th> - <span data-toggle="tooltip" data-placement="above" title="Topic"> + <span data-toggle="tooltip" data-placement="top" title="Topic"> Topic </span> </th> <th> - <span data-toggle="tooltip" data-placement="above" title="Partition"> + <span data-toggle="tooltip" data-placement="top" title="Partition"> Partition </span> </th> <th> - <span data-toggle="tooltip" data-placement="above" title="Latest Offset"> + <span data-toggle="tooltip" data-placement="top" title="Latest Offset"> Latest Offset </span> </th> <th> - <span data-toggle="tooltip" data-placement="above" title="Offset of last spout message successfully acked"> + <span data-toggle="tooltip" data-placement="top" title="Offset of last spout message successfully acked"> Spout Committed Offset </span> </th> @@ -267,7 +277,7 @@ </span> </th> <th> - <span data-toggle="tooltip" data-placement="above" title="Type of spout"> + <span data-toggle="tooltip" data-placement="top" title="Type of spout"> Type </span> </th> @@ -353,32 +363,32 @@ </span> </th> <th class="header"> - <span data-toggle="tooltip" data-placement="above" title="A Task is an instance of a Bolt or Spout. The number of Tasks is almost always equal to the number of Executors."> + <span data-toggle="tooltip" data-placement="top" title="A Task is an instance of a Bolt or Spout. The number of Tasks is almost always equal to the number of Executors."> Tasks </span> </th> <th class="header"> - <span data-toggle="tooltip" data-placement="above" title="The number of Tuples emitted."> + <span data-toggle="tooltip" data-placement="top" title="The number of Tuples emitted."> Emitted </span> </th> <th class="header"> - <span data-toggle="tooltip" data-placement="above" title="The number of Tuples emitted that sent to one or more bolts."> + <span data-toggle="tooltip" data-placement="top" title="The number of Tuples emitted that sent to one or more bolts."> Transferred </span> </th> <th class="header"> - <span data-toggle="tooltip" data-placement="above" title="The average time a Tuple "tree" takes to be completely processed by the Topology. A value of 0 is expected if no acking is done."> + <span data-toggle="tooltip" data-placement="top" title="The average time a Tuple "tree" takes to be completely processed by the Topology. A value of 0 is expected if no acking is done."> Complete latency (ms) </span> </th> <th class="header"> - <span data-toggle="tooltip" data-placement="above" title="The number of Tuple "trees" successfully processed. A value of 0 is expected if no acking is done."> + <span data-toggle="tooltip" data-placement="top" title="The number of Tuple "trees" successfully processed. A value of 0 is expected if no acking is done."> Acked </span> </th> <th class="header"> - <span data-toggle="tooltip" data-placement="above" title="The number of Tuple "trees" that were explicitly failed or timed out before acking was completed. A value of 0 is expected if no acking is done."> + <span data-toggle="tooltip" data-placement="top" title="The number of Tuple "trees" that were explicitly failed or timed out before acking was completed. A value of 0 is expected if no acking is done."> Failed </span> </th> @@ -432,42 +442,42 @@ </span> </th> <th class="header"> - <span data-toggle="tooltip" data-placement="above" title="A Task is an instance of a Bolt or Spout. The number of Tasks is almost always equal to the number of Executors."> + <span data-toggle="tooltip" data-placement="top" title="A Task is an instance of a Bolt or Spout. The number of Tasks is almost always equal to the number of Executors."> Tasks </span> </th> <th class="header"> - <span data-toggle="tooltip" data-placement="above" title="The number of Tuples emitted."> + <span data-toggle="tooltip" data-placement="top" title="The number of Tuples emitted."> Emitted </span> </th> <th class="header"> - <span data-toggle="tooltip" data-placement="above" title="The number of Tuples emitted that sent to one or more bolts."> + <span data-toggle="tooltip" data-placement="top" title="The number of Tuples emitted that sent to one or more bolts."> Transferred </span> </th> <th class="header"> - <span data-original-title="If this is around 1.0, the corresponding Bolt is running as fast as it can, so you may want to increase the Bolt's parallelism. This is (number executed * average execute latency) / measurement time." data-toggle="tooltip" data-placement="above"> + <span data-original-title="If this is around 1.0, the corresponding Bolt is running as fast as it can, so you may want to increase the Bolt's parallelism. This is (number executed * average execute latency) / measurement time." data-toggle="tooltip" data-placement="top"> Capacity (last 10m) </span> </th> <th class="header"> - <span data-toggle="tooltip" data-placement="above" title="The average time a Tuple spends in the execute method. The execute method may complete without sending an Ack for the tuple."> + <span data-toggle="tooltip" data-placement="top" title="The average time a Tuple spends in the execute method. The execute method may complete without sending an Ack for the tuple."> Execute latency (ms) </span> </th> <th class="header"> - <span data-toggle="tooltip" data-placement="above" title="The number of incoming Tuples processed."> + <span data-toggle="tooltip" data-placement="top" title="The number of incoming Tuples processed."> Executed </span> </th> <th class="header"> - <span data-toggle="tooltip" data-placement="above" title="The average time it takes to Ack a Tuple after it is first received. Bolts that join, aggregate or batch may not Ack a tuple until a number of other Tuples have been received."> + <span data-toggle="tooltip" data-placement="top" title="The average time it takes to Ack a Tuple after it is first received. Bolts that join, aggregate or batch may not Ack a tuple until a number of other Tuples have been received."> Process latency (ms) </span> </th> <th class="header"> - <span data-toggle="tooltip" data-placement="above" title="The number of Tuples acknowledged by this Bolt."> + <span data-toggle="tooltip" data-placement="top" title="The number of Tuples acknowledged by this Bolt."> Acked </span> </th> @@ -512,7 +522,57 @@ {{/bolts}} </tbody> </script> - +<script id="worker-stats-template" type="text/html"> + <h2>Worker Resources</h2> + <table class="table table-striped compact" id="worker-stats-table"> + <thead> + <tr> + <th class="header headerSortDown"> + <span data-original-title="The hostname reported by the remote host. (Note that this hostname is not the result of a reverse lookup at the Nimbus node.)" data-toggle="tooltip" data-placement="top"> + Host + </span> + </th> + <th class="header"> + <span data-toggle="tooltip" data-placement="top" title="A unique identifier given to a Supervisor when it joins the cluster."> + Supervisor Id + </span> + </th> + <th class="header"> + <span data-toggle="tooltip" data-placement="top" title="The port number used by the Worker. Click on the port number to open the logviewer page for this Worker."> + Port + </span> + </th> + <th class="header"> + <span data-toggle="tooltip" data-placement="top" title="The length of time a Worker has been alive."> + Uptime + </span> + </th> + <th class="header"> + <span data-toggle="tooltip" data-placement="top" title="The number of executors"> + Num executors + </span> + </th> + <th class="header"> + <span data-toggle="tooltip" data-placement="top" title="Assigned Total Memory by Scheduler."> + Assigned Mem (MB) + </span> + </th> + {{#schedulerDisplayResource}} + <th class="header"> + <span data-toggle="tooltip" data-placement="top" title="Assigned Total CPU by Scheduler. Every 100 means 1 core."> + Assigned CPU (%) + </span> + </th> + {{/schedulerDisplayResource}} + <th class="header"> + <span data-toggle="tooltip" data-placement="top" title="The components running in this worker and the number of tasks per component."> + Components + </span> + </th> + </tr></thead> + <tbody> + </tbody> +</script> <script id="topology-actions-template" type="text/html"> <h2>Topology actions</h2> <p id="topology-actions"> http://git-wip-us.apache.org/repos/asf/storm/blob/0e0bcf27/storm-core/src/ui/public/topology.html ---------------------------------------------------------------------- diff --git a/storm-core/src/ui/public/topology.html b/storm-core/src/ui/public/topology.html index dc3c0b3..bb1279f 100644 --- a/storm-core/src/ui/public/topology.html +++ b/storm-core/src/ui/public/topology.html @@ -80,6 +80,9 @@ <div id="bolt-stats" class="col-md-12"></div> </div> <div class="row"> + <div id="worker-stats" class="col-md-12"></div> + </div> + <div class="row"> <div id="topology-visualization" class="col-md-12"></div> </div> <div class="row"> @@ -284,6 +287,7 @@ $(document).ready(function() { var topologySpoutsLag = $("#topology-spouts-lag"); var spoutStats = $("#spout-stats"); var boltStats = $("#bolt-stats"); + var workerStats = $("#worker-stats"); var config = $("#topology-configuration"); var topologyActions = $("#topology-actions"); var topologyVisualization = $("#topology-visualization") @@ -332,6 +336,12 @@ $(document).ready(function() { ] }); + jsError(function() { + workerStats.append(Mustache.render($(template).filter("#worker-stats-template").html(),response)); + makeTopologyWorkerStatsTable (response, '#worker-stats-table', '#worker-stats'); + }); + + jsError(function() { topologyVisualization.append(Mustache.render($(template).filter("#topology-visualization-template").html(), response)); $("#show-hide-visualization").click(function () { show_visualization(null) }); @@ -439,9 +449,9 @@ $(document).ready(function() { } } }); + }); }}); }); - }); </script> </html> http://git-wip-us.apache.org/repos/asf/storm/blob/0e0bcf27/storm-core/test/clj/org/apache/storm/nimbus_test.clj ---------------------------------------------------------------------- diff --git a/storm-core/test/clj/org/apache/storm/nimbus_test.clj b/storm-core/test/clj/org/apache/storm/nimbus_test.clj index dbb89bd..4daa556 100644 --- a/storm-core/test/clj/org/apache/storm/nimbus_test.clj +++ b/storm-core/test/clj/org/apache/storm/nimbus_test.clj @@ -26,7 +26,7 @@ (:import [org.apache.storm.generated Credentials NotAliveException SubmitOptions TopologyInitialStatus TopologyStatus AlreadyAliveException KillOptions RebalanceOptions InvalidTopologyException AuthorizationException - LogConfig LogLevel LogLevelAction]) + LogConfig LogLevel LogLevelAction NodeInfo]) (:import [java.util HashMap]) (:import [java.io File]) (:import [org.apache.storm.utils Time Utils]) @@ -1214,7 +1214,42 @@ nimbus/check-authorization! [1 2 3] expected-name expected-conf expected-operation) (verify-first-call-args-for-indices - nimbus/try-read-storm-topology [0] "fake-id")))))))))) + nimbus/try-read-storm-topology [0] "fake-id")))))) + + (testing "getSupervisorPageInfo only calls check-authorization as getTopology" + (let [expected-operation "getTopology" + assignment {:executor->node+port {[1 1] ["super1" "host1"], + [2 2] ["super2" "host1"]}} + topo-assignment {expected-name assignment} + check-auth-state (atom []) + mock-check-authorization (fn [nimbus storm-name storm-conf operation] + (swap! check-auth-state conj {:nimbus nimbus + :storm-name storm-name + :storm-conf storm-conf + :operation operation}))] + (stubbing [nimbus/check-authorization! mock-check-authorization + nimbus/try-read-storm-conf expected-conf + nimbus/try-read-storm-topology nil + storm-task-info nil + nimbus/all-supervisor-info {"super1" {:hostname "host1", :meta [1234], :uptime-secs 123} + "super2" {:hostname "host2", :meta [1234], :uptime-secs 123}} + nimbus/topology-assignments topo-assignment + nimbus/get-launch-time-secs 0] + ;; not called yet + (verify-call-times-for nimbus/check-authorization! 0) + (.getSupervisorPageInfo nimbus "super1" nil true) + + ;; afterwards, it should get called twice + (verify-call-times-for nimbus/check-authorization! 2) + (let [first-call (nth @check-auth-state 0) + second-call (nth @check-auth-state 1)] + (is (= expected-name (:storm-name first-call))) + (is (= expected-conf (:storm-conf first-call))) + (is (= "getTopology" (:operation first-call))) + + (is (= expected-name (:storm-name second-call))) + (is (= expected-conf (:storm-conf second-call))) + (is (= "getSupervisorPageInfo" (:operation second-call))))))))))) (deftest test-nimbus-iface-getTopology-methods-throw-correctly (with-local-cluster [cluster] @@ -1271,7 +1306,8 @@ :status {:type bogus-type}} } ] - (stubbing [topology-bases bogus-bases + (stubbing [nimbus/get-resources-for-topology nil + topology-bases bogus-bases nimbus/get-blob-replication-count 1] (let [topos (.get_topologies (.getClusterInfo nimbus))] ; The number of topologies in the summary is correct. @@ -1614,3 +1650,33 @@ (is (= (count @hb-cache) 2)) (is (contains? @hb-cache "topo1")) (is (contains? @hb-cache "topo2")))))) + + +(deftest user-topologies-for-supervisor + (let [assignment {:executor->node+port {[1 1] ["super1" "host1"], + [2 2] ["super2" "host2"]}} + assignment2 {:executor->node+port {[1 1] ["super2" "host2"], + [2 2] ["super2" "host2"]}} + assignments {"topo1" assignment, "topo2" assignment2}] + (stubbing [nimbus/is-authorized? true] + (let [topos1 (nimbus/user-and-supervisor-topos nil nil nil assignments "super1") + topos2 (nimbus/user-and-supervisor-topos nil nil nil assignments "super2")] + (is (= (list "topo1") (:supervisor-topologies topos1))) + (is (= #{"topo1"} (:user-topologies topos1))) + (is (= (list "topo1" "topo2") (:supervisor-topologies topos2))) + (is (= #{"topo1" "topo2"} (:user-topologies topos2))))))) + +(defn- mock-check-auth + [nimbus conf blob-store op topo-name] + (= topo-name "authorized")) + +(deftest user-topologies-for-supervisor-with-unauthorized-user + (let [assignment {:executor->node+port {[1 1] ["super1" "host1"], + [2 2] ["super2" "host2"]}} + assignment2 {:executor->node+port {[1 1] ["super1" "host1"], + [2 2] ["super2" "host2"]}} + assignments {"topo1" assignment, "authorized" assignment2}] + (stubbing [nimbus/is-authorized? mock-check-auth] + (let [topos (nimbus/user-and-supervisor-topos nil nil nil assignments "super1")] + (is (= (list "topo1" "authorized") (:supervisor-topologies topos))) + (is (= #{"authorized"} (:user-topologies topos))))))) http://git-wip-us.apache.org/repos/asf/storm/blob/0e0bcf27/storm-core/test/clj/org/apache/storm/stats_test.clj ---------------------------------------------------------------------- diff --git a/storm-core/test/clj/org/apache/storm/stats_test.clj b/storm-core/test/clj/org/apache/storm/stats_test.clj new file mode 100644 index 0000000..5c1ec50 --- /dev/null +++ b/storm-core/test/clj/org/apache/storm/stats_test.clj @@ -0,0 +1,134 @@ +;; Licensed to the Apache Software Foundation (ASF) under one +;; or more contributor license agreements. See the NOTICE file +;; distributed with this work for additional information +;; regarding copyright ownership. The ASF licenses this file +;; to you under the Apache License, Version 2.0 (the +;; "License"); you may not use this file except in compliance +;; with the License. You may obtain a copy of the License at +;; +;; http://www.apache.org/licenses/LICENSE-2.0 +;; +;; Unless required by applicable law or agreed to in writing, software +;; distributed under the License is distributed on an "AS IS" BASIS, +;; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +;; See the License for the specific language governing permissions and +;; limitations under the License. +(ns org.apache.storm.stats-test + (:use [clojure test]) + (:import [org.apache.storm.scheduler WorkerSlot]) + (:import [org.apache.storm.generated WorkerResources]) + (:require [org.apache.storm [stats :as stats]])) + +(defn- make-topo-info-no-beats + [] + {:storm-name "testing", + :assignment {:executor->node+port {[1 3] ["node" 1234] + [4 4] ["node" 1234]} + :node->host {"node" "host"}}}) + +(defn- make-topo-info + [] + (merge + {:beats {[1 3] {:uptime 6} + [4 4] {:uptime 6}}} + {:task->component {1 "exclaim1", 2 "__sys", 3 "exclaim1", 4 "__sys2"}} + (make-topo-info-no-beats))) + +(defn- make-worker-resources + [] + (doto (WorkerResources.) + (.set_mem_on_heap 3) + (.set_mem_off_heap 4) + (.set_cpu 5))) + +(deftest agg-worker-populates-worker-summary + (let [storm-id "foo" + topo-info (make-topo-info) + worker->resources {(WorkerSlot. "node" 1234) (make-worker-resources)} + include-sys? true + user-authorized true + worker-summaries (stats/agg-worker-stats storm-id + topo-info + worker->resources + include-sys? + user-authorized)] + (let [summ (first worker-summaries) + comps (.get_component_to_num_tasks summ)] + (is (= 1 (count worker-summaries))) + (is (= "host" (.get_host summ))) + (is (= 6 (.get_uptime_secs summ))) + (is (= "node" (.get_supervisor_id summ))) + (is (= 1234 (.get_port summ))) + (is (= "foo" (.get_topology_id summ))) + (is (= "testing" (.get_topology_name summ))) + (is (= 2 (.get_num_executors summ))) + (is (= 3.0 (.get_assigned_memonheap summ))) + (is (= 4.0 (.get_assigned_memoffheap summ))) + (is (= 5.0 (.get_assigned_cpu summ))) + ;; agg-worker-stats groups the components together + (is (= 2 (get comps "exclaim1"))) + (is (= 1 (get comps "__sys")))))) + +(deftest agg-worker-skips-sys-if-not-enabled + (let [storm-id "foo" + topo-info (make-topo-info) + worker->resources {(WorkerSlot. "node" 1234) (make-worker-resources)} + include-sys? false + user-authorized true + worker-summaries (stats/agg-worker-stats storm-id + topo-info + worker->resources + include-sys? + user-authorized)] + (let [summ (first worker-summaries) + comps (.get_component_to_num_tasks summ)] + (is (= nil (get comps "__sys"))) + (is (= 2 (.get_num_executors summ))) + (is (= 2 (get comps "exclaim1")))))) + +(deftest agg-worker-gracefully-handles-missing-beats + (let [storm-id "foo" + topo-info (make-topo-info-no-beats) + worker->resources {(WorkerSlot. "node" 1234) (make-worker-resources)} + include-sys? false + user-authorized true + worker-summaries (stats/agg-worker-stats storm-id + topo-info + worker->resources + include-sys? + user-authorized)] + (let [summ (first worker-summaries)] + (is (= 0 (.get_uptime_secs summ)))))) + +(deftest agg-worker-stats-exclude-components-if-not-authorized + (let [storm-id "foo" + topo-info (make-topo-info-no-beats) + worker->resources {(WorkerSlot. "node" 1234) (make-worker-resources)} + include-sys? false + user-authorized false + worker-summaries (stats/agg-worker-stats storm-id + topo-info + worker->resources + include-sys? + user-authorized)] + (let [summ (first worker-summaries)] + (is (= 0 (.get_uptime_secs summ))) + (is (= nil (.get_component_to_num_tasks summ)))))) + +(deftest agg-worker-stats-can-handle-nil-worker->resources + (let [storm-id "foo" + topo-info (make-topo-info-no-beats) + worker->resources nil + include-sys? false + user-authorized false + worker-summaries (stats/agg-worker-stats storm-id + topo-info + worker->resources + include-sys? + user-authorized)] + (let [summ (first worker-summaries)] + (is (= 0 (.get_uptime_secs summ))) + (is (= 0.0 (.get_assigned_memonheap summ))) + (is (= 0.0 (.get_assigned_memoffheap summ))) + (is (= 0.0 (.get_assigned_cpu summ))) + (is (= nil (.get_component_to_num_tasks summ))))))
