AMBARI-18439 - [Grafana] Add Kafka-Offset dashboard for Storm (prajwal)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/7c8ada18 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/7c8ada18 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/7c8ada18 Branch: refs/heads/branch-dev-patch-upgrade Commit: 7c8ada187d9f80ecc93a901b403c4e13821888b5 Parents: 69e8f6f Author: Prajwal Rao <[email protected]> Authored: Mon Sep 26 13:54:08 2016 -0700 Committer: Prajwal Rao <[email protected]> Committed: Mon Sep 26 13:54:08 2016 -0700 ---------------------------------------------------------------------- .../ambari-metrics/datasource.js | 64 +++++ .../HDF/grafana-storm-kafka-offset.json | 258 +++++++++++++++++++ .../HDP/grafana-storm-kafka-offset.json | 258 +++++++++++++++++++ 3 files changed, 580 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/7c8ada18/ambari-metrics/ambari-metrics-grafana/ambari-metrics/datasource.js ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-grafana/ambari-metrics/datasource.js b/ambari-metrics/ambari-metrics-grafana/ambari-metrics/datasource.js index fa0cc47..626439d 100644 --- a/ambari-metrics/ambari-metrics-grafana/ambari-metrics/datasource.js +++ b/ambari-metrics/ambari-metrics-grafana/ambari-metrics/datasource.js @@ -418,6 +418,21 @@ define([ })); } + //Templatized Dashboard for Storm Kafka Offset + if (templateSrv.variables[0].query === "topologies" && templateSrv.variables[1] && + templateSrv.variables[1].name === "topic") { + var selectedTopology = templateSrv._values.topologies; + var selectedTopic = templateSrv._values.topic; + metricsPromises.push(_.map(options.targets, function(target) { + target.sTopology = selectedTopology; + target.sTopic = selectedTopic; + target.sPartition = options.scopedVars.partition.value; + target.sTopoMetric = target.metric.replace('*', target.sTopology).replace('*', target.sTopic) + .replace('*', target.sPartition); + return getStormData(target); + })); + } + // To speed up querying on templatized dashboards. if (templateSrv.variables[1] && templateSrv.variables[1].name === "hosts") { var allHosts = templateSrv._values.hosts.lastIndexOf('}') > 0 ? templateSrv._values.hosts.slice(1,-1) : @@ -616,6 +631,55 @@ define([ }); }); } + var stormEntities = {}; + AmbariMetricsDatasource.prototype.getStormEntities = function () { + return this.initMetricAppidMapping() + .then(function () { + var storm = allMetrics["nimbus"]; + var extractTopologies = storm.filter(/./.test.bind(new + RegExp("partition", 'g'))); + _.map(extractTopologies, function(topology){ + topology = topology.split('.').slice(0,5); + var topologyName = topologyN = topology[1]; // Topology + var topologyTopicName = topicN = topology[3]; // Topic + var topologyTopicPartitionName = topology[4]; // Partition + if (stormEntities[topologyName]) { + if (stormEntities[topologyName][topologyTopicName]) { + stormEntities[topologyName][topologyTopicName].push(topologyTopicPartitionName); + } else { + stormEntities[topologyName][topologyTopicName] = [topologyTopicPartitionName]; + } + } else { + stormEntities[topologyName] = {}; + stormEntities[topologyName][topologyTopicName] = [topologyTopicPartitionName]; + } + }); + }); + }; + //Templated Variables for Storm Topics per Topology + if (interpolated.includes("stormTopic")) { + var topicName = interpolated.substring(0,interpolated.indexOf('.')); + return this.getStormEntities().then(function () { + var topicNames = Object.keys(stormEntities[topicName]); + return _.map(topicNames, function(names){ + return { + text: names + }; + }); + }); + } + //Templated Variables for Storm Partitions per Topic + if (interpolated.includes("stormPartition")) { + var topicN, topologyN; + return this.getStormEntities().then(function () { + var partitionNames = _.uniq(stormEntities[topologyN][topicN]); + return _.map(partitionNames, function(names){ + return { + text: names + }; + }); + }); + } // Templated Variable for YARN Queues. // It will search the cluster and populate the queues. if(interpolated === "yarnqueues") { http://git-wip-us.apache.org/repos/asf/ambari/blob/7c8ada18/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/files/grafana-dashboards/HDF/grafana-storm-kafka-offset.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/files/grafana-dashboards/HDF/grafana-storm-kafka-offset.json b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/files/grafana-dashboards/HDF/grafana-storm-kafka-offset.json new file mode 100644 index 0000000..ac1f829 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/files/grafana-dashboards/HDF/grafana-storm-kafka-offset.json @@ -0,0 +1,258 @@ +{ + "id": null, + "title": "Storm - Kafka-Offset", + "originalTitle": "Storm - Kafka-Offset", + "tags": [ + "storm", + "builtin", + "2.4.0.0" + ], + "style": "dark", + "timezone": "browser", + "editable": true, + "hideControls": false, + "sharedCrosshair": false, + "rows": [ + { + "collapse": false, + "editable": true, + "height": "25px", + "panels": [ + { + "content": "<h4 align=\"center\">Metrics to see the status for the Storm topics on a per partition level. Click on each row title to expand on demand to look at various metrics. </h4>\n<h6 style=\"color:red;\" align=\"center\">This dashboard is managed by Ambari. You may lose any changes made to this dashboard. If you want to customize, make your own copy.</h6>\n<h5 align=\"center\">Note: Period ('.') contained topology names are not supported.</h5>", + "editable": true, + "error": false, + "id": 14, + "isNew": true, + "links": [], + "mode": "html", + "span": 12, + "style": {}, + "title": "", + "type": "text" + } + ], + "title": "New row" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "leftLogBase": 1, + "leftMax": null, + "leftMin": null, + "rightLogBase": 1, + "rightMax": null, + "rightMin": null, + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "id": 15, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "minSpan": 4, + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "partition", + "scopedVars": { + "partition": { + "text": "partition-0", + "value": "partition-0", + "selected": true + } + }, + "seriesOverrides": [], + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "aggregator": "avg", + "alias": "Kafka Offset", + "app": "nimbus", + "downsampleAggregator": "avg", + "errors": {}, + "hosts": "", + "metric": "topology.*.kafka-topic.*.*.latestTimeOffset", + "precision": "default", + "refId": "B", + "sPartition": "partition-0", + "sTopic": "myKafkaTopic", + "sTopoMetric": "topology.kafka-topology.kafka-topic.myKafkaTopic.partition-0.latestTimeOffset", + "sTopology": "kafka-topology", + "seriesAggregator": "none", + "transform": "none" + }, + { + "aggregator": "avg", + "alias": "Storm Offset", + "app": "nimbus", + "downsampleAggregator": "avg", + "errors": {}, + "metric": "topology.*.kafka-topic.*.*.latestCompletedOffset", + "precision": "default", + "refId": "A", + "sComponent": "--acker", + "sPartition": "partition-0", + "sTopic": "myKafkaTopic", + "sTopoMetric": "topology.kafka-topology.kafka-topic.myKafkaTopic.partition-0.latestCompletedOffset", + "sTopology": "kafka-topology", + "seriesAggregator": "none", + "transform": "none" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "$partition", + "tooltip": { + "shared": true, + "value_type": "cumulative" + }, + "type": "graph", + "x-axis": true, + "y-axis": true, + "y_formats": [ + "short", + "short" + ] + } + ], + "title": "New row" + } + ], + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "now": true, + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "templating": { + "list": [ + { + "allFormat": "glob", + "current": { + "text": "kafka-topology", + "value": "kafka-topology" + }, + "datasource": null, + "includeAll": false, + "multi": false, + "multiFormat": "glob", + "name": "topologies", + "options": [ + ], + "query": "topologies", + "refresh": true, + "regex": "", + "type": "query" + }, + { + "allFormat": "glob", + "current": { + "text": "", + "value": "" + }, + "datasource": null, + "includeAll": false, + "multi": false, + "multiFormat": "glob", + "name": "topic", + "options": [ + + ], + "query": "$topologies.stormTopic", + "refresh": true, + "regex": "", + "type": "query" + }, + { + "allFormat": "glob", + "current": { + "text": "partition-0", + "value": "partition-0" + }, + "datasource": null, + "includeAll": true, + "multi": true, + "multiFormat": "glob", + "name": "partition", + "options": [ + { + "text": "partition-0", + "value": "partition-0", + "selected": true + } + ], + "query": "$topic.stormPartition", + "refresh": true, + "type": "query" + } + ] + }, + "annotations": { + "list": [] + }, + "refresh": false, + "schemaVersion": 8, + "version": 23, + "links": [ + { + "asDropdown": true, + "icon": "external link", + "tags": [ + "storm" + ], + "title": "Storm Dashboards", + "type": "dashboards" + } + ] +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/7c8ada18/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/files/grafana-dashboards/HDP/grafana-storm-kafka-offset.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/files/grafana-dashboards/HDP/grafana-storm-kafka-offset.json b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/files/grafana-dashboards/HDP/grafana-storm-kafka-offset.json new file mode 100644 index 0000000..ac1f829 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/files/grafana-dashboards/HDP/grafana-storm-kafka-offset.json @@ -0,0 +1,258 @@ +{ + "id": null, + "title": "Storm - Kafka-Offset", + "originalTitle": "Storm - Kafka-Offset", + "tags": [ + "storm", + "builtin", + "2.4.0.0" + ], + "style": "dark", + "timezone": "browser", + "editable": true, + "hideControls": false, + "sharedCrosshair": false, + "rows": [ + { + "collapse": false, + "editable": true, + "height": "25px", + "panels": [ + { + "content": "<h4 align=\"center\">Metrics to see the status for the Storm topics on a per partition level. Click on each row title to expand on demand to look at various metrics. </h4>\n<h6 style=\"color:red;\" align=\"center\">This dashboard is managed by Ambari. You may lose any changes made to this dashboard. If you want to customize, make your own copy.</h6>\n<h5 align=\"center\">Note: Period ('.') contained topology names are not supported.</h5>", + "editable": true, + "error": false, + "id": 14, + "isNew": true, + "links": [], + "mode": "html", + "span": 12, + "style": {}, + "title": "", + "type": "text" + } + ], + "title": "New row" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "leftLogBase": 1, + "leftMax": null, + "leftMin": null, + "rightLogBase": 1, + "rightMax": null, + "rightMin": null, + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "id": 15, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "minSpan": 4, + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "partition", + "scopedVars": { + "partition": { + "text": "partition-0", + "value": "partition-0", + "selected": true + } + }, + "seriesOverrides": [], + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "aggregator": "avg", + "alias": "Kafka Offset", + "app": "nimbus", + "downsampleAggregator": "avg", + "errors": {}, + "hosts": "", + "metric": "topology.*.kafka-topic.*.*.latestTimeOffset", + "precision": "default", + "refId": "B", + "sPartition": "partition-0", + "sTopic": "myKafkaTopic", + "sTopoMetric": "topology.kafka-topology.kafka-topic.myKafkaTopic.partition-0.latestTimeOffset", + "sTopology": "kafka-topology", + "seriesAggregator": "none", + "transform": "none" + }, + { + "aggregator": "avg", + "alias": "Storm Offset", + "app": "nimbus", + "downsampleAggregator": "avg", + "errors": {}, + "metric": "topology.*.kafka-topic.*.*.latestCompletedOffset", + "precision": "default", + "refId": "A", + "sComponent": "--acker", + "sPartition": "partition-0", + "sTopic": "myKafkaTopic", + "sTopoMetric": "topology.kafka-topology.kafka-topic.myKafkaTopic.partition-0.latestCompletedOffset", + "sTopology": "kafka-topology", + "seriesAggregator": "none", + "transform": "none" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "$partition", + "tooltip": { + "shared": true, + "value_type": "cumulative" + }, + "type": "graph", + "x-axis": true, + "y-axis": true, + "y_formats": [ + "short", + "short" + ] + } + ], + "title": "New row" + } + ], + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "now": true, + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "templating": { + "list": [ + { + "allFormat": "glob", + "current": { + "text": "kafka-topology", + "value": "kafka-topology" + }, + "datasource": null, + "includeAll": false, + "multi": false, + "multiFormat": "glob", + "name": "topologies", + "options": [ + ], + "query": "topologies", + "refresh": true, + "regex": "", + "type": "query" + }, + { + "allFormat": "glob", + "current": { + "text": "", + "value": "" + }, + "datasource": null, + "includeAll": false, + "multi": false, + "multiFormat": "glob", + "name": "topic", + "options": [ + + ], + "query": "$topologies.stormTopic", + "refresh": true, + "regex": "", + "type": "query" + }, + { + "allFormat": "glob", + "current": { + "text": "partition-0", + "value": "partition-0" + }, + "datasource": null, + "includeAll": true, + "multi": true, + "multiFormat": "glob", + "name": "partition", + "options": [ + { + "text": "partition-0", + "value": "partition-0", + "selected": true + } + ], + "query": "$topic.stormPartition", + "refresh": true, + "type": "query" + } + ] + }, + "annotations": { + "list": [] + }, + "refresh": false, + "schemaVersion": 8, + "version": 23, + "links": [ + { + "asDropdown": true, + "icon": "external link", + "tags": [ + "storm" + ], + "title": "Storm Dashboards", + "type": "dashboards" + } + ] +} \ No newline at end of file
