This is an automated email from the ASF dual-hosted git repository.

dmitriusan pushed a commit to branch branch-2.7
in repository https://gitbox.apache.org/repos/asf/ambari.git


The following commit(s) were added to refs/heads/branch-2.7 by this push:
     new 7405c68  AMBARI-25370 Producer and Customer Request /s graphs are 
failing on Kafa Grafana dashboards (#3080)
7405c68 is described below

commit 7405c68f5daca31353455722cf78724fca584fd8
Author: payert <35402259+pay...@users.noreply.github.com>
AuthorDate: Fri Sep 6 11:41:13 2019 +0200

    AMBARI-25370 Producer and Customer Request /s graphs are failing on Kafa 
Grafana dashboards (#3080)
    
    From kafka 2.0.0 there has been addition of version tag in 
kafka.network.RequestMetrics.RequestsPerSec.request.* metrics.
    This is breaking the the default Grafana dashboard provided by Ambari. On 
the Kafka - Home and Kafka - Hosts dashboards
    the Producer requests /s and Consumer requests /s graphs are failing to 
show any data.
    
    Change-Id: I08656a6f43923b08b3bbc1f1e0346bd93211bbe8
---
 .../ambari-metrics/datasource.js                   | 31 +++++++++++++++++-----
 .../conf/unix/metrics_whitelist                    |  2 ++
 .../conf/windows/metrics_whitelist                 |  2 ++
 .../grafana-dashboards/HDP/grafana-kafka-home.json | 12 +++++----
 .../HDP/grafana-kafka-hosts.json                   | 12 +++++----
 5 files changed, 42 insertions(+), 17 deletions(-)

diff --git a/ambari-metrics/ambari-metrics-grafana/ambari-metrics/datasource.js 
b/ambari-metrics/ambari-metrics-grafana/ambari-metrics/datasource.js
index ec5ee10..b838ad5 100644
--- a/ambari-metrics/ambari-metrics-grafana/ambari-metrics/datasource.js
+++ b/ambari-metrics/ambari-metrics-grafana/ambari-metrics/datasource.js
@@ -99,7 +99,7 @@ define([
             if(!_.isEmpty(templateSrv.variables) && 
templateSrv.variables[0].query === "yarnqueues") {
               alias = alias + ' on ' + target.qmetric; }
             if(!_.isEmpty(templateSrv.variables) && 
templateSrv.variables[0].query === "kafka-topics") {
-            alias = alias + ' on ' + target.kbTopic; }
+              alias = alias + ' on ' + target.kbTopic; }
             return function (res) {
               res = res.data;
               console.log('processing metric ' + target.metric);
@@ -595,12 +595,29 @@ define([
               if (indexOfHosts >= 0) {
               var allHosts = templateSrv._values.hosts.lastIndexOf('}') > 0 ? 
templateSrv._values.hosts.slice(1,-1) :
               templateSrv._values.hosts;
-              allHosts = templateSrv._texts.hosts === "All" ? '%' : allHosts;
-              metricsPromises.push(_.map(options.targets, function(target) {
-                  target.templatedHost = allHosts? allHosts : '';
-                  target.templatedCluster = templatedCluster;
-                  return getAllHostData(target);
-              }));
+
+              /* The Producer & Comsumer Requests graphs on the Kafka Hosts 
dashboard should display metrics that are
+               * versioned, thus the value of different versions should be 
aggregated and grouped by hosts.
+               * In order to have a 'grouped by hosts' like view the metric 
results are queried for each hosts separately.
+               */
+              if (!_.isEmpty(options.targets.filter(function(target) {
+                    return target.metric.endsWith(".%.count"); }))) {
+                allHosts = allHosts.split(',');
+                _.forEach(allHosts, function(host) {
+                    metricsPromises.push(_.map(options.targets, 
function(target) {
+                        target.templatedHost = host;
+                        target.templatedCluster = templatedCluster;
+                        return getAllHostData(target);
+                    }));
+                });
+              } else {
+                  allHosts = templateSrv._texts.hosts === "All" ? '%' : 
allHosts;
+                  metricsPromises.push(_.map(options.targets, function(target) 
{
+                      target.templatedHost = allHosts? allHosts : '';
+                      target.templatedCluster = templatedCluster;
+                      return getAllHostData(target);
+                }));
+              }
             }
             metricsPromises = _.flatten(metricsPromises);
           } else {
diff --git 
a/ambari-metrics/ambari-metrics-timelineservice/conf/unix/metrics_whitelist 
b/ambari-metrics/ambari-metrics-timelineservice/conf/unix/metrics_whitelist
index fd03d6e..689fd0d 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/conf/unix/metrics_whitelist
+++ b/ambari-metrics/ambari-metrics-timelineservice/conf/unix/metrics_whitelist
@@ -273,6 +273,8 @@ kafka.controller.KafkaController.OfflinePartitionsCount
 kafka.log.LogFlushStats.LogFlushRateAndTimeMs.1MinuteRate
 kafka.network.RequestMetrics.RequestsPerSec.request.FetchConsumer.count
 kafka.network.RequestMetrics.RequestsPerSec.request.Produce.count
+kafka.network.RequestMetrics.RequestsPerSec.request.FetchConsumer.version.*.count
+kafka.network.RequestMetrics.RequestsPerSec.request.Produce.version.*.count
 kafka.network.RequestMetrics.TotalTimeMs.request.FetchConsumer.99percentile
 kafka.network.RequestMetrics.TotalTimeMs.request.FetchConsumer.max
 kafka.network.RequestMetrics.TotalTimeMs.request.FetchConsumer.mean
diff --git 
a/ambari-metrics/ambari-metrics-timelineservice/conf/windows/metrics_whitelist 
b/ambari-metrics/ambari-metrics-timelineservice/conf/windows/metrics_whitelist
index fd03d6e..689fd0d 100644
--- 
a/ambari-metrics/ambari-metrics-timelineservice/conf/windows/metrics_whitelist
+++ 
b/ambari-metrics/ambari-metrics-timelineservice/conf/windows/metrics_whitelist
@@ -273,6 +273,8 @@ kafka.controller.KafkaController.OfflinePartitionsCount
 kafka.log.LogFlushStats.LogFlushRateAndTimeMs.1MinuteRate
 kafka.network.RequestMetrics.RequestsPerSec.request.FetchConsumer.count
 kafka.network.RequestMetrics.RequestsPerSec.request.Produce.count
+kafka.network.RequestMetrics.RequestsPerSec.request.FetchConsumer.version.*.count
+kafka.network.RequestMetrics.RequestsPerSec.request.Produce.version.*.count
 kafka.network.RequestMetrics.TotalTimeMs.request.FetchConsumer.99percentile
 kafka.network.RequestMetrics.TotalTimeMs.request.FetchConsumer.max
 kafka.network.RequestMetrics.TotalTimeMs.request.FetchConsumer.mean
diff --git 
a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/files/grafana-dashboards/HDP/grafana-kafka-home.json
 
b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/files/grafana-dashboards/HDP/grafana-kafka-home.json
index 983142a..e9b6fa2 100644
--- 
a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/files/grafana-dashboards/HDP/grafana-kafka-home.json
+++ 
b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/files/grafana-dashboards/HDP/grafana-kafka-home.json
@@ -555,11 +555,12 @@
               "app": "kafka_broker",
               "downsampleAggregator": "avg",
               "errors": {},
-              "metric": 
"kafka.network.RequestMetrics.RequestsPerSec.request.Produce.count",
+              "metric": 
"kafka.network.RequestMetrics.RequestsPerSec.request.Produce.%.count",
               "precision": "default",
               "refId": "A",
               "transform": "rate",
-              "transformData": "none"
+              "transformData": "none",
+              "seriesAggregator": "sum"
             }
           ],
           "timeFrom": null,
@@ -626,11 +627,12 @@
               "app": "kafka_broker",
               "downsampleAggregator": "avg",
               "errors": {},
-              "metric": 
"kafka.network.RequestMetrics.RequestsPerSec.request.FetchConsumer.count",
+              "metric": 
"kafka.network.RequestMetrics.RequestsPerSec.request.FetchConsumer.%.count",
               "precision": "default",
               "refId": "A",
               "transform": "rate",
-              "transformData": "none"
+              "transformData": "none",
+              "seriesAggregator": "sum"
             }
           ],
           "timeFrom": null,
@@ -1073,7 +1075,7 @@
     "list": []
   },
   "schemaVersion": 8,
-  "version": 16,
+  "version": 17,
   "links": [
     {
       "asDropdown": true,
diff --git 
a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/files/grafana-dashboards/HDP/grafana-kafka-hosts.json
 
b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/files/grafana-dashboards/HDP/grafana-kafka-hosts.json
index 85deb92..61b2e77 100644
--- 
a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/files/grafana-dashboards/HDP/grafana-kafka-hosts.json
+++ 
b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/files/grafana-dashboards/HDP/grafana-kafka-hosts.json
@@ -323,12 +323,13 @@
               "app": "kafka_broker",
               "downsampleAggregator": "avg",
               "errors": {},
-              "metric": 
"kafka.network.RequestMetrics.RequestsPerSec.request.Produce.count",
+              "metric": 
"kafka.network.RequestMetrics.RequestsPerSec.request.Produce.%.count",
               "precision": "default",
               "refId": "A",
               "templatedHost": "",
               "transform": "rate",
-              "transformData": "none"
+              "transformData": "none",
+              "seriesAggregator": "sum"
             }
           ],
           "timeFrom": null,
@@ -395,12 +396,13 @@
               "app": "kafka_broker",
               "downsampleAggregator": "avg",
               "errors": {},
-              "metric": 
"kafka.network.RequestMetrics.RequestsPerSec.request.FetchConsumer.count",
+              "metric": 
"kafka.network.RequestMetrics.RequestsPerSec.request.FetchConsumer.%.count",
               "precision": "default",
               "refId": "A",
               "templatedHost": "",
               "transform": "rate",
-              "transformData": "none"
+              "transformData": "none",
+              "seriesAggregator": "sum"
             }
           ],
           "timeFrom": null,
@@ -1991,7 +1993,7 @@
     "list": []
   },
   "schemaVersion": 8,
-  "version": 16,
+  "version": 17,
   "links": [
     {
       "asDropdown": true,

Reply via email to