AMBARI-18170 : Kafka Metrics do not show up in AMS HA enabled cluster. (avijayan)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/a7f9090a Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/a7f9090a Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/a7f9090a Branch: refs/heads/branch-2.5 Commit: a7f9090a349f1a11460e114e1bf9066b7267201e Parents: a105d58 Author: Aravindan Vijayan <[email protected]> Authored: Mon Nov 14 20:42:18 2016 -0800 Committer: Aravindan Vijayan <[email protected]> Committed: Tue Nov 15 11:02:13 2016 -0800 ---------------------------------------------------------------------- .../sink/kafka/KafkaTimelineMetricsReporter.java | 13 +++---------- .../sink/kafka/KafkaTimelineMetricsReporterTest.java | 2 +- .../KAFKA/0.8.1/configuration/kafka-broker.xml | 4 ++-- .../KAFKA/0.8.1/package/scripts/kafka.py | 2 +- .../KAFKA/0.8.1/package/scripts/params.py | 8 ++++++++ .../stacks/2.2/configs/ranger-admin-upgrade.json | 2 +- .../stacks/2.2/configs/ranger-usersync-upgrade.json | 2 +- .../app/assets/data/stacks/HDP-2.2/configurations.json | 6 +++--- 8 files changed, 20 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/a7f9090a/ambari-metrics/ambari-metrics-kafka-sink/src/main/java/org/apache/hadoop/metrics2/sink/kafka/KafkaTimelineMetricsReporter.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-kafka-sink/src/main/java/org/apache/hadoop/metrics2/sink/kafka/KafkaTimelineMetricsReporter.java b/ambari-metrics/ambari-metrics-kafka-sink/src/main/java/org/apache/hadoop/metrics2/sink/kafka/KafkaTimelineMetricsReporter.java index 11a1c75..9c16564 100644 --- a/ambari-metrics/ambari-metrics-kafka-sink/src/main/java/org/apache/hadoop/metrics2/sink/kafka/KafkaTimelineMetricsReporter.java +++ b/ambari-metrics/ambari-metrics-kafka-sink/src/main/java/org/apache/hadoop/metrics2/sink/kafka/KafkaTimelineMetricsReporter.java @@ -61,7 +61,7 @@ public class KafkaTimelineMetricsReporter extends AbstractTimelineMetricsSink private static final String TIMELINE_METRICS_SEND_INTERVAL_PROPERTY = "kafka.timeline.metrics.sendInterval"; private static final String TIMELINE_METRICS_MAX_ROW_CACHE_SIZE_PROPERTY = "kafka.timeline.metrics.maxRowCacheSize"; - private static final String TIMELINE_HOST_PROPERTY = "kafka.timeline.metrics.host"; + private static final String TIMELINE_HOSTS_PROPERTY = "kafka.timeline.metrics.hosts"; private static final String TIMELINE_PORT_PROPERTY = "kafka.timeline.metrics.port"; private static final String TIMELINE_PROTOCOL_PROPERTY = "kafka.timeline.metrics.protocol"; private static final String TIMELINE_REPORTER_ENABLED_PROPERTY = "kafka.timeline.metrics.reporter.enabled"; @@ -74,7 +74,6 @@ public class KafkaTimelineMetricsReporter extends AbstractTimelineMetricsSink private volatile boolean initialized = false; private boolean running = false; private final Object lock = new Object(); - private String collectorUri; private String hostname; private String metricCollectorPort; private String collectors; @@ -147,17 +146,12 @@ public class KafkaTimelineMetricsReporter extends AbstractTimelineMetricsSink int maxRowCacheSize = props.getInt(TIMELINE_METRICS_MAX_ROW_CACHE_SIZE_PROPERTY, MAX_RECS_PER_NAME_DEFAULT); zookeeperQuorum = props.getString("zookeeper.connect"); - collectors = props.getString(TIMELINE_HOST_PROPERTY, TIMELINE_DEFAULT_HOST); - metricCollectorProtocol = props.getString(TIMELINE_PROTOCOL_PROPERTY, TIMELINE_DEFAULT_PROTOCOL); - - String metricCollectorHost = props.getString(TIMELINE_HOST_PROPERTY, TIMELINE_DEFAULT_HOST); metricCollectorPort = props.getString(TIMELINE_PORT_PROPERTY, TIMELINE_DEFAULT_PORT); + collectors = props.getString(TIMELINE_HOSTS_PROPERTY, TIMELINE_DEFAULT_HOST + ":" + metricCollectorPort); + metricCollectorProtocol = props.getString(TIMELINE_PROTOCOL_PROPERTY, TIMELINE_DEFAULT_PROTOCOL); setMetricsCache(new TimelineMetricsCache(maxRowCacheSize, metricsSendInterval)); - collectorUri = constructTimelineMetricUri(metricCollectorProtocol, - metricCollectorHost, metricCollectorPort); - if (metricCollectorProtocol.contains("https")) { String trustStorePath = props.getString(SSL_KEYSTORE_PATH_PROPERTY).trim(); String trustStoreType = props.getString(SSL_KEYSTORE_TYPE_PROPERTY).trim(); @@ -181,7 +175,6 @@ public class KafkaTimelineMetricsReporter extends AbstractTimelineMetricsSink startReporter(metricsConfig.pollingIntervalSecs()); } if (LOG.isDebugEnabled()) { - LOG.debug("CollectorUri = " + collectorUri); LOG.debug("MetricsSendInterval = " + metricsSendInterval); LOG.debug("MaxRowCacheSize = " + maxRowCacheSize); LOG.debug("Excluded metrics prefixes = " + excludedMetricsStr); http://git-wip-us.apache.org/repos/asf/ambari/blob/a7f9090a/ambari-metrics/ambari-metrics-kafka-sink/src/test/java/org/apache/hadoop/metrics2/sink/kafka/KafkaTimelineMetricsReporterTest.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-kafka-sink/src/test/java/org/apache/hadoop/metrics2/sink/kafka/KafkaTimelineMetricsReporterTest.java b/ambari-metrics/ambari-metrics-kafka-sink/src/test/java/org/apache/hadoop/metrics2/sink/kafka/KafkaTimelineMetricsReporterTest.java index 9027716..c569d88 100644 --- a/ambari-metrics/ambari-metrics-kafka-sink/src/test/java/org/apache/hadoop/metrics2/sink/kafka/KafkaTimelineMetricsReporterTest.java +++ b/ambari-metrics/ambari-metrics-kafka-sink/src/test/java/org/apache/hadoop/metrics2/sink/kafka/KafkaTimelineMetricsReporterTest.java @@ -79,7 +79,7 @@ public class KafkaTimelineMetricsReporterTest { properties.setProperty("zookeeper.connect", "localhost:2181"); properties.setProperty("kafka.timeline.metrics.sendInterval", "5900"); properties.setProperty("kafka.timeline.metrics.maxRowCacheSize", "10000"); - properties.setProperty("kafka.timeline.metrics.host", "localhost"); + properties.setProperty("kafka.timeline.metrics.hosts", "localhost:6188"); properties.setProperty("kafka.timeline.metrics.port", "6188"); properties.setProperty("kafka.timeline.metrics.reporter.enabled", "true"); properties.setProperty("external.kafka.metrics.exclude.prefix", "a.b.c"); http://git-wip-us.apache.org/repos/asf/ambari/blob/a7f9090a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/configuration/kafka-broker.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/configuration/kafka-broker.xml b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/configuration/kafka-broker.xml index f7da62c..ab4d701 100644 --- a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/configuration/kafka-broker.xml +++ b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/configuration/kafka-broker.xml @@ -357,8 +357,8 @@ <on-ambari-upgrade add="true"/> </property> <property> - <name>kafka.timeline.metrics.host</name> - <value>{{metric_collector_host}}</value> + <name>kafka.timeline.metrics.hosts</name> + <value>{{metric_collector_hosts}}</value> <description>Timeline host</description> <on-ambari-upgrade add="true"/> </property> http://git-wip-us.apache.org/repos/asf/ambari/blob/a7f9090a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/kafka.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/kafka.py b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/kafka.py index f0c8289..635cbe9 100644 --- a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/kafka.py +++ b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/kafka.py @@ -99,7 +99,7 @@ def kafka(upgrade_type=None): kafka_server_config['host.name'] = params.hostname if params.has_metric_collector: - kafka_server_config['kafka.timeline.metrics.host'] = params.metric_collector_host + kafka_server_config['kafka.timeline.metrics.hosts'] = params.metric_collector_hosts kafka_server_config['kafka.timeline.metrics.port'] = params.metric_collector_port kafka_server_config['kafka.timeline.metrics.protocol'] = params.metric_collector_protocol kafka_server_config['kafka.timeline.metrics.truststore.path'] = params.metric_truststore_path http://git-wip-us.apache.org/repos/asf/ambari/blob/a7f9090a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py index 11c1ad0..5c26796 100644 --- a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py @@ -144,6 +144,14 @@ if has_metric_collector: else: metric_collector_protocol = 'http' pass + + # Collector hosts + metric_collector_hosts = "" + if ams_collector_hosts: + for host in ams_collector_hosts: + metric_collector_hosts += host + ':' + metric_collector_port + ',' + metric_collector_hosts = metric_collector_hosts[:-1] + # Security-related params security_enabled = config['configurations']['cluster-env']['security_enabled'] kafka_kerberos_enabled = (('security.inter.broker.protocol' in config['configurations']['kafka-broker']) and http://git-wip-us.apache.org/repos/asf/ambari/blob/a7f9090a/ambari-server/src/test/python/stacks/2.2/configs/ranger-admin-upgrade.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.2/configs/ranger-admin-upgrade.json b/ambari-server/src/test/python/stacks/2.2/configs/ranger-admin-upgrade.json index d348645..0a49368 100644 --- a/ambari-server/src/test/python/stacks/2.2/configs/ranger-admin-upgrade.json +++ b/ambari-server/src/test/python/stacks/2.2/configs/ranger-admin-upgrade.json @@ -455,7 +455,7 @@ "replica.high.watermark.checkpoint.interval.ms": "5000", "zookeeper.connect": "c6407.ambari.apache.org:2181,c6408.ambari.apache.org:2181,c6409.ambari.apache.org:2181", "controlled.shutdown.retry.backoff.ms": "5000", - "kafka.timeline.metrics.host": "{{metric_collector_host}}", + "kafka.timeline.metrics.host": "{{metric_collector_hosts}}", "kafka.timeline.metrics.reporter.sendInterval": "5900", "num.partitions": "1", "log.flush.interval.messages": "10000", http://git-wip-us.apache.org/repos/asf/ambari/blob/a7f9090a/ambari-server/src/test/python/stacks/2.2/configs/ranger-usersync-upgrade.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.2/configs/ranger-usersync-upgrade.json b/ambari-server/src/test/python/stacks/2.2/configs/ranger-usersync-upgrade.json index 27af82f..b6615da 100644 --- a/ambari-server/src/test/python/stacks/2.2/configs/ranger-usersync-upgrade.json +++ b/ambari-server/src/test/python/stacks/2.2/configs/ranger-usersync-upgrade.json @@ -450,7 +450,7 @@ "replica.high.watermark.checkpoint.interval.ms": "5000", "zookeeper.connect": "c6407.ambari.apache.org:2181,c6408.ambari.apache.org:2181,c6409.ambari.apache.org:2181", "controlled.shutdown.retry.backoff.ms": "5000", - "kafka.timeline.metrics.host": "{{metric_collector_host}}", + "kafka.timeline.metrics.host": "{{metric_collector_hosts}}", "kafka.timeline.metrics.reporter.sendInterval": "5900", "num.partitions": "1", "log.flush.interval.messages": "10000", http://git-wip-us.apache.org/repos/asf/ambari/blob/a7f9090a/ambari-web/app/assets/data/stacks/HDP-2.2/configurations.json ---------------------------------------------------------------------- diff --git a/ambari-web/app/assets/data/stacks/HDP-2.2/configurations.json b/ambari-web/app/assets/data/stacks/HDP-2.2/configurations.json index 85d4a9f..e0aa9eb 100644 --- a/ambari-web/app/assets/data/stacks/HDP-2.2/configurations.json +++ b/ambari-web/app/assets/data/stacks/HDP-2.2/configurations.json @@ -8800,13 +8800,13 @@ } }, { - "href" : "http://c6401:8080/api/v1/stacks/HDP/versions/2.2/services/KAFKA/configurations/kafka.timeline.metrics.host";, + "href" : "http://c6401:8080/api/v1/stacks/HDP/versions/2.2/services/KAFKA/configurations/kafka.timeline.metrics.hosts";, "StackConfigurations" : { "final" : "false", "property_description" : "Timeline host", - "property_name" : "kafka.timeline.metrics.host", + "property_name" : "kafka.timeline.metrics.hosts", "property_type" : [ ], - "property_value" : "{{metric_collector_host}}", + "property_value" : "{{metric_collector_hosts}}", "service_name" : "KAFKA", "stack_name" : "HDP", "stack_version" : "2.2",
