AMBARI-18362 : Update sinks to read multiple collector hostnames from configs. (dsen via avijayan)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/0d01ab25 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/0d01ab25 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/0d01ab25 Branch: refs/heads/branch-2.5 Commit: 0d01ab25857a3b073f7cc78a508ed41d5436f069 Parents: fd42b6a Author: Aravindan Vijayan <[email protected]> Authored: Mon Nov 14 21:36:03 2016 -0800 Committer: Aravindan Vijayan <[email protected]> Committed: Tue Nov 15 11:06:40 2016 -0800 ---------------------------------------------------------------------- .../logsearch/solr/metrics/SolrAmsClient.java | 9 +++- .../timeline/AbstractTimelineMetricsSink.java | 57 ++++++++++++++------ .../availability/MetricCollectorHATest.java | 22 +++++--- .../cache/HandleConnectExceptionTest.java | 12 +++-- .../sink/flume/FlumeTimelineMetricsSink.java | 14 +++-- .../timeline/HadoopTimelineMetricsSink.java | 27 ++++++++-- .../timeline/HadoopTimelineMetricsSinkTest.java | 12 +++-- .../kafka/KafkaTimelineMetricsReporter.java | 14 +++-- .../storm/StormTimelineMetricsReporter.java | 14 +++-- .../sink/storm/StormTimelineMetricsSink.java | 22 ++++---- .../storm/StormTimelineMetricsReporter.java | 16 +++--- .../sink/storm/StormTimelineMetricsSink.java | 18 ++++--- .../1.6.1.2.2.0/package/scripts/params.py | 5 -- .../hadoop-metrics2-accumulo.properties.j2 | 12 +++-- .../hadoop-metrics2-hbase.properties.j2 | 14 +++-- .../FLUME/1.4.0.2.0/package/scripts/flume.py | 1 + .../FLUME/1.4.0.2.0/package/scripts/params.py | 7 --- .../templates/flume-metrics2.properties.j2 | 2 +- .../0.96.0.2.0/package/scripts/params_linux.py | 12 ----- ...-metrics2-hbase.properties-GANGLIA-MASTER.j2 | 15 ++++-- ...doop-metrics2-hbase.properties-GANGLIA-RS.j2 | 15 ++++-- .../hadoop-metrics2.properties.xml | 18 +++---- .../0.12.0.2.0/package/scripts/params_linux.py | 5 -- .../hadoop-metrics2-hivemetastore.properties.j2 | 3 +- .../hadoop-metrics2-hiveserver2.properties.j2 | 4 +- .../templates/hadoop-metrics2-llapdaemon.j2 | 3 +- .../hadoop-metrics2-llaptaskscheduler.j2 | 3 +- .../KAFKA/0.8.1/configuration/kafka-broker.xml | 2 +- .../KAFKA/0.8.1/package/scripts/kafka.py | 2 +- .../KAFKA/0.8.1/package/scripts/params.py | 7 --- .../STORM/0.9.1/package/scripts/params_linux.py | 11 ---- .../0.9.1/package/templates/config.yaml.j2 | 6 ++- .../templates/storm-metrics2.properties.j2 | 2 +- .../2.0.6/hooks/before-START/scripts/params.py | 7 --- .../templates/hadoop-metrics2.properties.j2 | 14 ++--- .../stacks/HDP/2.0.6/services/stack_advisor.py | 10 ++-- 36 files changed, 239 insertions(+), 178 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/0d01ab25/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/solr/metrics/SolrAmsClient.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/solr/metrics/SolrAmsClient.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/solr/metrics/SolrAmsClient.java index 85ea69d..0d4deab 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/solr/metrics/SolrAmsClient.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/solr/metrics/SolrAmsClient.java @@ -22,6 +22,8 @@ package org.apache.ambari.logsearch.solr.metrics; import org.apache.hadoop.metrics2.sink.timeline.AbstractTimelineMetricsSink; import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics; +import java.util.Collection; + // TODO: Refactor for failover public class SolrAmsClient extends AbstractTimelineMetricsSink { private final String collectorHost; @@ -46,7 +48,7 @@ public class SolrAmsClient extends AbstractTimelineMetricsSink { } @Override - protected String getConfiguredCollectors() { + protected Collection<String> getConfiguredCollectorHosts() { return null; } @@ -64,4 +66,9 @@ public class SolrAmsClient extends AbstractTimelineMetricsSink { protected String getCollectorProtocol() { return null; } + + @Override + protected String getCollectorPort() { + return null; + } } http://git-wip-us.apache.org/repos/asf/ambari/blob/0d01ab25/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java b/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java index 426eb42..8be871f 100644 --- a/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java +++ b/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java @@ -61,7 +61,7 @@ public abstract class AbstractTimelineMetricsSink { public static final String MAX_METRIC_ROW_CACHE_SIZE = "maxRowCacheSize"; public static final String METRICS_SEND_INTERVAL = "sendInterval"; public static final String METRICS_POST_TIMEOUT_SECONDS = "timeout"; - public static final String COLLECTOR_PROPERTY = "collector"; + public static final String COLLECTOR_HOSTS_PROPERTY = "collector.hosts"; public static final String COLLECTOR_PROTOCOL = "protocol"; public static final String COLLECTOR_PORT = "port"; public static final String ZOOKEEPER_QUORUM = "zookeeper.quorum"; @@ -211,6 +211,7 @@ public abstract class AbstractTimelineMetricsSink { String connectUrl = getCollectorUri(collectorHost); String jsonData = null; + LOG.debug("EmitMetrics connectUrl = " + connectUrl); try { jsonData = mapper.writeValueAsString(metrics); } catch (IOException e) { @@ -326,22 +327,16 @@ public abstract class AbstractTimelineMetricsSink { } } - String collectorHosts = getConfiguredCollectors(); + Collection<String> collectorHosts = getConfiguredCollectorHosts(); + + LOG.debug("Trying to find live collector host from : " + collectorHosts); // Reach out to all configured collectors before Zookeeper if (collectorHosts != null && !collectorHosts.isEmpty()) { - String[] hosts = collectorHosts.split(","); - for (String hostPortStr : hosts) { - if (hostPortStr != null && !hostPortStr.isEmpty()) { - String[] hostPortPair = hostPortStr.split(":"); - if (hostPortPair.length < 2) { - LOG.warn("Collector port is missing from the configuration."); - continue; - } - String hostStr = hostPortPair[0].trim(); - String portStr = hostPortPair[1].trim(); - // Check liveliness and get known instances + for (String hostStr : collectorHosts) { + hostStr = hostStr.trim(); + if (!hostStr.isEmpty()) { try { - Collection<String> liveHosts = findLiveCollectorHostsFromKnownCollector(hostStr, portStr); + Collection<String> liveHosts = findLiveCollectorHostsFromKnownCollector(hostStr, getCollectorPort()); // Update live Hosts - current host will already be a part of this for (String host : liveHosts) { allKnownLiveCollectors.add(host); @@ -377,6 +372,7 @@ public abstract class AbstractTimelineMetricsSink { return targetCollectorHostSupplier.get(); } + LOG.warn("Couldn't find any live collectors. Returning null"); return null; } @@ -455,6 +451,31 @@ public abstract class AbstractTimelineMetricsSink { sb.append(WS_V1_TIMELINE_METRICS); return sb.toString(); } + /** + * Parses input Sting of format "['host1', 'host2']" into Collection of hostnames + */ + protected Collection<String> parseHostsStringIntoCollection(String hostsString) { + Set<String> hosts = new HashSet<>(); + + if (hostsString == null) { + LOG.error("No Metric collector configured."); + return hosts; + } + + hostsString = hostsString.replace("[", ""); + hostsString = hostsString.replace("]", ""); + hostsString = hostsString.replace("'", ""); + + String [] hostNamesWithApostrophes = hostsString.split(","); + + for (String host : hostNamesWithApostrophes) { + host = host.trim(); + if (host.equals("")) continue; + hosts.add(host); + } + + return hosts; + } /** * Get a pre-formatted URI for the collector @@ -463,6 +484,8 @@ public abstract class AbstractTimelineMetricsSink { abstract protected String getCollectorProtocol(); + abstract protected String getCollectorPort(); + /** * How soon to timeout on the emit calls in seconds. */ @@ -475,10 +498,10 @@ public abstract class AbstractTimelineMetricsSink { abstract protected String getZookeeperQuorum(); /** - * Get pre-configured list of collectors available - * @return String "host1:port,host2:port" + * Get pre-configured list of collectors hosts available + * @return Collection<String> host1,host2 */ - abstract protected String getConfiguredCollectors(); + abstract protected Collection<String> getConfiguredCollectorHosts(); /** * Get hostname used for calculating write shard. http://git-wip-us.apache.org/repos/asf/ambari/blob/0d01ab25/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/availability/MetricCollectorHATest.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/availability/MetricCollectorHATest.java b/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/availability/MetricCollectorHATest.java index 7fadeb2..ac1f52d 100644 --- a/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/availability/MetricCollectorHATest.java +++ b/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/availability/MetricCollectorHATest.java @@ -27,11 +27,12 @@ import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; import java.io.IOException; import java.io.InputStream; -import java.io.OutputStream; import java.net.HttpURLConnection; import java.net.URL; import java.util.ArrayList; -import java.util.HashSet; +import java.util.Arrays; +import java.util.Collection; + import static org.easymock.EasyMock.expect; import static org.powermock.api.easymock.PowerMock.createNiceMock; import static org.powermock.api.easymock.PowerMock.expectNew; @@ -49,7 +50,8 @@ public class MetricCollectorHATest { URL url = createNiceMock(URL.class); MetricCollectorHAHelper haHelper = createNiceMock(MetricCollectorHAHelper.class); - expectNew(URL.class, "http://localhost:2181/ws/v1/timeline/metrics/livenodes";).andReturn(url).anyTimes(); + expectNew(URL.class, "http://localhost1:2181/ws/v1/timeline/metrics/livenodes";).andReturn(url).anyTimes(); + expectNew(URL.class, "http://localhost2:2181/ws/v1/timeline/metrics/livenodes";).andReturn(url).anyTimes(); expect(url.openConnection()).andReturn(connection).anyTimes(); expect(connection.getInputStream()).andReturn(is).anyTimes(); expect(connection.getResponseCode()).andThrow(new IOException()).anyTimes(); @@ -82,7 +84,8 @@ public class MetricCollectorHATest { output.add("h3"); InputStream is = IOUtils.toInputStream(gson.toJson(output)); - expectNew(URL.class, "http://localhost:2181/ws/v1/timeline/metrics/livenodes";).andReturn(url).anyTimes(); + expectNew(URL.class, "http://localhost1:2181/ws/v1/timeline/metrics/livenodes";).andReturn(url).anyTimes(); + expectNew(URL.class, "http://localhost2:2181/ws/v1/timeline/metrics/livenodes";).andReturn(url).anyTimes(); expect(url.openConnection()).andReturn(connection).anyTimes(); expect(connection.getInputStream()).andReturn(is).anyTimes(); expect(connection.getResponseCode()).andReturn(200).anyTimes(); @@ -127,18 +130,23 @@ public class MetricCollectorHATest { } @Override + protected String getCollectorPort() { + return "2181"; + } + + @Override protected int getTimeoutSeconds() { return 10; } @Override protected String getZookeeperQuorum() { - return "localhost:2181"; + return "localhost1:2181"; } @Override - protected String getConfiguredCollectors() { - return "localhost:2181"; + protected Collection<String> getConfiguredCollectorHosts() { + return Arrays.asList("localhost1", "localhost2"); } @Override http://git-wip-us.apache.org/repos/asf/ambari/blob/0d01ab25/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/cache/HandleConnectExceptionTest.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/cache/HandleConnectExceptionTest.java b/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/cache/HandleConnectExceptionTest.java index ccaa574..36ec074 100644 --- a/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/cache/HandleConnectExceptionTest.java +++ b/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/cache/HandleConnectExceptionTest.java @@ -21,9 +21,10 @@ import java.io.IOException; import java.io.OutputStream; import java.net.HttpURLConnection; import java.net.URL; +import java.util.Arrays; +import java.util.Collection; import org.apache.hadoop.metrics2.sink.timeline.AbstractTimelineMetricsSink; -import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric; import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics; import org.apache.hadoop.metrics2.sink.timeline.UnableToConnectException; import org.junit.Assert; @@ -99,6 +100,11 @@ public class HandleConnectExceptionTest { } @Override + protected String getCollectorPort() { + return "2181"; + } + + @Override protected int getTimeoutSeconds() { return 10; } @@ -109,8 +115,8 @@ public class HandleConnectExceptionTest { } @Override - protected String getConfiguredCollectors() { - return "localhost:2181"; + protected Collection<String> getConfiguredCollectorHosts() { + return Arrays.asList("localhost"); } @Override http://git-wip-us.apache.org/repos/asf/ambari/blob/0d01ab25/ambari-metrics/ambari-metrics-flume-sink/src/main/java/org/apache/hadoop/metrics2/sink/flume/FlumeTimelineMetricsSink.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-flume-sink/src/main/java/org/apache/hadoop/metrics2/sink/flume/FlumeTimelineMetricsSink.java b/ambari-metrics/ambari-metrics-flume-sink/src/main/java/org/apache/hadoop/metrics2/sink/flume/FlumeTimelineMetricsSink.java index 1b36e9a..c1b684b 100644 --- a/ambari-metrics/ambari-metrics-flume-sink/src/main/java/org/apache/hadoop/metrics2/sink/flume/FlumeTimelineMetricsSink.java +++ b/ambari-metrics/ambari-metrics-flume-sink/src/main/java/org/apache/hadoop/metrics2/sink/flume/FlumeTimelineMetricsSink.java @@ -34,6 +34,7 @@ import java.io.IOException; import java.net.InetAddress; import java.net.UnknownHostException; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -55,7 +56,7 @@ public class FlumeTimelineMetricsSink extends AbstractTimelineMetricsSink implem private long pollFrequency; private String hostname; private String port; - private String collectors; + private Collection<String> collectorHosts; private String zookeeperQuorum; private final static String COUNTER_METRICS_PROPERTY = "counters"; private final Set<String> counterMetrics = new HashSet<String>(); @@ -99,7 +100,7 @@ public class FlumeTimelineMetricsSink extends AbstractTimelineMetricsSink implem metricsSendInterval = Integer.parseInt(configuration.getProperty(METRICS_SEND_INTERVAL, String.valueOf(TimelineMetricsCache.MAX_EVICTION_TIME_MILLIS))); metricsCaches = new HashMap<String, TimelineMetricsCache>(); - collectors = configuration.getProperty(COLLECTOR_PROPERTY); + collectorHosts = parseHostsStringIntoCollection(configuration.getProperty(COLLECTOR_HOSTS_PROPERTY)); zookeeperQuorum = configuration.getProperty("zookeeper.quorum"); protocol = configuration.getProperty(COLLECTOR_PROTOCOL, "http"); port = configuration.getProperty(COLLECTOR_PORT, "6188"); @@ -130,6 +131,11 @@ public class FlumeTimelineMetricsSink extends AbstractTimelineMetricsSink implem } @Override + protected String getCollectorPort() { + return port; + } + + @Override protected int getTimeoutSeconds() { return timeoutSeconds; } @@ -140,8 +146,8 @@ public class FlumeTimelineMetricsSink extends AbstractTimelineMetricsSink implem } @Override - protected String getConfiguredCollectors() { - return collectors; + protected Collection<String> getConfiguredCollectorHosts() { + return collectorHosts; } @Override http://git-wip-us.apache.org/repos/asf/ambari/blob/0d01ab25/ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSink.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSink.java b/ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSink.java index 8e78e6f..802e684 100644 --- a/ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSink.java +++ b/ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSink.java @@ -52,7 +52,7 @@ public class HadoopTimelineMetricsSink extends AbstractTimelineMetricsSink imple private TimelineMetricsCache metricsCache; private String hostName = "UNKNOWN.example.com"; private String serviceName = ""; - private String collectors; + private Collection<String> collectorHosts; private String collectorUri; private String containerMetricsUri; private String protocol; @@ -101,10 +101,10 @@ public class HadoopTimelineMetricsSink extends AbstractTimelineMetricsSink imple // Load collector configs protocol = conf.getString(COLLECTOR_PROTOCOL, "http"); - collectors = conf.getString(COLLECTOR_PROPERTY, "").trim(); + collectorHosts = parseHostsStringArrayIntoCollection(conf.getStringArray(COLLECTOR_HOSTS_PROPERTY)); port = conf.getString(COLLECTOR_PORT, "6188"); - if (StringUtils.isEmpty(collectors)) { + if (collectorHosts.isEmpty()) { LOG.error("No Metric collector configured."); } else { String preferredCollectorHost = findPreferredCollectHost(); @@ -119,6 +119,7 @@ public class HadoopTimelineMetricsSink extends AbstractTimelineMetricsSink imple } LOG.info("Collector Uri: " + collectorUri); + LOG.info("Container Metrics Uri: " + containerMetricsUri); timeoutSeconds = conf.getInt(METRICS_POST_TIMEOUT_SECONDS, DEFAULT_POST_TIMEOUT_SECONDS); @@ -192,6 +193,18 @@ public class HadoopTimelineMetricsSink extends AbstractTimelineMetricsSink imple return conf.getPrefix(); } + /** + * Parses input Stings array of format "['host1'", '"host2']" into Collection of hostnames + */ + protected Collection<String> parseHostsStringArrayIntoCollection(String[] hostStrings) { + Collection<String> result = new HashSet<>(); + if (hostStrings == null) return result; + for (String s : hostStrings) { + result.addAll(parseHostsStringIntoCollection(s)); + } + return result; + } + @Override protected String getCollectorUri(String host) { return constructTimelineMetricUri(protocol, host, port); @@ -213,11 +226,15 @@ public class HadoopTimelineMetricsSink extends AbstractTimelineMetricsSink imple } @Override - protected String getConfiguredCollectors() { - return collectors; + protected Collection<String> getConfiguredCollectorHosts() { + return collectorHosts; } @Override + protected String getCollectorPort() { + return port; + } + @Override protected String getHostname() { return hostName; } http://git-wip-us.apache.org/repos/asf/ambari/blob/0d01ab25/ambari-metrics/ambari-metrics-hadoop-sink/src/test/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSinkTest.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-hadoop-sink/src/test/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSinkTest.java b/ambari-metrics/ambari-metrics-hadoop-sink/src/test/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSinkTest.java index 5f22065..da00155 100644 --- a/ambari-metrics/ambari-metrics-hadoop-sink/src/test/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSinkTest.java +++ b/ambari-metrics/ambari-metrics-hadoop-sink/src/test/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSinkTest.java @@ -53,7 +53,7 @@ import java.util.List; import java.util.Set; import static org.apache.hadoop.metrics2.sink.timeline.AbstractTimelineMetricsSink.COLLECTOR_PORT; -import static org.apache.hadoop.metrics2.sink.timeline.AbstractTimelineMetricsSink.COLLECTOR_PROPERTY; +import static org.apache.hadoop.metrics2.sink.timeline.AbstractTimelineMetricsSink.COLLECTOR_HOSTS_PROPERTY; import static org.apache.hadoop.metrics2.sink.timeline.AbstractTimelineMetricsSink.COLLECTOR_PROTOCOL; import static org.apache.hadoop.metrics2.sink.timeline.AbstractTimelineMetricsSink.MAX_METRIC_ROW_CACHE_SIZE; import static org.apache.hadoop.metrics2.sink.timeline.AbstractTimelineMetricsSink.METRICS_SEND_INTERVAL; @@ -100,7 +100,7 @@ public class HadoopTimelineMetricsSinkTest { expect(conf.getString("slave.host.name")).andReturn("localhost").anyTimes(); expect(conf.getParent()).andReturn(null).anyTimes(); expect(conf.getPrefix()).andReturn("service").anyTimes(); - expect(conf.getString(eq(COLLECTOR_PROPERTY), eq(""))).andReturn("localhost:6188").anyTimes(); + expect(conf.getStringArray(eq(COLLECTOR_HOSTS_PROPERTY))).andReturn(new String[]{"['localhost", "'localhost2']"}).anyTimes(); expect(conf.getString(eq("serviceName-prefix"), eq(""))).andReturn("").anyTimes(); expect(conf.getString(eq(COLLECTOR_PROTOCOL), eq("http"))).andReturn("http").anyTimes(); expect(conf.getString(eq(COLLECTOR_PORT), eq("6188"))).andReturn("6188").anyTimes(); @@ -172,7 +172,7 @@ public class HadoopTimelineMetricsSinkTest { expect(conf.getString("slave.host.name")).andReturn("localhost").anyTimes(); expect(conf.getParent()).andReturn(null).anyTimes(); expect(conf.getPrefix()).andReturn("service").anyTimes(); - expect(conf.getString(eq(COLLECTOR_PROPERTY), eq(""))).andReturn("localhost:6188").anyTimes(); + expect(conf.getStringArray(eq(COLLECTOR_HOSTS_PROPERTY))).andReturn(new String[]{"['localhost", "'localhost2']"}).anyTimes(); expect(conf.getString(eq("serviceName-prefix"), eq(""))).andReturn("").anyTimes(); expect(conf.getString(eq(COLLECTOR_PROTOCOL), eq("http"))).andReturn("http").anyTimes(); expect(conf.getString(eq(COLLECTOR_PORT), eq("6188"))).andReturn("6188").anyTimes(); @@ -184,6 +184,8 @@ public class HadoopTimelineMetricsSinkTest { expect(sink.findLiveCollectorHostsFromKnownCollector("localhost", "6188")) .andReturn(Collections.singletonList("localhost")).anyTimes(); + expect(sink.findLiveCollectorHostsFromKnownCollector("localhost2", "6188")) + .andReturn(Collections.singletonList("localhost2")).anyTimes(); conf.setListDelimiter(eq(',')); expectLastCall().anyTimes(); @@ -300,13 +302,15 @@ public class HadoopTimelineMetricsSinkTest { expect(conf.getString("slave.host.name")).andReturn("localhost").anyTimes(); expect(conf.getParent()).andReturn(null).anyTimes(); expect(conf.getPrefix()).andReturn("service").anyTimes(); - expect(conf.getString(eq(COLLECTOR_PROPERTY), eq(""))).andReturn("localhost:6188").anyTimes(); + expect(conf.getStringArray(eq(COLLECTOR_HOSTS_PROPERTY))).andReturn(new String[]{"['localhost", "'localhost2']"}).anyTimes(); expect(conf.getString(eq("serviceName-prefix"), eq(""))).andReturn("").anyTimes(); expect(conf.getString(eq(COLLECTOR_PROTOCOL), eq("http"))).andReturn("http").anyTimes(); expect(conf.getString(eq(COLLECTOR_PORT), eq("6188"))).andReturn("6188").anyTimes(); expect(sink.findLiveCollectorHostsFromKnownCollector("localhost", "6188")) .andReturn(Collections.singletonList("localhost")).anyTimes(); + expect(sink.findLiveCollectorHostsFromKnownCollector("localhost2", "6188")) + .andReturn(Collections.singletonList("localhost2")).anyTimes(); expect(conf.getInt(eq(MAX_METRIC_ROW_CACHE_SIZE), anyInt())).andReturn(10).anyTimes(); expect(conf.getInt(eq(METRICS_SEND_INTERVAL), anyInt())).andReturn(10).anyTimes(); http://git-wip-us.apache.org/repos/asf/ambari/blob/0d01ab25/ambari-metrics/ambari-metrics-kafka-sink/src/main/java/org/apache/hadoop/metrics2/sink/kafka/KafkaTimelineMetricsReporter.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-kafka-sink/src/main/java/org/apache/hadoop/metrics2/sink/kafka/KafkaTimelineMetricsReporter.java b/ambari-metrics/ambari-metrics-kafka-sink/src/main/java/org/apache/hadoop/metrics2/sink/kafka/KafkaTimelineMetricsReporter.java index 9c16564..fef1f24 100644 --- a/ambari-metrics/ambari-metrics-kafka-sink/src/main/java/org/apache/hadoop/metrics2/sink/kafka/KafkaTimelineMetricsReporter.java +++ b/ambari-metrics/ambari-metrics-kafka-sink/src/main/java/org/apache/hadoop/metrics2/sink/kafka/KafkaTimelineMetricsReporter.java @@ -45,6 +45,7 @@ import org.apache.hadoop.metrics2.sink.timeline.cache.TimelineMetricsCache; import java.net.InetAddress; import java.net.UnknownHostException; import java.util.ArrayList; +import java.util.Collection; import java.util.HashSet; import java.util.List; import java.util.Map.Entry; @@ -76,7 +77,7 @@ public class KafkaTimelineMetricsReporter extends AbstractTimelineMetricsSink private final Object lock = new Object(); private String hostname; private String metricCollectorPort; - private String collectors; + private Collection<String> collectorHosts; private String metricCollectorProtocol; private TimelineScheduledReporter reporter; private TimelineMetricsCache metricsCache; @@ -99,6 +100,11 @@ public class KafkaTimelineMetricsReporter extends AbstractTimelineMetricsSink } @Override + protected String getCollectorPort() { + return metricCollectorPort; + } + + @Override protected int getTimeoutSeconds() { return timeoutSeconds; } @@ -109,8 +115,8 @@ public class KafkaTimelineMetricsReporter extends AbstractTimelineMetricsSink } @Override - protected String getConfiguredCollectors() { - return collectors; + protected Collection<String> getConfiguredCollectorHosts() { + return collectorHosts; } @Override @@ -147,7 +153,7 @@ public class KafkaTimelineMetricsReporter extends AbstractTimelineMetricsSink zookeeperQuorum = props.getString("zookeeper.connect"); metricCollectorPort = props.getString(TIMELINE_PORT_PROPERTY, TIMELINE_DEFAULT_PORT); - collectors = props.getString(TIMELINE_HOSTS_PROPERTY, TIMELINE_DEFAULT_HOST + ":" + metricCollectorPort); + collectorHosts = parseHostsStringIntoCollection(props.getString(TIMELINE_HOSTS_PROPERTY, TIMELINE_DEFAULT_HOST)); metricCollectorProtocol = props.getString(TIMELINE_PROTOCOL_PROPERTY, TIMELINE_DEFAULT_PROTOCOL); setMetricsCache(new TimelineMetricsCache(maxRowCacheSize, metricsSendInterval)); http://git-wip-us.apache.org/repos/asf/ambari/blob/0d01ab25/ambari-metrics/ambari-metrics-storm-sink-legacy/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsReporter.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-storm-sink-legacy/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsReporter.java b/ambari-metrics/ambari-metrics-storm-sink-legacy/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsReporter.java index 4294837..95e5ba1 100644 --- a/ambari-metrics/ambari-metrics-storm-sink-legacy/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsReporter.java +++ b/ambari-metrics/ambari-metrics-storm-sink-legacy/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsReporter.java @@ -33,6 +33,7 @@ import org.apache.hadoop.metrics2.sink.timeline.UnableToConnectException; import java.net.InetAddress; import java.net.UnknownHostException; import java.util.ArrayList; +import java.util.Collection; import java.util.List; import java.util.Map; @@ -48,7 +49,7 @@ public class StormTimelineMetricsReporter extends AbstractTimelineMetricsSink private String applicationId; private int timeoutSeconds; private String port; - private String collectors; + private Collection<String> collectorHosts; private String zkQuorum; private String protocol; @@ -77,8 +78,13 @@ public class StormTimelineMetricsReporter extends AbstractTimelineMetricsSink } @Override - protected String getConfiguredCollectors() { - return collectors; + protected Collection<String> getConfiguredCollectorHosts() { + return collectorHosts; + } + + @Override + protected String getCollectorPort() { + return port; } @Override @@ -105,7 +111,7 @@ public class StormTimelineMetricsReporter extends AbstractTimelineMetricsSink Map stormConf = Utils.readStormConfig(); this.nimbusClient = NimbusClient.getConfiguredClient(stormConf); - collectors = cf.get(COLLECTOR_PROPERTY).toString(); + collectorHosts = parseHostsStringIntoCollection(cf.get(COLLECTOR_HOSTS_PROPERTY).toString()); protocol = cf.get(COLLECTOR_PROTOCOL) != null ? cf.get(COLLECTOR_PROTOCOL).toString() : "http"; port = cf.get(COLLECTOR_PORT) != null ? cf.get(COLLECTOR_PORT).toString() : "6188"; zkQuorum = cf.get(ZOOKEEPER_QUORUM) != null ? cf.get(ZOOKEEPER_QUORUM).toString() : null; http://git-wip-us.apache.org/repos/asf/ambari/blob/0d01ab25/ambari-metrics/ambari-metrics-storm-sink-legacy/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsSink.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-storm-sink-legacy/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsSink.java b/ambari-metrics/ambari-metrics-storm-sink-legacy/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsSink.java index 80f0333..4989696 100644 --- a/ambari-metrics/ambari-metrics-storm-sink-legacy/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsSink.java +++ b/ambari-metrics/ambari-metrics-storm-sink-legacy/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsSink.java @@ -53,16 +53,16 @@ public class StormTimelineMetricsSink extends AbstractTimelineMetricsSink implem private TimelineMetricsCache metricsCache; private String hostname; private int timeoutSeconds; - private String topologyName; - private String applicationId; - private String collectors; + private Collection<String> collectorHosts; private String zkQuorum; private String protocol; private String port; + private String topologyName; + private String applicationId; @Override protected String getCollectorUri(String host) { - return collectorUri; + return constructTimelineMetricUri(protocol, host, port); } @Override @@ -81,8 +81,13 @@ public class StormTimelineMetricsSink extends AbstractTimelineMetricsSink implem } @Override - protected String getConfiguredCollectors() { - return collectors; + protected Collection<String> getConfiguredCollectorHosts() { + return collectorHosts; + } + + @Override + protected String getCollectorPort() { + return port; } @Override @@ -112,7 +117,6 @@ public class StormTimelineMetricsSink extends AbstractTimelineMetricsSink implem String.valueOf(MAX_EVICTION_TIME_MILLIS))); applicationId = configuration.getProperty(CLUSTER_REPORTER_APP_ID, DEFAULT_CLUSTER_REPORTER_APP_ID); metricsCache = new TimelineMetricsCache(maxRowCacheSize, metricsSendInterval); - collectors = configuration.getProperty(COLLECTOR_PROPERTY); zkQuorum = configuration.getProperty("zookeeper.quorum"); protocol = configuration.getProperty(COLLECTOR_PROTOCOL, "http"); @@ -121,7 +125,7 @@ public class StormTimelineMetricsSink extends AbstractTimelineMetricsSink implem // Initialize the collector write strategy super.init(); - if (protocol.toLowerCase().startsWith("https://";)) { + if (protocol.contains("https")) { String trustStorePath = configuration.getProperty(SSL_KEYSTORE_PATH_PROPERTY).trim(); String trustStoreType = configuration.getProperty(SSL_KEYSTORE_TYPE_PROPERTY).trim(); String trustStorePwd = configuration.getProperty(SSL_KEYSTORE_PASSWORD_PROPERTY).trim(); @@ -235,7 +239,7 @@ public class StormTimelineMetricsSink extends AbstractTimelineMetricsSink implem } private TimelineMetric createTimelineMetric(long currentTimeMillis, String hostName, - String attributeName, Double attributeValue) { + String attributeName, Double attributeValue) { TimelineMetric timelineMetric = new TimelineMetric(); timelineMetric.setMetricName(attributeName); timelineMetric.setHostName(hostName); http://git-wip-us.apache.org/repos/asf/ambari/blob/0d01ab25/ambari-metrics/ambari-metrics-storm-sink/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsReporter.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-storm-sink/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsReporter.java b/ambari-metrics/ambari-metrics-storm-sink/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsReporter.java index 4c546ad..5c61e03 100644 --- a/ambari-metrics/ambari-metrics-storm-sink/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsReporter.java +++ b/ambari-metrics/ambari-metrics-storm-sink/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsReporter.java @@ -18,7 +18,6 @@ package org.apache.hadoop.metrics2.sink.storm; -import org.apache.commons.lang3.Validate; import org.apache.commons.lang3.ClassUtils; import org.apache.hadoop.metrics2.sink.timeline.AbstractTimelineMetricsSink; import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric; @@ -27,7 +26,6 @@ import org.apache.hadoop.metrics2.sink.timeline.UnableToConnectException; import org.apache.hadoop.metrics2.sink.timeline.configuration.Configuration; import org.apache.storm.metric.api.DataPoint; import org.apache.storm.metric.api.IClusterMetricsConsumer; -import org.apache.storm.utils.Utils; import java.net.InetAddress; import java.net.UnknownHostException; @@ -47,7 +45,7 @@ public class StormTimelineMetricsReporter extends AbstractTimelineMetricsSink private String applicationId; private int timeoutSeconds; private String port; - private String collectors; + private Collection<String> collectorHosts; private String zkQuorum; private String protocol; @@ -75,12 +73,15 @@ public class StormTimelineMetricsReporter extends AbstractTimelineMetricsSink return zkQuorum; } - @Override - protected String getConfiguredCollectors() { - return collectors; + protected Collection<String> getConfiguredCollectorHosts() { + return collectorHosts; } @Override + protected String getCollectorPort() { + return port; + } + @Override protected String getHostname() { return hostname; } @@ -102,8 +103,7 @@ public class StormTimelineMetricsReporter extends AbstractTimelineMetricsSink } Configuration conf = new Configuration("/storm-metrics2.properties"); - collectors = conf.getProperty(COLLECTOR_PROPERTY); - protocol = conf.getProperty(COLLECTOR_PROTOCOL) != null ? conf.getProperty(COLLECTOR_PROTOCOL) : "http"; + collectorHosts = parseHostsStringIntoCollection(conf.getProperty(COLLECTOR_HOSTS_PROPERTY).toString()); port = conf.getProperty(COLLECTOR_PORT) != null ? conf.getProperty(COLLECTOR_PORT) : "6188"; zkQuorum = conf.getProperty(ZOOKEEPER_QUORUM) != null ? conf.getProperty(ZOOKEEPER_QUORUM) : null; http://git-wip-us.apache.org/repos/asf/ambari/blob/0d01ab25/ambari-metrics/ambari-metrics-storm-sink/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsSink.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-storm-sink/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsSink.java b/ambari-metrics/ambari-metrics-storm-sink/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsSink.java index f8c34d5..3c5655f 100644 --- a/ambari-metrics/ambari-metrics-storm-sink/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsSink.java +++ b/ambari-metrics/ambari-metrics-storm-sink/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsSink.java @@ -61,12 +61,12 @@ public class StormTimelineMetricsSink extends AbstractTimelineMetricsSink implem private TimelineMetricsCache metricsCache; private String hostname; private int timeoutSeconds; - private String topologyName; - private String applicationId; - private String collectors; + private Collection<String> collectorHosts; private String zkQuorum; private String protocol; private String port; + private String topologyName; + private String applicationId; @Override protected String getCollectorUri(String host) { @@ -89,8 +89,13 @@ public class StormTimelineMetricsSink extends AbstractTimelineMetricsSink implem } @Override - protected String getConfiguredCollectors() { - return collectors; + protected Collection<String> getConfiguredCollectorHosts() { + return collectorHosts; + } + + @Override + protected String getCollectorPort() { + return port; } @Override @@ -120,8 +125,7 @@ public class StormTimelineMetricsSink extends AbstractTimelineMetricsSink implem String.valueOf(MAX_EVICTION_TIME_MILLIS))); applicationId = configuration.getProperty(CLUSTER_REPORTER_APP_ID, DEFAULT_CLUSTER_REPORTER_APP_ID); metricsCache = new TimelineMetricsCache(maxRowCacheSize, metricsSendInterval); - - collectors = configuration.getProperty(COLLECTOR_PROPERTY); + collectorHosts = parseHostsStringIntoCollection(configuration.getProperty(COLLECTOR_HOSTS_PROPERTY)); zkQuorum = configuration.getProperty("zookeeper.quorum"); protocol = configuration.getProperty(COLLECTOR_PROTOCOL, "http"); port = configuration.getProperty(COLLECTOR_PORT, "6188"); http://git-wip-us.apache.org/repos/asf/ambari/blob/0d01ab25/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/scripts/params.py index a8a7327..e00c1f5 100644 --- a/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/scripts/params.py @@ -126,11 +126,6 @@ ams_collector_hosts = default("/clusterHostInfo/metrics_collector_hosts", []) has_metric_collector = not len(ams_collector_hosts) == 0 if has_metric_collector: if 'cluster-env' in config['configurations'] and \ - 'metrics_collector_vip_host' in config['configurations']['cluster-env']: - metric_collector_host = config['configurations']['cluster-env']['metrics_collector_vip_host'] - else: - metric_collector_host = select_metric_collector_hosts_from_hostnames(ams_collector_hosts) - if 'cluster-env' in config['configurations'] and \ 'metrics_collector_vip_port' in config['configurations']['cluster-env']: metric_collector_port = config['configurations']['cluster-env']['metrics_collector_vip_port'] else: http://git-wip-us.apache.org/repos/asf/ambari/blob/0d01ab25/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/templates/hadoop-metrics2-accumulo.properties.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/templates/hadoop-metrics2-accumulo.properties.j2 b/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/templates/hadoop-metrics2-accumulo.properties.j2 index 3cd535e..f7926d0 100644 --- a/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/templates/hadoop-metrics2-accumulo.properties.j2 +++ b/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/templates/hadoop-metrics2-accumulo.properties.j2 @@ -22,20 +22,24 @@ *.sink.timeline.slave.host.name={{hostname}} accumulo.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink accumulo.period={{metrics_collection_period}} -accumulo.collector={{metric_collector_host}}:{{metric_collector_port}} +accumulo.collector.hosts={{ams_collector_hosts}} +accumulo.port={{metric_collector_port}} jvm.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink jvm.period={{metrics_collection_period}} -jvm.collector={{metric_collector_host}}:{{metric_collector_port}} +jvm.collector.hosts={{ams_collector_hosts}} +jvm.port={{metric_collector_port}} rpc.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink rpc.period={{metrics_collection_period}} -rpc.collector={{metric_collector_host}}:{{metric_collector_port}} +rpc.collector.hosts={{ams_collector_hosts}} +rpc.port={{metric_collector_port}} accumulo.sink.timeline.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink accumulo.sink.timeline.period={{metrics_collection_period}} accumulo.sink.timeline.sendInterval={{metrics_report_interval}}000 -accumulo.sink.timeline.collector={{metric_collector_host}}:{{metric_collector_port}} +accumulo.sink.timeline.collector.hosts={{ams_collector_hosts}} +accumulo.sink.timeline.port={{metric_collector_port}} # HTTPS properties accumulo.sink.timeline.truststore.path = {{metric_truststore_path}} http://git-wip-us.apache.org/repos/asf/ambari/blob/0d01ab25/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/hadoop-metrics2-hbase.properties.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/hadoop-metrics2-hbase.properties.j2 b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/hadoop-metrics2-hbase.properties.j2 index 586cd47..8c20f2b 100644 --- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/hadoop-metrics2-hbase.properties.j2 +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/hadoop-metrics2-hbase.properties.j2 @@ -40,22 +40,26 @@ hbase.extendedperiod = 3600 hbase.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink hbase.period=30 -hbase.collector={{metric_collector_host}}:{{metric_collector_port}} +hbase.collector.hosts={{ams_collector_hosts}} +hbase.port={{metric_collector_port}} jvm.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink jvm.period=30 -jvm.collector={{metric_collector_host}}:{{metric_collector_port}} +jvm.collector.hosts={{ams_collector_hosts}} +jvm.port={{metric_collector_port}} rpc.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink rpc.period=30 -rpc.collector={{metric_collector_host}}:{{metric_collector_port}} +rpc.collector.hosts={{ams_collector_hosts}} +rpc.port={{metric_collector_port}} *.timeline.plugin.urls=file:///usr/lib/ambari-metrics-hadoop-sink/ambari-metrics-hadoop-sink.jar *.sink.timeline.slave.host.name={{hostname}} hbase.sink.timeline.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink hbase.sink.timeline.period={{metrics_collection_period}} hbase.sink.timeline.sendInterval={{metrics_report_interval}}000 -hbase.sink.timeline.collector={{metric_collector_host}}:{{metric_collector_port}} +hbase.sink.timeline.collector.hosts={{ams_collector_hosts}} +hbase.sink.timeline.port={{metric_collector_port}} hbase.sink.timeline.serviceName-prefix=ams # HTTPS properties @@ -65,4 +69,4 @@ hbase.sink.timeline.truststore.password = {{metric_truststore_password}} # Switch off metrics generation on a per region basis *.source.filter.class=org.apache.hadoop.metrics2.filter.RegexFilter -hbase.*.source.filter.exclude=.*(Regions|Users|Tables).* \ No newline at end of file +hbase.*.source.filter.exclude=.*(Regions|Users|Tables).* http://git-wip-us.apache.org/repos/asf/ambari/blob/0d01ab25/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/flume.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/flume.py b/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/flume.py index 1ade2c1..81b077f 100644 --- a/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/flume.py +++ b/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/flume.py @@ -177,6 +177,7 @@ def flume(action = None): if params.has_metric_collector: extra_args = '-Dflume.monitoring.type=org.apache.hadoop.metrics2.sink.flume.FlumeTimelineMetricsSink ' \ '-Dflume.monitoring.node={0}:{1}' + # TODO check if this is used. extra_args = extra_args.format(params.metric_collector_host, params.metric_collector_port) flume_cmd = flume_base.format(agent, flume_agent_conf_dir, http://git-wip-us.apache.org/repos/asf/ambari/blob/0d01ab25/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/params.py index 9c79909..008e9d6 100644 --- a/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/params.py @@ -114,13 +114,6 @@ if has_metric_collector: metrics_report_interval = default("/configurations/ams-site/timeline.metrics.sink.report.interval", 60) metrics_collection_period = default("/configurations/ams-site/timeline.metrics.sink.collection.period", 10) -#Collector hosts -metric_collector_hosts = "" -if ams_collector_hosts: - for host in ams_collector_hosts: - metric_collector_hosts += host + ':' + metric_collector_port + ',' - metric_collector_hosts = metric_collector_hosts[:-1] - # Cluster Zookeeper quorum zookeeper_quorum = None if not len(default("/clusterHostInfo/zookeeper_hosts", [])) == 0: http://git-wip-us.apache.org/repos/asf/ambari/blob/0d01ab25/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/templates/flume-metrics2.properties.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/templates/flume-metrics2.properties.j2 b/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/templates/flume-metrics2.properties.j2 index b960296..cf2bd6c 100644 --- a/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/templates/flume-metrics2.properties.j2 +++ b/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/templates/flume-metrics2.properties.j2 @@ -16,7 +16,7 @@ # limitations under the License. #} -collector={{metric_collector_hosts}} +collector.hosts={{ams_collector_hosts}} protocol={{metric_collector_protocol}} zookeeper.quorum={{zookeeper_quorum}} port={{metric_collector_port}} http://git-wip-us.apache.org/repos/asf/ambari/blob/0d01ab25/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py index 10b1e7d..1f32c41 100644 --- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py @@ -155,11 +155,6 @@ ams_collector_hosts = default("/clusterHostInfo/metrics_collector_hosts", []) has_metric_collector = not len(ams_collector_hosts) == 0 if has_metric_collector: if 'cluster-env' in config['configurations'] and \ - 'metrics_collector_vip_host' in config['configurations']['cluster-env']: - metric_collector_host = config['configurations']['cluster-env']['metrics_collector_vip_host'] - else: - metric_collector_host = select_metric_collector_hosts_from_hostnames(ams_collector_hosts) - if 'cluster-env' in config['configurations'] and \ 'metrics_collector_vip_port' in config['configurations']['cluster-env']: metric_collector_port = config['configurations']['cluster-env']['metrics_collector_vip_port'] else: @@ -180,13 +175,6 @@ if has_metric_collector: metrics_report_interval = default("/configurations/ams-site/timeline.metrics.sink.report.interval", 60) metrics_collection_period = default("/configurations/ams-site/timeline.metrics.sink.collection.period", 10) -#Collector hosts -metric_collector_hosts = "" -if ams_collector_hosts: - for host in ams_collector_hosts: - metric_collector_hosts += host + ':' + metric_collector_port + ',' - metric_collector_hosts = metric_collector_hosts[:-1] - # if hbase is selected the hbase_rs_hosts, should not be empty, but still default just in case if 'slave_hosts' in config['clusterHostInfo']: rs_hosts = default('/clusterHostInfo/hbase_rs_hosts', '/clusterHostInfo/slave_hosts') #if hbase_rs_hosts not given it is assumed that region servers on same nodes as slaves http://git-wip-us.apache.org/repos/asf/ambari/blob/0d01ab25/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-MASTER.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-MASTER.j2 b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-MASTER.j2 index e62ce9e..e808f5d 100644 --- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-MASTER.j2 +++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-MASTER.j2 @@ -54,21 +54,28 @@ hbase.extendedperiod = 3600 hbase.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink hbase.period={{metrics_collection_period}} -hbase.collector={{metric_collector_host}}:{{metric_collector_port}} +hbase.collector.hosts={{ams_collector_hosts}} +hbase.protocol={{metric_collector_protocol}} +hbase.port={{metric_collector_port}} jvm.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink jvm.period={{metrics_collection_period}} -jvm.collector={{metric_collector_host}}:{{metric_collector_port}} +jvm.collector.hosts={{ams_collector_hosts}} +jvm.protocol={{metric_collector_protocol}} +jvm.port={{metric_collector_port}} rpc.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink rpc.period={{metrics_collection_period}} -rpc.collector={{metric_collector_host}}:{{metric_collector_port}} +rpc.collector.hosts={{ams_collector_hosts}} +rpc.protocol={{metric_collector_protocol}} +rpc.port={{metric_collector_port}} hbase.sink.timeline.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink hbase.sink.timeline.period={{metrics_collection_period}} hbase.sink.timeline.sendInterval={{metrics_report_interval}}000 -hbase.sink.timeline.collector={{metric_collector_hosts}} +hbase.sink.timeline.collector.hosts={{ams_collector_hosts}} hbase.sink.timeline.protocol={{metric_collector_protocol}} +hbase.sink.timeline.port={{metric_collector_port}} # HTTPS properties hbase.sink.timeline.truststore.path = {{metric_truststore_path}} http://git-wip-us.apache.org/repos/asf/ambari/blob/0d01ab25/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-RS.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-RS.j2 b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-RS.j2 index 8183333..7e1abbc 100644 --- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-RS.j2 +++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-RS.j2 @@ -52,21 +52,28 @@ hbase.extendedperiod = 3600 *.sink.timeline.slave.host.name={{hostname}} hbase.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink hbase.period={{metrics_collection_period}} -hbase.collector={{metric_collector_host}}:{{metric_collector_port}} +hbase.collector.hosts={{ams_collector_hosts}} +hbase.protocol={{metric_collector_protocol}} +hbase.port={{metric_collector_port}} jvm.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink jvm.period={{metrics_collection_period}} -jvm.collector={{metric_collector_host}}:{{metric_collector_port}} +jvm.collector.hosts={{ams_collector_hosts}} +jvm.protocol={{metric_collector_protocol}} +jvm.port={{metric_collector_port}} rpc.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink rpc.period={{metrics_collection_period}} -rpc.collector={{metric_collector_host}}:{{metric_collector_port}} +rpc.collector.hosts={{ams_collector_hosts}} +rpc.protocol={{metric_collector_protocol}} +rpc.port={{metric_collector_port}} hbase.sink.timeline.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink hbase.sink.timeline.period={{metrics_collection_period}} hbase.sink.timeline.sendInterval={{metrics_report_interval}}000 -hbase.sink.timeline.collector={{metric_collector_hosts}} +hbase.sink.timeline.collector.hosts={{ams_collector_hosts}} hbase.sink.timeline.protocol={{metric_collector_protocol}} +hbase.sink.timeline.port={{metric_collector_port}} # HTTPS properties http://git-wip-us.apache.org/repos/asf/ambari/blob/0d01ab25/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/configuration/hadoop-metrics2.properties.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/configuration/hadoop-metrics2.properties.xml b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/configuration/hadoop-metrics2.properties.xml index 2b9964b..6b45e84 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/configuration/hadoop-metrics2.properties.xml +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/configuration/hadoop-metrics2.properties.xml @@ -92,15 +92,15 @@ resourcemanager.sink.ganglia.tagsForPrefix.yarn=Queue *.sink.timeline.truststore.type = {{metric_truststore_type}} *.sink.timeline.truststore.password = {{metric_truststore_password}} -datanode.sink.timeline.collector={{metric_collector_hosts}} -namenode.sink.timeline.collector={{metric_collector_hosts}} -resourcemanager.sink.timeline.collector={{metric_collector_hosts}} -nodemanager.sink.timeline.collector={{metric_collector_hosts}} -jobhistoryserver.sink.timeline.collector={{metric_collector_hosts}} -journalnode.sink.timeline.collector={{metric_collector_hosts}} -maptask.sink.timeline.collector={{metric_collector_hosts}} -reducetask.sink.timeline.collector={{metric_collector_hosts}} -applicationhistoryserver.sink.timeline.collector={{metric_collector_hosts}} +datanode.sink.timeline.collector.hosts={{ams_collector_hosts}} +namenode.sink.timeline.collector.hosts={{ams_collector_hosts}} +resourcemanager.sink.timeline.collector.hosts={{ams_collector_hosts}} +nodemanager.sink.timeline.collector.hosts={{ams_collector_hosts}} +jobhistoryserver.sink.timeline.collector.hosts={{ams_collector_hosts}} +journalnode.sink.timeline.collector.hosts={{ams_collector_hosts}} +maptask.sink.timeline.collector.hosts={{ams_collector_hosts}} +reducetask.sink.timeline.collector.hosts={{ams_collector_hosts}} +applicationhistoryserver.sink.timeline.collector.hosts={{ams_collector_hosts}} resourcemanager.sink.timeline.tagsForPrefix.yarn=Queue http://git-wip-us.apache.org/repos/asf/ambari/blob/0d01ab25/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py index 6fd53a8..e214547 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py @@ -493,11 +493,6 @@ ams_collector_hosts = default("/clusterHostInfo/metrics_collector_hosts", []) has_metric_collector = not len(ams_collector_hosts) == 0 if has_metric_collector: if 'cluster-env' in config['configurations'] and \ - 'metrics_collector_vip_host' in config['configurations']['cluster-env']: - metric_collector_host = config['configurations']['cluster-env']['metrics_collector_vip_host'] - else: - metric_collector_host = select_metric_collector_hosts_from_hostnames(ams_collector_hosts) - if 'cluster-env' in config['configurations'] and \ 'metrics_collector_vip_port' in config['configurations']['cluster-env']: metric_collector_port = config['configurations']['cluster-env']['metrics_collector_vip_port'] else: http://git-wip-us.apache.org/repos/asf/ambari/blob/0d01ab25/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-hivemetastore.properties.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-hivemetastore.properties.j2 b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-hivemetastore.properties.j2 index 88be81b..d2e43e9 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-hivemetastore.properties.j2 +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-hivemetastore.properties.j2 @@ -48,7 +48,8 @@ *.sink.timeline.truststore.type = {{metric_truststore_type}} *.sink.timeline.truststore.password = {{metric_truststore_password}} - hivemetastore.sink.timeline.collector={{metric_collector_host}}:{{metric_collector_port}} + hivemetastore.sink.timeline.collector.hosts={{ams_collector_hosts}} + hivemetastore.sink.timeline.port={{metric_collector_port}} {% endif %} http://git-wip-us.apache.org/repos/asf/ambari/blob/0d01ab25/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-hiveserver2.properties.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-hiveserver2.properties.j2 b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-hiveserver2.properties.j2 index 3d71867..ed89679 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-hiveserver2.properties.j2 +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-hiveserver2.properties.j2 @@ -48,7 +48,7 @@ *.sink.timeline.truststore.type = {{metric_truststore_type}} *.sink.timeline.truststore.password = {{metric_truststore_password}} - hiveserver2.sink.timeline.collector={{metric_collector_host}}:{{metric_collector_port}} - + hiveserver2.sink.timeline.collector.hosts={{ams_collector_hosts}} + hiveserver2.sink.timeline.port={{metric_collector_port}} {% endif %} http://git-wip-us.apache.org/repos/asf/ambari/blob/0d01ab25/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-llapdaemon.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-llapdaemon.j2 b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-llapdaemon.j2 index ba44af5..6b609dd 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-llapdaemon.j2 +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-llapdaemon.j2 @@ -47,6 +47,7 @@ *.sink.timeline.truststore.type = {{metric_truststore_type}} *.sink.timeline.truststore.password = {{metric_truststore_password}} - llapdaemon.sink.timeline.collector={{metric_collector_host}}:{{metric_collector_port}} + llapdaemon.sink.timeline.collector.hosts={{ams_collector_hosts}} + llapdaemon.sink.timeline.port={{metric_collector_port}} {% endif %} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/0d01ab25/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-llaptaskscheduler.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-llaptaskscheduler.j2 b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-llaptaskscheduler.j2 index 77f128b..c229b75 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-llaptaskscheduler.j2 +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-llaptaskscheduler.j2 @@ -47,6 +47,7 @@ *.sink.timeline.truststore.type = {{metric_truststore_type}} *.sink.timeline.truststore.password = {{metric_truststore_password}} - llaptaskscheduler.sink.timeline.collector={{metric_collector_host}}:{{metric_collector_port}} + llaptaskscheduler.sink.timeline.collector.hosts={{ams_collector_hosts}} + llaptaskscheduler.sink.timeline.port={{metric_collector_port}} {% endif %} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/0d01ab25/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/configuration/kafka-broker.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/configuration/kafka-broker.xml b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/configuration/kafka-broker.xml index ab4d701..96b7750 100644 --- a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/configuration/kafka-broker.xml +++ b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/configuration/kafka-broker.xml @@ -358,7 +358,7 @@ </property> <property> <name>kafka.timeline.metrics.hosts</name> - <value>{{metric_collector_hosts}}</value> + <value>{{ams_collector_hosts}}</value> <description>Timeline host</description> <on-ambari-upgrade add="true"/> </property> http://git-wip-us.apache.org/repos/asf/ambari/blob/0d01ab25/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/kafka.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/kafka.py b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/kafka.py index 635cbe9..4e97022 100644 --- a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/kafka.py +++ b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/kafka.py @@ -99,7 +99,7 @@ def kafka(upgrade_type=None): kafka_server_config['host.name'] = params.hostname if params.has_metric_collector: - kafka_server_config['kafka.timeline.metrics.hosts'] = params.metric_collector_hosts + kafka_server_config['kafka.timeline.metrics.hosts'] = params.ams_collector_hosts kafka_server_config['kafka.timeline.metrics.port'] = params.metric_collector_port kafka_server_config['kafka.timeline.metrics.protocol'] = params.metric_collector_protocol kafka_server_config['kafka.timeline.metrics.truststore.path'] = params.metric_truststore_path http://git-wip-us.apache.org/repos/asf/ambari/blob/0d01ab25/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py index 9c427ee..0cb88fe 100644 --- a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py @@ -139,13 +139,6 @@ if has_metric_collector: metric_collector_protocol = 'http' pass - # Collector hosts - metric_collector_hosts = "" - if ams_collector_hosts: - for host in ams_collector_hosts: - metric_collector_hosts += host + ':' + metric_collector_port + ',' - metric_collector_hosts = metric_collector_hosts[:-1] - # Security-related params security_enabled = config['configurations']['cluster-env']['security_enabled'] kafka_kerberos_enabled = (('security.inter.broker.protocol' in config['configurations']['kafka-broker']) and http://git-wip-us.apache.org/repos/asf/ambari/blob/0d01ab25/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py index 697015f..ff590ee 100644 --- a/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py @@ -174,11 +174,6 @@ has_metric_collector = not len(ams_collector_hosts) == 0 metric_collector_port = None if has_metric_collector: if 'cluster-env' in config['configurations'] and \ - 'metrics_collector_vip_host' in config['configurations']['cluster-env']: - metric_collector_host = config['configurations']['cluster-env']['metrics_collector_vip_host'] - else: - metric_collector_host = select_metric_collector_hosts_from_hostnames(ams_collector_hosts) - if 'cluster-env' in config['configurations'] and \ 'metrics_collector_vip_port' in config['configurations']['cluster-env']: metric_collector_port = config['configurations']['cluster-env']['metrics_collector_vip_port'] else: @@ -203,12 +198,6 @@ metrics_collection_period = default("/configurations/ams-site/timeline.metrics.s metric_collector_sink_jar = "/usr/lib/storm/lib/ambari-metrics-storm-sink-with-common-*.jar" metric_collector_legacy_sink_jar = "/usr/lib/storm/lib/ambari-metrics-storm-sink-legacy-with-common-*.jar" -# Collector hosts -metric_collector_hosts = "" -if ams_collector_hosts: - for host in ams_collector_hosts: - metric_collector_hosts += host + ':' + metric_collector_port + ',' - metric_collector_hosts = metric_collector_hosts[:-1] # Cluster Zookeeper quorum zookeeper_quorum = "" http://git-wip-us.apache.org/repos/asf/ambari/blob/0d01ab25/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/templates/config.yaml.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/templates/config.yaml.j2 b/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/templates/config.yaml.j2 index a9760cb..f2a75cf 100644 --- a/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/templates/config.yaml.j2 +++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/templates/config.yaml.j2 @@ -57,7 +57,9 @@ enableMetricsSink: True metrics_collector: reportInterval: {{metric_collector_report_interval}} - collector: "{{metric_collector_protocol}}://{{metric_collector_host}}:{{metric_collector_port}}" + collector.hosts: "{{ams_collector_hosts}}" + protocol: "{{metric_collector_protocol}}" + port: "{{metric_collector_port}}" appId: "{{metric_collector_app_id}}" # HTTPS settings @@ -65,4 +67,4 @@ metrics_collector: truststore.type : "{{metric_truststore_type}}" truststore.password : "{{metric_truststore_password}}" -{% endif %} \ No newline at end of file +{% endif %} http://git-wip-us.apache.org/repos/asf/ambari/blob/0d01ab25/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/templates/storm-metrics2.properties.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/templates/storm-metrics2.properties.j2 b/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/templates/storm-metrics2.properties.j2 index 1f0875f..001ff73 100644 --- a/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/templates/storm-metrics2.properties.j2 +++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/templates/storm-metrics2.properties.j2 @@ -16,7 +16,7 @@ # limitations under the License. #} -collector={{metric_collector_hosts}} +collector.hosts={{ams_collector_hosts}} protocol={{metric_collector_protocol}} port={{metric_collector_port}} zookeeper.quorum={{zookeeper_quorum}} http://git-wip-us.apache.org/repos/asf/ambari/blob/0d01ab25/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/params.py index c678a72..4603c8b 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/params.py @@ -136,13 +136,6 @@ if has_metric_collector: metrics_report_interval = default("/configurations/ams-site/timeline.metrics.sink.report.interval", 60) metrics_collection_period = default("/configurations/ams-site/timeline.metrics.sink.collection.period", 10) -#Collector hosts -metric_collector_hosts = "" -if ams_collector_hosts: - for host in ams_collector_hosts: - metric_collector_hosts += host + ':' + metric_collector_port + ',' - metric_collector_hosts = metric_collector_hosts[:-1] - # Cluster Zookeeper quorum zookeeper_quorum = None if has_zk_host: http://git-wip-us.apache.org/repos/asf/ambari/blob/0d01ab25/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/templates/hadoop-metrics2.properties.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/templates/hadoop-metrics2.properties.j2 b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/templates/hadoop-metrics2.properties.j2 index 28102d2..f44ef0e 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/templates/hadoop-metrics2.properties.j2 +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/templates/hadoop-metrics2.properties.j2 @@ -81,13 +81,13 @@ resourcemanager.sink.ganglia.tagsForPrefix.yarn=Queue *.sink.timeline.truststore.type = {{metric_truststore_type}} *.sink.timeline.truststore.password = {{metric_truststore_password}} -datanode.sink.timeline.collector={{metric_collector_hosts}} -namenode.sink.timeline.collector={{metric_collector_hosts}} -resourcemanager.sink.timeline.collector={{metric_collector_hosts}} -nodemanager.sink.timeline.collector={{metric_collector_hosts}} -jobhistoryserver.sink.timeline.collector={{metric_collector_hosts}} -journalnode.sink.timeline.collector={{metric_collector_hosts}} -applicationhistoryserver.sink.timeline.collector={{metric_collector_hosts}} +datanode.sink.timeline.collector={{ams_collector_hosts}} +namenode.sink.timeline.collector={{ams_collector_hosts}} +resourcemanager.sink.timeline.collector={{ams_collector_hosts}} +nodemanager.sink.timeline.collector={{ams_collector_hosts}} +jobhistoryserver.sink.timeline.collector={{ams_collector_hosts}} +journalnode.sink.timeline.collector={{ams_collector_hosts}} +applicationhistoryserver.sink.timeline.collector={{ams_collector_hosts}} resourcemanager.sink.timeline.tagsForPrefix.yarn=Queue http://git-wip-us.apache.org/repos/asf/ambari/blob/0d01ab25/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py index b0ebc6b..8fda093 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py @@ -636,14 +636,10 @@ class HDP206StackAdvisor(DefaultStackAdvisor): amsCollectorHosts = self.getComponentHostNames(services, "AMBARI_METRICS", "METRICS_COLLECTOR") - if 'cluster-env' in services['configurations'] and \ - 'metrics_collector_vip_host' in services['configurations']['cluster-env']['properties']: - metric_collector_host = services['configurations']['cluster-env']['properties']['metrics_collector_vip_host'] - else: - # TODO set "timeline.metrics.service.webapp.address" to 0.0.0.0:port in upgrade catalog - metric_collector_host = '0.0.0.0' + # TODO set "timeline.metrics.service.webapp.address" to 0.0.0.0:port in upgrade catalog + timeline_metrics_service_webapp_address = '0.0.0.0' - putAmsSiteProperty("timeline.metrics.service.webapp.address", str(metric_collector_host) + ":6188") + putAmsSiteProperty("timeline.metrics.service.webapp.address", str(timeline_metrics_service_webapp_address) + ":6188") log_dir = "/var/log/ambari-metrics-collector" if "ams-env" in services["configurations"]:
