AMBARI-21244 Add https support to local metrics aggregator application (dsen)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/35e2851b Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/35e2851b Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/35e2851b Branch: refs/heads/branch-3.0-ams Commit: 35e2851b965f1b742e6e5044d7302988ee3e7245 Parents: d496b4e Author: Dmytro Sen <[email protected]> Authored: Wed Jun 21 15:49:27 2017 +0300 Committer: Dmytro Sen <[email protected]> Committed: Wed Jun 21 15:49:27 2017 +0300 ---------------------------------------------------------------------- .../logfeeder/metrics/LogFeederAMSClient.java | 5 + .../timeline/AbstractTimelineMetricsSink.java | 13 +- .../AbstractTimelineMetricSinkTest.java | 5 + .../availability/MetricCollectorHATest.java | 5 + .../cache/HandleConnectExceptionTest.java | 5 + .../sink/flume/FlumeTimelineMetricsSink.java | 13 +- .../timeline/HadoopTimelineMetricsSink.java | 14 +- .../timeline/HadoopTimelineMetricsSinkTest.java | 4 + .../ambari-metrics-host-aggregator/pom.xml | 10 ++ .../host/aggregator/AggregatorApplication.java | 51 ++++++- .../sink/timeline/AbstractMetricPublisher.java | 12 +- .../timeline/AggregatedMetricsPublisher.java | 5 + .../sink/timeline/RawMetricsPublisher.java | 5 + .../src/main/python/core/aggregator.py | 6 +- .../src/main/python/core/config_reader.py | 9 +- .../src/main/python/core/emitter.py | 42 +++--- .../src/main/python/core/host_info.py | 1 - .../src/main/python/core/stop_handler.py | 4 +- .../kafka/KafkaTimelineMetricsReporter.java | 10 +- .../storm/StormTimelineMetricsReporter.java | 16 ++- .../sink/storm/StormTimelineMetricsSink.java | 13 +- .../storm/StormTimelineMetricsReporter.java | 13 +- .../sink/storm/StormTimelineMetricsSink.java | 13 +- .../system/impl/AmbariMetricSinkImpl.java | 5 + .../1.6.1.2.2.0/package/scripts/params.py | 6 + .../hadoop-metrics2-accumulo.properties.j2 | 3 + .../0.1.0/configuration/ams-site.xml | 5 + .../AMBARI_METRICS/0.1.0/package/scripts/ams.py | 77 ++++++---- .../0.1.0/package/scripts/params.py | 8 +- .../hadoop-metrics2-hbase.properties.j2 | 3 + .../package/templates/metric_monitor.ini.j2 | 3 + .../FLUME/1.4.0.2.0/package/scripts/params.py | 6 + .../templates/flume-metrics2.properties.j2 | 3 + .../0.96.0.2.0/package/scripts/params_linux.py | 6 + ...-metrics2-hbase.properties-GANGLIA-MASTER.j2 | 3 + ...doop-metrics2-hbase.properties-GANGLIA-RS.j2 | 3 + .../hadoop-metrics2.properties.xml | 3 + .../0.12.0.2.0/package/scripts/params_linux.py | 6 + .../hadoop-metrics2-hivemetastore.properties.j2 | 4 +- .../hadoop-metrics2-hiveserver2.properties.j2 | 3 + .../templates/hadoop-metrics2-llapdaemon.j2 | 3 + .../hadoop-metrics2-llaptaskscheduler.j2 | 3 + .../2.1.0.3.0/package/scripts/params_linux.py | 6 + .../hadoop-metrics2-hivemetastore.properties.j2 | 4 +- .../hadoop-metrics2-hiveserver2.properties.j2 | 3 + .../templates/hadoop-metrics2-llapdaemon.j2 | 3 + .../hadoop-metrics2-llaptaskscheduler.j2 | 3 + .../KAFKA/0.8.1/configuration/kafka-broker.xml | 5 + .../KAFKA/0.8.1/package/scripts/params.py | 6 + .../STORM/0.9.1/package/scripts/params_linux.py | 6 + .../0.9.1/package/templates/config.yaml.j2 | 7 +- .../templates/storm-metrics2.properties.j2 | 3 + .../1.0.1.3.0/package/scripts/params_linux.py | 6 + .../1.0.1.3.0/package/templates/config.yaml.j2 | 7 +- .../templates/storm-metrics2.properties.j2 | 3 + .../2.0.6/hooks/before-START/scripts/params.py | 6 + .../templates/hadoop-metrics2.properties.j2 | 3 + .../hadoop-metrics2.properties.xml | 3 + .../3.0/hooks/before-START/scripts/params.py | 6 + .../templates/hadoop-metrics2.properties.j2 | 3 + .../system/impl/TestAmbariMetricsSinkImpl.java | 5 + .../AMBARI_METRICS/test_metrics_monitor.py | 142 +++++++++++++++++++ .../2.0.6/configs/default_ams_embedded.json | 1 + .../2.0/hooks/before-START/scripts/params.py | 6 + 64 files changed, 568 insertions(+), 96 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/35e2851b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/metrics/LogFeederAMSClient.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/metrics/LogFeederAMSClient.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/metrics/LogFeederAMSClient.java index 39526a5..bcdae1b 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/metrics/LogFeederAMSClient.java +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/metrics/LogFeederAMSClient.java @@ -99,6 +99,11 @@ public class LogFeederAMSClient extends AbstractTimelineMetricsSink { } @Override + protected String getHostInMemoryAggregationProtocol() { + return "http"; + } + + @Override protected boolean emitMetrics(TimelineMetrics metrics) { return super.emitMetrics(metrics); } http://git-wip-us.apache.org/repos/asf/ambari/blob/35e2851b/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java b/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java index a0e59d6..73ed3c4 100644 --- a/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java +++ b/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractTimelineMetricsSink.java @@ -80,6 +80,7 @@ public abstract class AbstractTimelineMetricsSink { public static final String SSL_KEYSTORE_PASSWORD_PROPERTY = "truststore.password"; public static final String HOST_IN_MEMORY_AGGREGATION_ENABLED_PROPERTY = "host_in_memory_aggregation"; public static final String HOST_IN_MEMORY_AGGREGATION_PORT_PROPERTY = "host_in_memory_aggregation_port"; + public static final String HOST_IN_MEMORY_AGGREGATION_PROTOCOL_PROPERTY = "host_in_memory_aggregation_protocol"; public static final String COLLECTOR_LIVE_NODES_PATH = "/ws/v1/timeline/metrics/livenodes"; public static final String INSTANCE_ID_PROPERTY = "instanceId"; public static final String SET_INSTANCE_ID_PROPERTY = "set.instanceId"; @@ -245,7 +246,11 @@ public abstract class AbstractTimelineMetricsSink { protected boolean emitMetrics(TimelineMetrics metrics) { String connectUrl; if (isHostInMemoryAggregationEnabled()) { - connectUrl = constructTimelineMetricUri("http", "localhost", String.valueOf(getHostInMemoryAggregationPort())); + String hostname = "localhost"; + if (getHostInMemoryAggregationProtocol().equalsIgnoreCase("https")) { + hostname = getHostname(); + } + connectUrl = constructTimelineMetricUri(getHostInMemoryAggregationProtocol(), hostname, String.valueOf(getHostInMemoryAggregationPort())); } else { String collectorHost = getCurrentCollectorHost(); connectUrl = getCollectorUri(collectorHost); @@ -582,4 +587,10 @@ public abstract class AbstractTimelineMetricsSink { * @return */ abstract protected int getHostInMemoryAggregationPort(); + + /** + * In memory aggregation protocol + * @return + */ + abstract protected String getHostInMemoryAggregationProtocol(); } http://git-wip-us.apache.org/repos/asf/ambari/blob/35e2851b/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/availability/AbstractTimelineMetricSinkTest.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/availability/AbstractTimelineMetricSinkTest.java b/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/availability/AbstractTimelineMetricSinkTest.java index ce2cf79..396d08d 100644 --- a/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/availability/AbstractTimelineMetricSinkTest.java +++ b/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/availability/AbstractTimelineMetricSinkTest.java @@ -100,6 +100,11 @@ public class AbstractTimelineMetricSinkTest { } @Override + protected String getHostInMemoryAggregationProtocol() { + return "http"; + } + + @Override public boolean emitMetrics(TimelineMetrics metrics) { super.init(); return super.emitMetrics(metrics); http://git-wip-us.apache.org/repos/asf/ambari/blob/35e2851b/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/availability/MetricCollectorHATest.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/availability/MetricCollectorHATest.java b/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/availability/MetricCollectorHATest.java index f0174d5..0abc5fc 100644 --- a/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/availability/MetricCollectorHATest.java +++ b/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/availability/MetricCollectorHATest.java @@ -202,5 +202,10 @@ public class MetricCollectorHATest { protected int getHostInMemoryAggregationPort() { return 61888; } + + @Override + protected String getHostInMemoryAggregationProtocol() { + return "http"; + } } } http://git-wip-us.apache.org/repos/asf/ambari/blob/35e2851b/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/cache/HandleConnectExceptionTest.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/cache/HandleConnectExceptionTest.java b/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/cache/HandleConnectExceptionTest.java index 4eb75eb..c91d88c 100644 --- a/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/cache/HandleConnectExceptionTest.java +++ b/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/cache/HandleConnectExceptionTest.java @@ -135,6 +135,11 @@ public class HandleConnectExceptionTest { } @Override + protected String getHostInMemoryAggregationProtocol() { + return "http"; + } + + @Override public boolean emitMetrics(TimelineMetrics metrics) { super.init(); return super.emitMetrics(metrics); http://git-wip-us.apache.org/repos/asf/ambari/blob/35e2851b/ambari-metrics/ambari-metrics-flume-sink/src/main/java/org/apache/hadoop/metrics2/sink/flume/FlumeTimelineMetricsSink.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-flume-sink/src/main/java/org/apache/hadoop/metrics2/sink/flume/FlumeTimelineMetricsSink.java b/ambari-metrics/ambari-metrics-flume-sink/src/main/java/org/apache/hadoop/metrics2/sink/flume/FlumeTimelineMetricsSink.java index 6277907..720c371 100644 --- a/ambari-metrics/ambari-metrics-flume-sink/src/main/java/org/apache/hadoop/metrics2/sink/flume/FlumeTimelineMetricsSink.java +++ b/ambari-metrics/ambari-metrics-flume-sink/src/main/java/org/apache/hadoop/metrics2/sink/flume/FlumeTimelineMetricsSink.java @@ -65,6 +65,7 @@ public class FlumeTimelineMetricsSink extends AbstractTimelineMetricsSink implem private String instanceId; private boolean hostInMemoryAggregationEnabled; private int hostInMemoryAggregationPort; + private String hostInMemoryAggregationProtocol; @Override @@ -114,12 +115,13 @@ public class FlumeTimelineMetricsSink extends AbstractTimelineMetricsSink implem setInstanceId = Boolean.valueOf(configuration.getProperty(SET_INSTANCE_ID_PROPERTY, "false")); instanceId = configuration.getProperty(INSTANCE_ID_PROPERTY, ""); - hostInMemoryAggregationEnabled = Boolean.getBoolean(configuration.getProperty(HOST_IN_MEMORY_AGGREGATION_ENABLED_PROPERTY)); - hostInMemoryAggregationPort = Integer.valueOf(configuration.getProperty(HOST_IN_MEMORY_AGGREGATION_PORT_PROPERTY)); + hostInMemoryAggregationEnabled = Boolean.getBoolean(configuration.getProperty(HOST_IN_MEMORY_AGGREGATION_ENABLED_PROPERTY, "false")); + hostInMemoryAggregationPort = Integer.valueOf(configuration.getProperty(HOST_IN_MEMORY_AGGREGATION_PORT_PROPERTY, "61888")); + hostInMemoryAggregationProtocol = configuration.getProperty(HOST_IN_MEMORY_AGGREGATION_PROTOCOL_PROPERTY, "http"); // Initialize the collector write strategy super.init(); - if (protocol.contains("https")) { + if (protocol.contains("https") || hostInMemoryAggregationProtocol.contains("https")) { String trustStorePath = configuration.getProperty(SSL_KEYSTORE_PATH_PROPERTY).trim(); String trustStoreType = configuration.getProperty(SSL_KEYSTORE_TYPE_PROPERTY).trim(); String trustStorePwd = configuration.getProperty(SSL_KEYSTORE_PASSWORD_PROPERTY).trim(); @@ -178,6 +180,11 @@ public class FlumeTimelineMetricsSink extends AbstractTimelineMetricsSink implem return hostInMemoryAggregationPort; } + @Override + protected String getHostInMemoryAggregationProtocol() { + return hostInMemoryAggregationProtocol; + } + public void setPollFrequency(long pollFrequency) { this.pollFrequency = pollFrequency; } http://git-wip-us.apache.org/repos/asf/ambari/blob/35e2851b/ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSink.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSink.java b/ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSink.java index a290ced..f37c2be 100644 --- a/ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSink.java +++ b/ambari-metrics/ambari-metrics-hadoop-sink/src/main/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSink.java @@ -34,7 +34,6 @@ import java.io.IOException; import java.net.UnknownHostException; import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -77,6 +76,7 @@ public class HadoopTimelineMetricsSink extends AbstractTimelineMetricsSink imple }); private int hostInMemoryAggregationPort; private boolean hostInMemoryAggregationEnabled; + private String hostInMemoryAggregationProtocol; @Override public void init(SubsetConfiguration conf) { @@ -109,12 +109,13 @@ public class HadoopTimelineMetricsSink extends AbstractTimelineMetricsSink imple protocol = conf.getString(COLLECTOR_PROTOCOL, "http"); collectorHosts = parseHostsStringArrayIntoCollection(conf.getStringArray(COLLECTOR_HOSTS_PROPERTY)); port = conf.getString(COLLECTOR_PORT, "6188"); - hostInMemoryAggregationEnabled = conf.getBoolean(HOST_IN_MEMORY_AGGREGATION_ENABLED_PROPERTY); - hostInMemoryAggregationPort = conf.getInt(HOST_IN_MEMORY_AGGREGATION_PORT_PROPERTY); + hostInMemoryAggregationEnabled = conf.getBoolean(HOST_IN_MEMORY_AGGREGATION_ENABLED_PROPERTY, false); + hostInMemoryAggregationPort = conf.getInt(HOST_IN_MEMORY_AGGREGATION_PORT_PROPERTY, 61888); + hostInMemoryAggregationProtocol = conf.getString(HOST_IN_MEMORY_AGGREGATION_PROTOCOL_PROPERTY, "http"); if (collectorHosts.isEmpty()) { LOG.error("No Metric collector configured."); } else { - if (protocol.contains("https")) { + if (protocol.contains("https") || hostInMemoryAggregationProtocol.contains("https")) { String trustStorePath = conf.getString(SSL_KEYSTORE_PATH_PROPERTY).trim(); String trustStoreType = conf.getString(SSL_KEYSTORE_TYPE_PROPERTY).trim(); String trustStorePwd = conf.getString(SSL_KEYSTORE_PASSWORD_PROPERTY).trim(); @@ -262,6 +263,11 @@ public class HadoopTimelineMetricsSink extends AbstractTimelineMetricsSink imple } @Override + protected String getHostInMemoryAggregationProtocol() { + return hostInMemoryAggregationProtocol; + } + + @Override public void putMetrics(MetricsRecord record) { try { String recordName = record.name(); http://git-wip-us.apache.org/repos/asf/ambari/blob/35e2851b/ambari-metrics/ambari-metrics-hadoop-sink/src/test/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSinkTest.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-hadoop-sink/src/test/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSinkTest.java b/ambari-metrics/ambari-metrics-hadoop-sink/src/test/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSinkTest.java index 30c5c23..8fde394 100644 --- a/ambari-metrics/ambari-metrics-hadoop-sink/src/test/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSinkTest.java +++ b/ambari-metrics/ambari-metrics-hadoop-sink/src/test/java/org/apache/hadoop/metrics2/sink/timeline/HadoopTimelineMetricsSinkTest.java @@ -59,6 +59,7 @@ import static org.apache.hadoop.metrics2.sink.timeline.AbstractTimelineMetricsSi import static org.apache.hadoop.metrics2.sink.timeline.AbstractTimelineMetricsSink.MAX_METRIC_ROW_CACHE_SIZE; import static org.apache.hadoop.metrics2.sink.timeline.AbstractTimelineMetricsSink.METRICS_SEND_INTERVAL; import static org.apache.hadoop.metrics2.sink.timeline.AbstractTimelineMetricsSink.SET_INSTANCE_ID_PROPERTY; +import static org.apache.hadoop.metrics2.sink.timeline.AbstractTimelineMetricsSink.HOST_IN_MEMORY_AGGREGATION_PROTOCOL_PROPERTY; import static org.easymock.EasyMock.anyInt; import static org.easymock.EasyMock.anyObject; import static org.easymock.EasyMock.anyString; @@ -115,6 +116,7 @@ public class HadoopTimelineMetricsSinkTest { expect(conf.getInt(eq(METRICS_SEND_INTERVAL), anyInt())).andReturn(1000).anyTimes(); expect(conf.getBoolean(eq(SET_INSTANCE_ID_PROPERTY), eq(false))).andReturn(true).anyTimes(); expect(conf.getString(eq(INSTANCE_ID_PROPERTY), anyString())).andReturn("instanceId").anyTimes(); + expect(conf.getString(eq(HOST_IN_MEMORY_AGGREGATION_PROTOCOL_PROPERTY), anyString())).andReturn("http").anyTimes(); conf.setListDelimiter(eq(',')); expectLastCall().anyTimes(); @@ -187,6 +189,7 @@ public class HadoopTimelineMetricsSinkTest { expect(conf.getString(eq("serviceName-prefix"), eq(""))).andReturn("").anyTimes(); expect(conf.getString(eq(COLLECTOR_PROTOCOL), eq("http"))).andReturn("http").anyTimes(); expect(conf.getString(eq(COLLECTOR_PORT), eq("6188"))).andReturn("6188").anyTimes(); + expect(conf.getString(eq(HOST_IN_MEMORY_AGGREGATION_PROTOCOL_PROPERTY), anyString())).andReturn("http").anyTimes(); expect(conf.getInt(eq(MAX_METRIC_ROW_CACHE_SIZE), anyInt())).andReturn(10).anyTimes(); // Return eviction time smaller than time diff for first 3 entries @@ -325,6 +328,7 @@ public class HadoopTimelineMetricsSinkTest { expect(conf.getInt(eq(MAX_METRIC_ROW_CACHE_SIZE), anyInt())).andReturn(10).anyTimes(); expect(conf.getInt(eq(METRICS_SEND_INTERVAL), anyInt())).andReturn(10).anyTimes(); + expect(conf.getString(eq(HOST_IN_MEMORY_AGGREGATION_PROTOCOL_PROPERTY), anyString())).andReturn("http").anyTimes(); conf.setListDelimiter(eq(',')); expectLastCall().anyTimes(); http://git-wip-us.apache.org/repos/asf/ambari/blob/35e2851b/ambari-metrics/ambari-metrics-host-aggregator/pom.xml ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-host-aggregator/pom.xml b/ambari-metrics/ambari-metrics-host-aggregator/pom.xml index 24432dd..d126be5 100644 --- a/ambari-metrics/ambari-metrics-host-aggregator/pom.xml +++ b/ambari-metrics/ambari-metrics-host-aggregator/pom.xml @@ -101,6 +101,16 @@ <version>4.2</version> <scope>test</scope> </dependency> + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-server</artifactId> + <version>9.2.11.v20150529</version> + </dependency> + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-webapp</artifactId> + <version>9.2.11.v20150529</version> + </dependency> </dependencies> <build> http://git-wip-us.apache.org/repos/asf/ambari/blob/35e2851b/ambari-metrics/ambari-metrics-host-aggregator/src/main/java/org/apache/hadoop/metrics2/host/aggregator/AggregatorApplication.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-host-aggregator/src/main/java/org/apache/hadoop/metrics2/host/aggregator/AggregatorApplication.java b/ambari-metrics/ambari-metrics-host-aggregator/src/main/java/org/apache/hadoop/metrics2/host/aggregator/AggregatorApplication.java index 1e5cc82..f8ed95f 100644 --- a/ambari-metrics/ambari-metrics-host-aggregator/src/main/java/org/apache/hadoop/metrics2/host/aggregator/AggregatorApplication.java +++ b/ambari-metrics/ambari-metrics-host-aggregator/src/main/java/org/apache/hadoop/metrics2/host/aggregator/AggregatorApplication.java @@ -22,20 +22,23 @@ import com.sun.jersey.api.core.PackagesResourceConfig; import com.sun.jersey.api.core.ResourceConfig; import com.sun.net.httpserver.HttpServer; +import javax.net.ssl.SSLContext; import javax.ws.rs.core.UriBuilder; -import java.io.IOException; import java.net.InetAddress; import java.net.URI; import java.net.URL; import java.net.UnknownHostException; import java.util.HashMap; +import com.sun.net.httpserver.HttpsConfigurator; +import com.sun.net.httpserver.HttpsServer; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.metrics2.sink.timeline.AbstractMetricPublisher; import org.apache.hadoop.metrics2.sink.timeline.AggregatedMetricsPublisher; import org.apache.hadoop.metrics2.sink.timeline.RawMetricsPublisher; +import org.eclipse.jetty.util.ssl.SslContextFactory; /** * WEB application with 2 publisher threads that processes received metrics and submits results to the collector @@ -45,10 +48,12 @@ public class AggregatorApplication private static final int STOP_SECONDS_DELAY = 0; private static final int JOIN_SECONDS_TIMEOUT = 5; private static final String METRICS_SITE_CONFIGURATION_FILE = "ams-site.xml"; + private static final String METRICS_SSL_SERVER_CONFIGURATION_FILE = "ssl-server.xml"; private Log LOG; private final int webApplicationPort; private final int rawPublishingInterval; private final int aggregationInterval; + private final String webServerProtocol; private Configuration configuration; private Thread aggregatePublisherThread; private Thread rawPublisherThread; @@ -65,10 +70,11 @@ public class AggregatorApplication this.aggregationInterval = configuration.getInt("timeline.metrics.host.aggregator.minute.interval", 300); this.rawPublishingInterval = configuration.getInt("timeline.metrics.sink.report.interval", 60); this.webApplicationPort = configuration.getInt("timeline.metrics.host.inmemory.aggregation.port", 61888); + this.webServerProtocol = configuration.get("timeline.metrics.host.inmemory.aggregation.http.policy", "HTTP_ONLY").equalsIgnoreCase("HTTP_ONLY") ? "http" : "https"; this.timelineMetricsHolder = TimelineMetricsHolder.getInstance(rawPublishingInterval, aggregationInterval); try { this.httpServer = createHttpServer(); - } catch (IOException e) { + } catch (Exception e) { LOG.error("Exception while starting HTTP server. Exiting", e); System.exit(1); } @@ -88,13 +94,20 @@ public class AggregatorApplication URL amsResUrl = classLoader.getResource(METRICS_SITE_CONFIGURATION_FILE); LOG.info("Found metric service configuration: " + amsResUrl); + URL sslConfUrl = classLoader.getResource(METRICS_SSL_SERVER_CONFIGURATION_FILE); + LOG.info("Found metric service configuration: " + sslConfUrl); if (amsResUrl == null) { - throw new IllegalStateException("Unable to initialize the metrics " + - "subsystem. No ams-site present in the classpath."); + throw new IllegalStateException(String.format("Unable to initialize the metrics " + + "subsystem. No %s present in the classpath.", METRICS_SITE_CONFIGURATION_FILE)); + } + if (sslConfUrl == null) { + throw new IllegalStateException(String.format("Unable to initialize the metrics " + + "subsystem. No %s present in the classpath.", METRICS_SSL_SERVER_CONFIGURATION_FILE)); } try { configuration.addResource(amsResUrl.toURI().toURL()); + configuration.addResource(sslConfUrl.toURI().toURL()); } catch (Exception e) { LOG.error("Couldn't init configuration. ", e); System.exit(1); @@ -112,17 +125,41 @@ public class AggregatorApplication } protected URI getURI() { - URI uri = UriBuilder.fromUri("http://"; + getHostName() + "/").port(this.webApplicationPort).build(); + URI uri = UriBuilder.fromUri("/").scheme(this.webServerProtocol).host(getHostName()).port(this.webApplicationPort).build(); LOG.info(String.format("Web server at %s", uri)); return uri; } - protected HttpServer createHttpServer() throws IOException { + protected HttpServer createHttpServer() throws Exception { ResourceConfig resourceConfig = new PackagesResourceConfig("org.apache.hadoop.metrics2.host.aggregator"); HashMap<String, Object> params = new HashMap(); params.put("com.sun.jersey.api.json.POJOMappingFeature", "true"); resourceConfig.setPropertiesAndFeatures(params); - return HttpServerFactory.create(getURI(), resourceConfig); + HttpServer server = HttpServerFactory.create(getURI(), resourceConfig); + + if (webServerProtocol.equalsIgnoreCase("https")) { + HttpsServer httpsServer = (HttpsServer) server; + SslContextFactory sslContextFactory = new SslContextFactory(); + String keyStorePath = configuration.get("ssl.server.keystore.location"); + String keyStorePassword = configuration.get("ssl.server.keystore.password"); + String keyManagerPassword = configuration.get("ssl.server.keystore.keypassword"); + String trustStorePath = configuration.get("ssl.server.truststore.location"); + String trustStorePassword = configuration.get("ssl.server.truststore.password"); + + sslContextFactory.setKeyStorePath(keyStorePath); + sslContextFactory.setKeyStorePassword(keyStorePassword); + sslContextFactory.setKeyManagerPassword(keyManagerPassword); + sslContextFactory.setTrustStorePath(trustStorePath); + sslContextFactory.setTrustStorePassword(trustStorePassword); + + sslContextFactory.start(); + SSLContext sslContext = sslContextFactory.getSslContext(); + sslContextFactory.stop(); + HttpsConfigurator httpsConfigurator = new HttpsConfigurator(sslContext); + httpsServer.setHttpsConfigurator(httpsConfigurator); + server = httpsServer; + } + return server; } private void startWebServer() { http://git-wip-us.apache.org/repos/asf/ambari/blob/35e2851b/ambari-metrics/ambari-metrics-host-aggregator/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractMetricPublisher.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-host-aggregator/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractMetricPublisher.java b/ambari-metrics/ambari-metrics-host-aggregator/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractMetricPublisher.java index 5af115f..7ce0815 100644 --- a/ambari-metrics/ambari-metrics-host-aggregator/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractMetricPublisher.java +++ b/ambari-metrics/ambari-metrics-host-aggregator/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AbstractMetricPublisher.java @@ -30,9 +30,9 @@ import java.util.Map; */ public abstract class AbstractMetricPublisher extends AbstractTimelineMetricsSink implements Runnable { - private static final String AMS_SITE_SSL_KEYSTORE_PATH_PROPERTY = "ssl.server.truststore.location"; - private static final String AMS_SITE_SSL_KEYSTORE_TYPE_PROPERTY = "ssl.server.truststore.password"; - private static final String AMS_SITE_SSL_KEYSTORE_PASSWORD_PROPERTY = "ssl.server.truststore.type"; + private static final String AMS_SITE_SSL_TRUSTSTORE_PATH_PROPERTY = "ssl.server.truststore.location"; + private static final String AMS_SITE_SSL_TRUSTSTORE_TYPE_PROPERTY = "ssl.server.truststore.type"; + private static final String AMS_SITE_SSL_TRUSTSTORE_PASSWORD_PROPERTY = "ssl.server.truststore.password"; private static final String AMS_SITE_HTTP_POLICY_PROPERTY = "timeline.metrics.service.http.policy"; private static final String AMS_SITE_COLLECTOR_WEBAPP_ADDRESS_PROPERTY = "timeline.metrics.service.webapp.address"; private static final String PUBLISHER_COLLECTOR_HOSTS_PROPERTY = "timeline.metrics.collector.hosts"; @@ -68,9 +68,9 @@ public abstract class AbstractMetricPublisher extends AbstractTimelineMetricsSin LOG.error("No Metric collector configured."); } else { if (collectorProtocol.contains("https")) { - String trustStorePath = configuration.get(AMS_SITE_SSL_KEYSTORE_PATH_PROPERTY).trim(); - String trustStoreType = configuration.get(AMS_SITE_SSL_KEYSTORE_TYPE_PROPERTY).trim(); - String trustStorePwd = configuration.get(AMS_SITE_SSL_KEYSTORE_PASSWORD_PROPERTY).trim(); + String trustStorePath = configuration.get(AMS_SITE_SSL_TRUSTSTORE_PATH_PROPERTY).trim(); + String trustStoreType = configuration.get(AMS_SITE_SSL_TRUSTSTORE_TYPE_PROPERTY).trim(); + String trustStorePwd = configuration.get(AMS_SITE_SSL_TRUSTSTORE_PASSWORD_PROPERTY).trim(); loadTruststore(trustStorePath, trustStoreType, trustStorePwd); } } http://git-wip-us.apache.org/repos/asf/ambari/blob/35e2851b/ambari-metrics/ambari-metrics-host-aggregator/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AggregatedMetricsPublisher.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-host-aggregator/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AggregatedMetricsPublisher.java b/ambari-metrics/ambari-metrics-host-aggregator/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AggregatedMetricsPublisher.java index c8dffab..fa0c8fb 100644 --- a/ambari-metrics/ambari-metrics-host-aggregator/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AggregatedMetricsPublisher.java +++ b/ambari-metrics/ambari-metrics-host-aggregator/src/main/java/org/apache/hadoop/metrics2/sink/timeline/AggregatedMetricsPublisher.java @@ -100,4 +100,9 @@ public class AggregatedMetricsPublisher extends AbstractMetricPublisher { protected String getPostUrl() { return BASE_POST_URL + AGGREGATED_POST_PREFIX; } + + @Override + protected String getHostInMemoryAggregationProtocol() { + return "http"; + } } http://git-wip-us.apache.org/repos/asf/ambari/blob/35e2851b/ambari-metrics/ambari-metrics-host-aggregator/src/main/java/org/apache/hadoop/metrics2/sink/timeline/RawMetricsPublisher.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-host-aggregator/src/main/java/org/apache/hadoop/metrics2/sink/timeline/RawMetricsPublisher.java b/ambari-metrics/ambari-metrics-host-aggregator/src/main/java/org/apache/hadoop/metrics2/sink/timeline/RawMetricsPublisher.java index 89addb7..2469449 100644 --- a/ambari-metrics/ambari-metrics-host-aggregator/src/main/java/org/apache/hadoop/metrics2/sink/timeline/RawMetricsPublisher.java +++ b/ambari-metrics/ambari-metrics-host-aggregator/src/main/java/org/apache/hadoop/metrics2/sink/timeline/RawMetricsPublisher.java @@ -62,4 +62,9 @@ public class RawMetricsPublisher extends AbstractMetricPublisher { protected String getPostUrl() { return BASE_POST_URL; } + + @Override + protected String getHostInMemoryAggregationProtocol() { + return "http"; + } } http://git-wip-us.apache.org/repos/asf/ambari/blob/35e2851b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/aggregator.py ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/aggregator.py b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/aggregator.py index ba05e9b..59cdd27 100644 --- a/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/aggregator.py +++ b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/aggregator.py @@ -59,7 +59,7 @@ class Aggregator(threading.Thread): def stop(self): self.stopped = True if self._aggregator_process : - logger.info('Stopping Aggregator thread.') + logger.info('Shutting down Aggregator thread.') self._aggregator_process.terminate() self._aggregator_process = None @@ -71,7 +71,7 @@ class AggregatorWatchdog(threading.Thread): threading.Thread.__init__(self) self._config = config self._stop_handler = stop_handler - self.URL = 'http://localhost:' + self._config.get_inmemory_aggregation_port() + self.AMS_AGGREGATOR_METRICS_CHECK_URL + self.URL = self._config.get_inmemory_aggregation_protocol() + '://localhost:' + self._config.get_inmemory_aggregation_port() + self.AMS_AGGREGATOR_METRICS_CHECK_URL self._is_ok = threading.Event() self.set_is_ok(True) self.stopped = False @@ -106,7 +106,7 @@ class AggregatorWatchdog(threading.Thread): def stop(self): - logger.info('Stopping watcher thread.') + logger.info('Shutting down watcher thread.') self.stopped = True http://git-wip-us.apache.org/repos/asf/ambari/blob/35e2851b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/config_reader.py ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/config_reader.py b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/config_reader.py index d1429ed..fc776ae 100644 --- a/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/config_reader.py +++ b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/config_reader.py @@ -250,17 +250,20 @@ class Configuration: def get_max_queue_size(self): return int(self.get("collector", "max_queue_size", 5000)) - def is_server_https_enabled(self): + def is_collector_https_enabled(self): return "true" == str(self.get("collector", "https_enabled")).lower() def get_java_home(self): return self.get("aggregation", "java_home") def is_inmemory_aggregation_enabled(self): - return "true" == str(self.get("aggregation", "host_in_memory_aggregation")).lower() + return "true" == str(self.get("aggregation", "host_in_memory_aggregation", "false")).lower() def get_inmemory_aggregation_port(self): - return self.get("aggregation", "host_in_memory_aggregation_port") + return self.get("aggregation", "host_in_memory_aggregation_port", "61888") + + def get_inmemory_aggregation_protocol(self): + return self.get("aggregation", "host_in_memory_aggregation_protocol", "http") def get_aggregator_jvm_agrs(self): hosts = self.get("aggregation", "jvm_arguments", "-Xmx256m -Xms128m -XX:PermSize=68m") http://git-wip-us.apache.org/repos/asf/ambari/blob/35e2851b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/emitter.py ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/emitter.py b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/emitter.py index 77b8c23..371907d 100644 --- a/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/emitter.py +++ b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/emitter.py @@ -45,17 +45,19 @@ class Emitter(threading.Thread): self.application_metric_map = application_metric_map self.collector_port = config.get_server_port() self.all_metrics_collector_hosts = config.get_metrics_collector_hosts_as_list() - self.is_server_https_enabled = config.is_server_https_enabled() + self.is_collector_https_enabled = config.is_collector_https_enabled() + self.collector_protocol = "https" if self.is_collector_https_enabled else "http" self.set_instanceid = config.is_set_instanceid() self.instanceid = config.get_instanceid() self.is_inmemory_aggregation_enabled = config.is_inmemory_aggregation_enabled() if self.is_inmemory_aggregation_enabled: - self.collector_port = config.get_inmemory_aggregation_port() - self.all_metrics_collector_hosts = ['localhost'] - self.is_server_https_enabled = False + self.inmemory_aggregation_port = config.get_inmemory_aggregation_port() + self.inmemory_aggregation_protocol = config.get_inmemory_aggregation_protocol() + if self.inmemory_aggregation_protocol == "https": + self.ca_certs = config.get_ca_certs() - if self.is_server_https_enabled: + if self.is_collector_https_enabled: self.ca_certs = config.get_ca_certs() # TimedRoundRobinSet @@ -91,22 +93,26 @@ class Emitter(threading.Thread): def push_metrics(self, data): success = False - while self.active_collector_hosts.get_actual_size() > 0: + if self.is_inmemory_aggregation_enabled: + success = self.try_with_collector(self.inmemory_aggregation_protocol, "localhost", self.inmemory_aggregation_port, data) + if not success: + logger.warning("Failed to submit metrics to local aggregator. Trying to post them to collector...") + while not success and self.active_collector_hosts.get_actual_size() > 0: collector_host = self.get_collector_host_shard() - success = self.try_with_collector_host(collector_host, data) - if success: - break + success = self.try_with_collector(self.collector_protocol, collector_host, self.collector_port, data) pass if not success: logger.info('No valid collectors found...') for collector_host in self.active_collector_hosts: - success = self.try_with_collector_host(collector_host, data) + success = self.try_with_collector(self.collector_protocol, collector_host, self.ollector_port, data) + if success: + break pass - def try_with_collector_host(self, collector_host, data): + def try_with_collector(self, collector_protocol, collector_host, collector_port, data): headers = {"Content-Type" : "application/json", "Accept" : "*/*"} - connection = self.get_connection(collector_host) + connection = self.get_connection(collector_protocol, collector_host, collector_port) logger.debug("message to send: %s" % data) retry_count = 0 while retry_count < self.MAX_RETRY_COUNT: @@ -126,16 +132,16 @@ class Emitter(threading.Thread): logger.warn("Metric collector host {0} was blacklisted.".format(collector_host)) return False - def get_connection(self, collector_host): + def get_connection(self, protocol, host, port): timeout = int(self.send_interval - 10) - if self.is_server_https_enabled: - connection = CachedHTTPSConnection(collector_host, - self.collector_port, + if protocol == "https": + connection = CachedHTTPSConnection(host, + port, timeout=timeout, ca_certs=self.ca_certs) else: - connection = CachedHTTPConnection(collector_host, - self.collector_port, + connection = CachedHTTPConnection(host, + port, timeout=timeout) return connection http://git-wip-us.apache.org/repos/asf/ambari/blob/35e2851b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/host_info.py ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/host_info.py b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/host_info.py index 035c833..6198c53 100644 --- a/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/host_info.py +++ b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/host_info.py @@ -265,7 +265,6 @@ class HostInfo(): skip_disk_patterns = self.__config.get_disk_metrics_skip_pattern() logger.debug('skip_disk_patterns: %s' % skip_disk_patterns) - print skip_disk_patterns if not skip_disk_patterns or skip_disk_patterns == 'None': io_counters = psutil.disk_io_counters() print io_counters http://git-wip-us.apache.org/repos/asf/ambari/blob/35e2851b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/stop_handler.py ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/stop_handler.py b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/stop_handler.py index 7a9fbec..330e018 100644 --- a/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/stop_handler.py +++ b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/stop_handler.py @@ -78,7 +78,7 @@ class StopHandlerWindows(StopHandler): raise FatalException(-1, "Error waiting for stop event: " + str(result)) if (win32event.WAIT_TIMEOUT == result): return -1 - logger.info("Stop event received") + logger.debug("Stop event received") return result # 0 -> stop @@ -119,7 +119,7 @@ class StopHandlerLinux(StopHandler): # Stop process when stop event received self.stop_event.wait(timeout) if self.stop_event.isSet(): - logger.info("Stop event received") + logger.debug("Stop event received") return 0 # Timeout return -1 http://git-wip-us.apache.org/repos/asf/ambari/blob/35e2851b/ambari-metrics/ambari-metrics-kafka-sink/src/main/java/org/apache/hadoop/metrics2/sink/kafka/KafkaTimelineMetricsReporter.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-kafka-sink/src/main/java/org/apache/hadoop/metrics2/sink/kafka/KafkaTimelineMetricsReporter.java b/ambari-metrics/ambari-metrics-kafka-sink/src/main/java/org/apache/hadoop/metrics2/sink/kafka/KafkaTimelineMetricsReporter.java index 6f5e9e0..f03627d 100644 --- a/ambari-metrics/ambari-metrics-kafka-sink/src/main/java/org/apache/hadoop/metrics2/sink/kafka/KafkaTimelineMetricsReporter.java +++ b/ambari-metrics/ambari-metrics-kafka-sink/src/main/java/org/apache/hadoop/metrics2/sink/kafka/KafkaTimelineMetricsReporter.java @@ -74,6 +74,7 @@ public class KafkaTimelineMetricsReporter extends AbstractTimelineMetricsSink private static final String TIMELINE_METRICS_KAFKA_SET_INSTANCE_ID_PROPERTY = TIMELINE_METRICS_KAFKA_PREFIX + SET_INSTANCE_ID_PROPERTY; private static final String TIMELINE_METRICS_KAFKA_HOST_IN_MEMORY_AGGREGATION_ENABLED_PROPERTY = TIMELINE_METRICS_KAFKA_PREFIX + HOST_IN_MEMORY_AGGREGATION_ENABLED_PROPERTY; private static final String TIMELINE_METRICS_KAFKA_HOST_IN_MEMORY_AGGREGATION_PORT_PROPERTY = TIMELINE_METRICS_KAFKA_PREFIX + HOST_IN_MEMORY_AGGREGATION_PORT_PROPERTY; + private static final String TIMELINE_METRICS_KAFKA_HOST_IN_MEMORY_AGGREGATION_PROTOCOL_PROPERTY = TIMELINE_METRICS_KAFKA_PREFIX + HOST_IN_MEMORY_AGGREGATION_PROTOCOL_PROPERTY; private static final String TIMELINE_DEFAULT_HOST = "localhost"; private static final String TIMELINE_DEFAULT_PORT = "6188"; private static final String TIMELINE_DEFAULT_PROTOCOL = "http"; @@ -100,6 +101,7 @@ public class KafkaTimelineMetricsReporter extends AbstractTimelineMetricsSink private Set<String> excludedMetrics = new HashSet<>(); private boolean hostInMemoryAggregationEnabled; private int hostInMemoryAggregationPort; + private String hostInMemoryAggregationProtocol; @Override protected String getCollectorUri(String host) { @@ -147,6 +149,11 @@ public class KafkaTimelineMetricsReporter extends AbstractTimelineMetricsSink return hostInMemoryAggregationPort; } + @Override + protected String getHostInMemoryAggregationProtocol() { + return hostInMemoryAggregationProtocol; + } + public void setMetricsCache(TimelineMetricsCache metricsCache) { this.metricsCache = metricsCache; } @@ -186,9 +193,10 @@ public class KafkaTimelineMetricsReporter extends AbstractTimelineMetricsSink hostInMemoryAggregationEnabled = props.getBoolean(TIMELINE_METRICS_KAFKA_HOST_IN_MEMORY_AGGREGATION_ENABLED_PROPERTY, false); hostInMemoryAggregationPort = props.getInt(TIMELINE_METRICS_KAFKA_HOST_IN_MEMORY_AGGREGATION_PORT_PROPERTY, 61888); + hostInMemoryAggregationProtocol = props.getString(TIMELINE_METRICS_KAFKA_HOST_IN_MEMORY_AGGREGATION_PROTOCOL_PROPERTY, "http"); setMetricsCache(new TimelineMetricsCache(maxRowCacheSize, metricsSendInterval)); - if (metricCollectorProtocol.contains("https")) { + if (metricCollectorProtocol.contains("https") || hostInMemoryAggregationProtocol.contains("https")) { String trustStorePath = props.getString(TIMELINE_METRICS_SSL_KEYSTORE_PATH_PROPERTY).trim(); String trustStoreType = props.getString(TIMELINE_METRICS_SSL_KEYSTORE_TYPE_PROPERTY).trim(); String trustStorePwd = props.getString(TIMELINE_METRICS_SSL_KEYSTORE_PASSWORD_PROPERTY).trim(); http://git-wip-us.apache.org/repos/asf/ambari/blob/35e2851b/ambari-metrics/ambari-metrics-storm-sink-legacy/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsReporter.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-storm-sink-legacy/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsReporter.java b/ambari-metrics/ambari-metrics-storm-sink-legacy/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsReporter.java index d408e1a..842fad8 100644 --- a/ambari-metrics/ambari-metrics-storm-sink-legacy/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsReporter.java +++ b/ambari-metrics/ambari-metrics-storm-sink-legacy/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsReporter.java @@ -57,6 +57,7 @@ public class StormTimelineMetricsReporter extends AbstractTimelineMetricsSink private int timeoutSeconds; private boolean hostInMemoryAggregationEnabled; private int hostInMemoryAggregationPort; + private String hostInMemoryAggregationProtocol; public StormTimelineMetricsReporter() { @@ -108,6 +109,11 @@ public class StormTimelineMetricsReporter extends AbstractTimelineMetricsSink } @Override + protected String getHostInMemoryAggregationProtocol() { + return hostInMemoryAggregationProtocol; + } + + @Override public void prepare(Map conf) { LOG.info("Preparing Storm Metrics Reporter"); try { @@ -144,11 +150,15 @@ public class StormTimelineMetricsReporter extends AbstractTimelineMetricsSink setInstanceId = Boolean.getBoolean(cf.get(SET_INSTANCE_ID_PROPERTY).toString()); instanceId = cf.get(INSTANCE_ID_PROPERTY).toString(); } - hostInMemoryAggregationEnabled = Boolean.valueOf(cf.get(HOST_IN_MEMORY_AGGREGATION_ENABLED_PROPERTY).toString()); - hostInMemoryAggregationPort = Integer.valueOf(cf.get(HOST_IN_MEMORY_AGGREGATION_PORT_PROPERTY).toString()); + hostInMemoryAggregationEnabled = Boolean.valueOf(cf.get(HOST_IN_MEMORY_AGGREGATION_ENABLED_PROPERTY) != null ? + cf.get(HOST_IN_MEMORY_AGGREGATION_ENABLED_PROPERTY).toString() : "false"); + hostInMemoryAggregationPort = Integer.valueOf(cf.get(HOST_IN_MEMORY_AGGREGATION_PORT_PROPERTY) != null ? + cf.get(HOST_IN_MEMORY_AGGREGATION_PORT_PROPERTY).toString() : "61888"); + hostInMemoryAggregationProtocol = cf.get(HOST_IN_MEMORY_AGGREGATION_PROTOCOL_PROPERTY) != null ? + cf.get(HOST_IN_MEMORY_AGGREGATION_PROTOCOL_PROPERTY).toString() : "http"; collectorUri = constructTimelineMetricUri(protocol, findPreferredCollectHost(), port); - if (protocol.contains("https")) { + if (protocol.contains("https") || hostInMemoryAggregationProtocol.contains("https")) { String trustStorePath = cf.get(SSL_KEYSTORE_PATH_PROPERTY).toString().trim(); String trustStoreType = cf.get(SSL_KEYSTORE_TYPE_PROPERTY).toString().trim(); String trustStorePwd = cf.get(SSL_KEYSTORE_PASSWORD_PROPERTY).toString().trim(); http://git-wip-us.apache.org/repos/asf/ambari/blob/35e2851b/ambari-metrics/ambari-metrics-storm-sink-legacy/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsSink.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-storm-sink-legacy/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsSink.java b/ambari-metrics/ambari-metrics-storm-sink-legacy/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsSink.java index ff72f24..e3494fd 100644 --- a/ambari-metrics/ambari-metrics-storm-sink-legacy/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsSink.java +++ b/ambari-metrics/ambari-metrics-storm-sink-legacy/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsSink.java @@ -63,6 +63,7 @@ public class StormTimelineMetricsSink extends AbstractTimelineMetricsSink implem private String instanceId; private boolean hostInMemoryAggregationEnabled; private int hostInMemoryAggregationPort; + private String hostInMemoryAggregationProtocol; @Override protected String getCollectorUri(String host) { @@ -110,6 +111,11 @@ public class StormTimelineMetricsSink extends AbstractTimelineMetricsSink implem } @Override + protected String getHostInMemoryAggregationProtocol() { + return hostInMemoryAggregationProtocol; + } + + @Override public void prepare(Map map, Object o, TopologyContext topologyContext, IErrorReporter iErrorReporter) { LOG.info("Preparing Storm Metrics Sink"); try { @@ -138,12 +144,13 @@ public class StormTimelineMetricsSink extends AbstractTimelineMetricsSink implem instanceId = configuration.getProperty(INSTANCE_ID_PROPERTY, null); setInstanceId = Boolean.valueOf(configuration.getProperty(SET_INSTANCE_ID_PROPERTY, "false")); - hostInMemoryAggregationEnabled = Boolean.valueOf(configuration.getProperty(HOST_IN_MEMORY_AGGREGATION_ENABLED_PROPERTY)); - hostInMemoryAggregationPort = Integer.valueOf(configuration.getProperty(HOST_IN_MEMORY_AGGREGATION_PORT_PROPERTY)); + hostInMemoryAggregationEnabled = Boolean.valueOf(configuration.getProperty(HOST_IN_MEMORY_AGGREGATION_ENABLED_PROPERTY, "false")); + hostInMemoryAggregationPort = Integer.valueOf(configuration.getProperty(HOST_IN_MEMORY_AGGREGATION_PORT_PROPERTY, "61888")); + hostInMemoryAggregationProtocol = configuration.getProperty(HOST_IN_MEMORY_AGGREGATION_PROTOCOL_PROPERTY, "http"); // Initialize the collector write strategy super.init(); - if (protocol.contains("https")) { + if (protocol.contains("https") || hostInMemoryAggregationProtocol.contains("https")) { String trustStorePath = configuration.getProperty(SSL_KEYSTORE_PATH_PROPERTY).trim(); String trustStoreType = configuration.getProperty(SSL_KEYSTORE_TYPE_PROPERTY).trim(); String trustStorePwd = configuration.getProperty(SSL_KEYSTORE_PASSWORD_PROPERTY).trim(); http://git-wip-us.apache.org/repos/asf/ambari/blob/35e2851b/ambari-metrics/ambari-metrics-storm-sink/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsReporter.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-storm-sink/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsReporter.java b/ambari-metrics/ambari-metrics-storm-sink/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsReporter.java index 5b75065..4fcf2fb 100644 --- a/ambari-metrics/ambari-metrics-storm-sink/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsReporter.java +++ b/ambari-metrics/ambari-metrics-storm-sink/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsReporter.java @@ -52,6 +52,7 @@ public class StormTimelineMetricsReporter extends AbstractTimelineMetricsSink private int timeoutSeconds; private boolean hostInMemoryAggregationEnabled; private int hostInMemoryAggregationPort; + private String hostInMemoryAggregationProtocol; public StormTimelineMetricsReporter() { @@ -103,6 +104,11 @@ public class StormTimelineMetricsReporter extends AbstractTimelineMetricsSink } @Override + protected String getHostInMemoryAggregationProtocol() { + return hostInMemoryAggregationProtocol; + } + + @Override public void prepare(Object registrationArgument) { LOG.info("Preparing Storm Metrics Reporter"); try { @@ -132,10 +138,11 @@ public class StormTimelineMetricsReporter extends AbstractTimelineMetricsSink setInstanceId = Boolean.valueOf(configuration.getProperty(SET_INSTANCE_ID_PROPERTY)); instanceId = configuration.getProperty(INSTANCE_ID_PROPERTY); - hostInMemoryAggregationEnabled = Boolean.valueOf(configuration.getProperty(HOST_IN_MEMORY_AGGREGATION_ENABLED_PROPERTY)); - hostInMemoryAggregationPort = Integer.valueOf(configuration.getProperty(HOST_IN_MEMORY_AGGREGATION_PORT_PROPERTY)); + hostInMemoryAggregationEnabled = Boolean.valueOf(configuration.getProperty(HOST_IN_MEMORY_AGGREGATION_ENABLED_PROPERTY, "false")); + hostInMemoryAggregationPort = Integer.valueOf(configuration.getProperty(HOST_IN_MEMORY_AGGREGATION_PORT_PROPERTY, "61888")); + hostInMemoryAggregationProtocol = configuration.getProperty(HOST_IN_MEMORY_AGGREGATION_PROTOCOL_PROPERTY, "http"); - if (protocol.contains("https")) { + if (protocol.contains("https") || hostInMemoryAggregationProtocol.contains("https")) { String trustStorePath = configuration.getProperty(SSL_KEYSTORE_PATH_PROPERTY).trim(); String trustStoreType = configuration.getProperty(SSL_KEYSTORE_TYPE_PROPERTY).trim(); String trustStorePwd = configuration.getProperty(SSL_KEYSTORE_PASSWORD_PROPERTY).trim(); http://git-wip-us.apache.org/repos/asf/ambari/blob/35e2851b/ambari-metrics/ambari-metrics-storm-sink/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsSink.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-storm-sink/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsSink.java b/ambari-metrics/ambari-metrics-storm-sink/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsSink.java index 4d5a229..dc92f80 100644 --- a/ambari-metrics/ambari-metrics-storm-sink/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsSink.java +++ b/ambari-metrics/ambari-metrics-storm-sink/src/main/java/org/apache/hadoop/metrics2/sink/storm/StormTimelineMetricsSink.java @@ -72,6 +72,7 @@ public class StormTimelineMetricsSink extends AbstractTimelineMetricsSink implem private boolean setInstanceId; private boolean hostInMemoryAggregationEnabled; private int hostInMemoryAggregationPort; + private String hostInMemoryAggregationProtocol; @Override protected String getCollectorUri(String host) { @@ -119,6 +120,11 @@ public class StormTimelineMetricsSink extends AbstractTimelineMetricsSink implem } @Override + protected String getHostInMemoryAggregationProtocol() { + return hostInMemoryAggregationProtocol; + } + + @Override public void prepare(Map map, Object o, TopologyContext topologyContext, IErrorReporter iErrorReporter) { LOG.info("Preparing Storm Metrics Sink"); try { @@ -150,13 +156,14 @@ public class StormTimelineMetricsSink extends AbstractTimelineMetricsSink implem instanceId = configuration.getProperty(INSTANCE_ID_PROPERTY, null); setInstanceId = Boolean.valueOf(configuration.getProperty(SET_INSTANCE_ID_PROPERTY, "false")); - hostInMemoryAggregationEnabled = Boolean.valueOf(configuration.getProperty(HOST_IN_MEMORY_AGGREGATION_ENABLED_PROPERTY)); - hostInMemoryAggregationPort = Integer.valueOf(configuration.getProperty(HOST_IN_MEMORY_AGGREGATION_PORT_PROPERTY)); + hostInMemoryAggregationEnabled = Boolean.valueOf(configuration.getProperty(HOST_IN_MEMORY_AGGREGATION_ENABLED_PROPERTY, "false")); + hostInMemoryAggregationPort = Integer.valueOf(configuration.getProperty(HOST_IN_MEMORY_AGGREGATION_PORT_PROPERTY, "61888")); + hostInMemoryAggregationProtocol = configuration.getProperty(HOST_IN_MEMORY_AGGREGATION_PROTOCOL_PROPERTY, "http"); // Initialize the collector write strategy super.init(); - if (protocol.contains("https")) { + if (protocol.contains("https") || hostInMemoryAggregationProtocol.contains("https")) { String trustStorePath = configuration.getProperty(SSL_KEYSTORE_PATH_PROPERTY).trim(); String trustStoreType = configuration.getProperty(SSL_KEYSTORE_TYPE_PROPERTY).trim(); String trustStorePwd = configuration.getProperty(SSL_KEYSTORE_PASSWORD_PROPERTY).trim(); http://git-wip-us.apache.org/repos/asf/ambari/blob/35e2851b/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/AmbariMetricSinkImpl.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/AmbariMetricSinkImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/AmbariMetricSinkImpl.java index 83e422c..1ba70d0 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/AmbariMetricSinkImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/metrics/system/impl/AmbariMetricSinkImpl.java @@ -309,6 +309,11 @@ public class AmbariMetricSinkImpl extends AbstractTimelineMetricsSink implements return 0; } + @Override + protected String getHostInMemoryAggregationProtocol() { + return "http"; + } + private List<TimelineMetric> getFilteredMetricList(List<SingleMetric> metrics) { final List<TimelineMetric> metricList = new ArrayList<>(); for (SingleMetric metric : metrics) { http://git-wip-us.apache.org/repos/asf/ambari/blob/35e2851b/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/scripts/params.py index 5d21514..5997787 100644 --- a/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/scripts/params.py @@ -155,6 +155,12 @@ metrics_report_interval = default("/configurations/ams-site/timeline.metrics.sin metrics_collection_period = default("/configurations/ams-site/timeline.metrics.sink.collection.period", 10) host_in_memory_aggregation = default("/configurations/ams-site/timeline.metrics.host.inmemory.aggregation", True) host_in_memory_aggregation_port = default("/configurations/ams-site/timeline.metrics.host.inmemory.aggregation.port", 61888) +is_aggregation_https_enabled = False +if default("/configurations/ams-site/timeline.metrics.host.inmemory.aggregation.http.policy", "HTTP_ONLY") == "HTTPS_ONLY": + host_in_memory_aggregation_protocol = 'https' + is_aggregation_https_enabled = True +else: + host_in_memory_aggregation_protocol = 'http' # if accumulo is selected accumulo_tserver_hosts should not be empty, but still default just in case if 'slave_hosts' in config['clusterHostInfo']: http://git-wip-us.apache.org/repos/asf/ambari/blob/35e2851b/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/templates/hadoop-metrics2-accumulo.properties.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/templates/hadoop-metrics2-accumulo.properties.j2 b/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/templates/hadoop-metrics2-accumulo.properties.j2 index e59ba11..282f904 100644 --- a/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/templates/hadoop-metrics2-accumulo.properties.j2 +++ b/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/templates/hadoop-metrics2-accumulo.properties.j2 @@ -18,6 +18,9 @@ *.host_in_memory_aggregation = {{host_in_memory_aggregation}} *.host_in_memory_aggregation_port = {{host_in_memory_aggregation_port}} +{% if is_aggregation_https_enabled %} +*.host_in_memory_aggregation_protocol = {{host_in_memory_aggregation_protocol}} +{% endif %} {% if has_metric_collector %} http://git-wip-us.apache.org/repos/asf/ambari/blob/35e2851b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-site.xml b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-site.xml index 1b085f6..d54db37 100644 --- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-site.xml +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-site.xml @@ -798,4 +798,9 @@ <value>61888</value> <on-ambari-upgrade add="true"/> </property> + <property> + <name>timeline.metrics.host.inmemory.aggregation.http.policy</name> + <value>HTTP_ONLY</value> + <on-ambari-upgrade add="true"/> + </property> </configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/35e2851b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams.py b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams.py index f49d47d..07d0223 100644 --- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams.py +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams.py @@ -163,19 +163,30 @@ def ams(name=None): create_parents = True ) - if params.host_in_memory_aggregation and params.log4j_props is not None: - File(os.path.join(params.ams_monitor_conf_dir, "log4j.properties"), - owner=params.ams_user, - content=params.log4j_props - ) + if params.host_in_memory_aggregation: + if params.log4j_props is not None: + File(os.path.join(params.ams_monitor_conf_dir, "log4j.properties"), + owner=params.ams_user, + content=params.log4j_props + ) + pass + + XmlConfig("ams-site.xml", + conf_dir=params.ams_monitor_conf_dir, + configurations=params.config['configurations']['ams-site'], + configuration_attributes=params.config['configuration_attributes']['ams-site'], + owner=params.ams_user, + group=params.user_group + ) - XmlConfig("ams-site.xml", - conf_dir=params.ams_monitor_conf_dir, - configurations=params.config['configurations']['ams-site'], - configuration_attributes=params.config['configuration_attributes']['ams-site'], - owner=params.ams_user, - group=params.user_group - ) + XmlConfig("ssl-server.xml", + conf_dir=params.ams_monitor_conf_dir, + configurations=params.config['configurations']['ams-ssl-server'], + configuration_attributes=params.config['configuration_attributes']['ams-site'], + owner=params.ams_user, + group=params.user_group + ) + pass TemplateConfig( os.path.join(params.ams_monitor_conf_dir, "metric_monitor.ini"), @@ -380,22 +391,32 @@ def ams(name=None, action=None): create_parents = True ) - if params.host_in_memory_aggregation and params.log4j_props is not None: - File(format("{params.ams_monitor_conf_dir}/log4j.properties"), - mode=0644, - group=params.user_group, - owner=params.ams_user, - content=InlineTemplate(params.log4j_props) - ) - - XmlConfig("ams-site.xml", - conf_dir=params.ams_monitor_conf_dir, - configurations=params.config['configurations']['ams-site'], - configuration_attributes=params.config['configuration_attributes']['ams-site'], - owner=params.ams_user, - group=params.user_group - ) + if params.host_in_memory_aggregation: + if params.log4j_props is not None: + File(format("{params.ams_monitor_conf_dir}/log4j.properties"), + mode=0644, + group=params.user_group, + owner=params.ams_user, + content=InlineTemplate(params.log4j_props) + ) + pass + + XmlConfig("ams-site.xml", + conf_dir=params.ams_monitor_conf_dir, + configurations=params.config['configurations']['ams-site'], + configuration_attributes=params.config['configuration_attributes']['ams-site'], + owner=params.ams_user, + group=params.user_group + ) + XmlConfig("ssl-server.xml", + conf_dir=params.ams_monitor_conf_dir, + configurations=params.config['configurations']['ams-ssl-server'], + configuration_attributes=params.config['configuration_attributes']['ams-site'], + owner=params.ams_user, + group=params.user_group + ) + pass Execute(format("{sudo} chown -R {ams_user}:{user_group} {ams_monitor_log_dir}") ) @@ -435,7 +456,7 @@ def ams(name=None, action=None): content=InlineTemplate(params.ams_env_sh_template) ) - if params.metric_collector_https_enabled: + if params.metric_collector_https_enabled or params.is_aggregation_https_enabled: export_ca_certs(params.ams_monitor_conf_dir) pass http://git-wip-us.apache.org/repos/asf/ambari/blob/35e2851b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py index b8c14f4..6abf305 100644 --- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py @@ -224,8 +224,14 @@ metrics_collector_heapsize = check_append_heap_property(str(metrics_collector_he master_heapsize = check_append_heap_property(str(master_heapsize), "m") regionserver_heapsize = check_append_heap_property(str(regionserver_heapsize), "m") -host_in_memory_aggregation = default("/configurations/ams-site/timeline.metrics.host.inmemory.aggregation", True) +host_in_memory_aggregation = default("/configurations/ams-site/timeline.metrics.host.inmemory.aggregation", False) host_in_memory_aggregation_port = default("/configurations/ams-site/timeline.metrics.host.inmemory.aggregation.port", 61888) +is_aggregation_https_enabled = False +if default("/configurations/ams-site/timeline.metrics.host.inmemory.aggregation.http.policy", "HTTP_ONLY") == "HTTPS_ONLY": + host_in_memory_aggregation_protocol = 'https' + is_aggregation_https_enabled = True +else: + host_in_memory_aggregation_protocol = 'http' host_in_memory_aggregation_jvm_arguments = default("/configurations/ams-env/timeline.metrics.host.inmemory.aggregation.jvm.arguments", "-Xmx256m -Xms128m -XX:PermSize=68m") http://git-wip-us.apache.org/repos/asf/ambari/blob/35e2851b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/hadoop-metrics2-hbase.properties.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/hadoop-metrics2-hbase.properties.j2 b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/hadoop-metrics2-hbase.properties.j2 index 978b795..0f05f5e 100644 --- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/hadoop-metrics2-hbase.properties.j2 +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/hadoop-metrics2-hbase.properties.j2 @@ -60,6 +60,9 @@ rpc.protocol={{metric_collector_protocol}} *.sink.timeline.slave.host.name={{hostname}} *.host_in_memory_aggregation = {{host_in_memory_aggregation}} *.host_in_memory_aggregation_port = {{host_in_memory_aggregation_port}} +{% if is_aggregation_https_enabled %} +*.host_in_memory_aggregation_protocol = {{host_in_memory_aggregation_protocol}} +{% endif %} hbase.sink.timeline.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink hbase.sink.timeline.period={{metrics_collection_period}} http://git-wip-us.apache.org/repos/asf/ambari/blob/35e2851b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/metric_monitor.ini.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/metric_monitor.ini.j2 b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/metric_monitor.ini.j2 index b7dee50..ceaf178 100644 --- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/metric_monitor.ini.j2 +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/metric_monitor.ini.j2 @@ -42,6 +42,9 @@ https_enabled = {{metric_collector_https_enabled}} [aggregation] host_in_memory_aggregation = {{host_in_memory_aggregation}} host_in_memory_aggregation_port = {{host_in_memory_aggregation_port}} +{% if is_aggregation_https_enabled %} +host_in_memory_aggregation_protocol = {{host_in_memory_aggregation_protocol}} +{% endif %} java_home = {{java64_home}} jvm_arguments = {{host_in_memory_aggregation_jvm_arguments}} ams_monitor_log_dir = {{ams_monitor_log_dir}} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/35e2851b/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/params.py index 0e0c9aa..011b605 100644 --- a/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/params.py @@ -126,6 +126,12 @@ metrics_collection_period = default("/configurations/ams-site/timeline.metrics.s host_in_memory_aggregation = default("/configurations/ams-site/timeline.metrics.host.inmemory.aggregation", True) host_in_memory_aggregation_port = default("/configurations/ams-site/timeline.metrics.host.inmemory.aggregation.port", 61888) +is_aggregation_https_enabled = False +if default("/configurations/ams-site/timeline.metrics.host.inmemory.aggregation.http.policy", "HTTP_ONLY") == "HTTPS_ONLY": + host_in_memory_aggregation_protocol = 'https' + is_aggregation_https_enabled = True +else: + host_in_memory_aggregation_protocol = 'http' # Cluster Zookeeper quorum zookeeper_quorum = None http://git-wip-us.apache.org/repos/asf/ambari/blob/35e2851b/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/templates/flume-metrics2.properties.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/templates/flume-metrics2.properties.j2 b/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/templates/flume-metrics2.properties.j2 index c476019..c9a320f 100644 --- a/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/templates/flume-metrics2.properties.j2 +++ b/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/templates/flume-metrics2.properties.j2 @@ -25,6 +25,9 @@ maxRowCacheSize=10000 sendInterval={{metrics_report_interval}}000 host_in_memory_aggregation = {{host_in_memory_aggregation}} host_in_memory_aggregation_port = {{host_in_memory_aggregation_port}} +{% if is_aggregation_https_enabled %} +host_in_memory_aggregation_protocol = {{host_in_memory_aggregation_protocol}} +{% endif %} # HTTPS properties truststore.path = {{metric_truststore_path}} http://git-wip-us.apache.org/repos/asf/ambari/blob/35e2851b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py index d45aea6..0b543a8 100644 --- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py @@ -186,6 +186,12 @@ metrics_collection_period = default("/configurations/ams-site/timeline.metrics.s host_in_memory_aggregation = default("/configurations/ams-site/timeline.metrics.host.inmemory.aggregation", True) host_in_memory_aggregation_port = default("/configurations/ams-site/timeline.metrics.host.inmemory.aggregation.port", 61888) +is_aggregation_https_enabled = False +if default("/configurations/ams-site/timeline.metrics.host.inmemory.aggregation.http.policy", "HTTP_ONLY") == "HTTPS_ONLY": + host_in_memory_aggregation_protocol = 'https' + is_aggregation_https_enabled = True +else: + host_in_memory_aggregation_protocol = 'http' # if hbase is selected the hbase_rs_hosts, should not be empty, but still default just in case if 'slave_hosts' in config['clusterHostInfo']: http://git-wip-us.apache.org/repos/asf/ambari/blob/35e2851b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-MASTER.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-MASTER.j2 b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-MASTER.j2 index 7368ffe..66796b4 100644 --- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-MASTER.j2 +++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-MASTER.j2 @@ -78,6 +78,9 @@ hbase.sink.timeline.protocol={{metric_collector_protocol}} hbase.sink.timeline.port={{metric_collector_port}} hbase.sink.timeline.host_in_memory_aggregation = {{host_in_memory_aggregation}} hbase.sink.timeline.host_in_memory_aggregation_port = {{host_in_memory_aggregation_port}} +{% if is_aggregation_https_enabled %} +hbase.sink.timeline.host_in_memory_aggregation_protocol = {{host_in_memory_aggregation_protocol}} +{% endif %} # HTTPS properties hbase.sink.timeline.truststore.path = {{metric_truststore_path}} http://git-wip-us.apache.org/repos/asf/ambari/blob/35e2851b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-RS.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-RS.j2 b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-RS.j2 index f245365..4ed68ba 100644 --- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-RS.j2 +++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-RS.j2 @@ -76,6 +76,9 @@ hbase.sink.timeline.protocol={{metric_collector_protocol}} hbase.sink.timeline.port={{metric_collector_port}} hbase.sink.timeline.host_in_memory_aggregation = {{host_in_memory_aggregation}} hbase.sink.timeline.host_in_memory_aggregation_port = {{host_in_memory_aggregation_port}} +{% if is_aggregation_https_enabled %} +hbase.sink.timeline.host_in_memory_aggregation_protocol = {{host_in_memory_aggregation_protocol}} +{% endif %} # HTTPS properties hbase.sink.timeline.truststore.path = {{metric_truststore_path}} http://git-wip-us.apache.org/repos/asf/ambari/blob/35e2851b/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/configuration/hadoop-metrics2.properties.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/configuration/hadoop-metrics2.properties.xml b/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/configuration/hadoop-metrics2.properties.xml index 84ea231..02be755 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/configuration/hadoop-metrics2.properties.xml +++ b/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/configuration/hadoop-metrics2.properties.xml @@ -88,6 +88,9 @@ resourcemanager.sink.ganglia.tagsForPrefix.yarn=Queue *.sink.timeline.port={{metric_collector_port}} *.sink.timeline.host_in_memory_aggregation = {{host_in_memory_aggregation}} *.sink.timeline.host_in_memory_aggregation_port = {{host_in_memory_aggregation_port}} +{% if is_aggregation_https_enabled %} +*.sink.timeline.host_in_memory_aggregation_protocol = {{host_in_memory_aggregation_protocol}} +{% endif %} # HTTPS properties *.sink.timeline.truststore.path = {{metric_truststore_path}} http://git-wip-us.apache.org/repos/asf/ambari/blob/35e2851b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py index c1128a5..82add3a 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py @@ -568,6 +568,12 @@ metrics_collection_period = default("/configurations/ams-site/timeline.metrics.s host_in_memory_aggregation = default("/configurations/ams-site/timeline.metrics.host.inmemory.aggregation", True) host_in_memory_aggregation_port = default("/configurations/ams-site/timeline.metrics.host.inmemory.aggregation.port", 61888) +is_aggregation_https_enabled = False +if default("/configurations/ams-site/timeline.metrics.host.inmemory.aggregation.http.policy", "HTTP_ONLY") == "HTTPS_ONLY": + host_in_memory_aggregation_protocol = 'https' + is_aggregation_https_enabled = True +else: + host_in_memory_aggregation_protocol = 'http' ######################################################## ############# Atlas related params ##################### ######################################################## http://git-wip-us.apache.org/repos/asf/ambari/blob/35e2851b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-hivemetastore.properties.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-hivemetastore.properties.j2 b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-hivemetastore.properties.j2 index 3093e56..d4573c3 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-hivemetastore.properties.j2 +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-hivemetastore.properties.j2 @@ -53,6 +53,8 @@ hivemetastore.sink.timeline.protocol={{metric_collector_protocol}} hivemetastore.sink.timeline.host_in_memory_aggregation = {{host_in_memory_aggregation}} hivemetastore.sink.timeline.host_in_memory_aggregation_port = {{host_in_memory_aggregation_port}} - + {% if is_aggregation_https_enabled %} + hivemetastore.sink.timeline.host_in_memory_aggregation_protocol = {{host_in_memory_aggregation_protocol}} + {% endif %} {% endif %} http://git-wip-us.apache.org/repos/asf/ambari/blob/35e2851b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-hiveserver2.properties.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-hiveserver2.properties.j2 b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-hiveserver2.properties.j2 index 59a7c1b..c67d002 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-hiveserver2.properties.j2 +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-hiveserver2.properties.j2 @@ -53,5 +53,8 @@ hiveserver2.sink.timeline.protocol={{metric_collector_protocol}} hiveserver2.sink.timeline.host_in_memory_aggregation = {{host_in_memory_aggregation}} hiveserver2.sink.timeline.host_in_memory_aggregation_port = {{host_in_memory_aggregation_port}} + {% if is_aggregation_https_enabled %} + hiveserver2.sink.timeline.host_in_memory_aggregation_protocol = {{host_in_memory_aggregation_protocol}} + {% endif %} {% endif %} http://git-wip-us.apache.org/repos/asf/ambari/blob/35e2851b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-llapdaemon.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-llapdaemon.j2 b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-llapdaemon.j2 index 69f6071..cd23e8a 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-llapdaemon.j2 +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-llapdaemon.j2 @@ -52,5 +52,8 @@ llapdaemon.sink.timeline.protocol={{metric_collector_protocol}} llapdaemon.sink.timeline.host_in_memory_aggregation = {{host_in_memory_aggregation}} llapdaemon.sink.timeline.host_in_memory_aggregation_port = {{host_in_memory_aggregation_port}} + {% if is_aggregation_https_enabled %} + llapdaemon.sink.timeline.host_in_memory_aggregation_protocol = {{host_in_memory_aggregation_protocol}} + {% endif %} {% endif %} http://git-wip-us.apache.org/repos/asf/ambari/blob/35e2851b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-llaptaskscheduler.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-llaptaskscheduler.j2 b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-llaptaskscheduler.j2 index c08a498..674d3cc 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-llaptaskscheduler.j2 +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/hadoop-metrics2-llaptaskscheduler.j2 @@ -52,5 +52,8 @@ llaptaskscheduler.sink.timeline.protocol={{metric_collector_protocol}} llaptaskscheduler.sink.timeline.host_in_memory_aggregation = {{host_in_memory_aggregation}} llaptaskscheduler.sink.timeline.host_in_memory_aggregation_port = {{host_in_memory_aggregation_port}} + {% if is_aggregation_https_enabled %} + llaptaskscheduler.sink.timeline.host_in_memory_aggregation_protocol = {{host_in_memory_aggregation_protocol}} + {% endif %} {% endif %} http://git-wip-us.apache.org/repos/asf/ambari/blob/35e2851b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/params_linux.py index a12d388..8003e63 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/params_linux.py @@ -567,6 +567,12 @@ metrics_collection_period = default("/configurations/ams-site/timeline.metrics.s host_in_memory_aggregation = default("/configurations/ams-site/timeline.metrics.host.inmemory.aggregation", True) host_in_memory_aggregation_port = default("/configurations/ams-site/timeline.metrics.host.inmemory.aggregation.port", 61888) +is_aggregation_https_enabled = False +if default("/configurations/ams-site/timeline.metrics.host.inmemory.aggregation.http.policy", "HTTP_ONLY") == "HTTPS_ONLY": + host_in_memory_aggregation_protocol = 'https' + is_aggregation_https_enabled = True +else: + host_in_memory_aggregation_protocol = 'http' ######################################################## ############# Atlas related params #####################
