Repository: ambari Updated Branches: refs/heads/trunk c7ed0f81d -> dcd198178
AMBARI-16137. Metrics support for Logsearch Solr (Miklos Gergely via oleewere) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/dcd19817 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/dcd19817 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/dcd19817 Branch: refs/heads/trunk Commit: dcd198178c8de5ca7a10471bdfae16968578ce66 Parents: c7ed0f8 Author: Miklos Gergely <[email protected]> Authored: Tue May 10 18:25:12 2016 +0200 Committer: oleewere <[email protected]> Committed: Tue May 10 18:25:12 2016 +0200 ---------------------------------------------------------------------- .../ambari-logsearch-portal/pom.xml | 10 ++ .../org/apache/ambari/logsearch/LogSearch.java | 3 + .../logsearch/solr/metrics/SolrAmsClient.java | 46 +++++ .../logsearch/solr/metrics/SolrJmxAdapter.java | 134 +++++++++++++++ .../solr/metrics/SolrMetricsLoader.java | 168 +++++++++++++++++++ .../logsearch/solr/AmbariSolrCloudClient.java | 21 ++- .../solr/commands/GetSolrHostsCommand.java | 53 ++++++ .../common-services/LOGSEARCH/0.5.0/alerts.json | 159 ++++++++++-------- .../configuration/logsearch-properties.xml | 7 + .../0.5.0/configuration/logsearch-solr-env.xml | 7 + .../LOGSEARCH/0.5.0/metainfo.xml | 1 + .../LOGSEARCH/0.5.0/metrics.json | 51 ++++++ .../package/alerts/alert_solr_cpu_usage.py | 104 ------------ .../package/alerts/alert_solr_index_size.py | 100 ----------- .../package/alerts/alert_solr_memory_usage.py | 103 ------------ .../LOGSEARCH/0.5.0/package/scripts/params.py | 3 + .../package/templates/logsearch.properties.j2 | 7 +- .../0.5.0/properties/logsearch-solr-env.sh.j2 | 4 +- .../test/python/stacks/2.4/configs/default.json | 4 +- ambari-web/app/data/HDP2/site_properties.js | 20 ++- 20 files changed, 617 insertions(+), 388 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/dcd19817/ambari-logsearch/ambari-logsearch-portal/pom.xml ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/pom.xml b/ambari-logsearch/ambari-logsearch-portal/pom.xml index b6dd40f..e83c75b 100755 --- a/ambari-logsearch/ambari-logsearch-portal/pom.xml +++ b/ambari-logsearch/ambari-logsearch-portal/pom.xml @@ -554,5 +554,15 @@ <artifactId>ambari-logsearch-appender</artifactId> <version>${project.version}</version> </dependency> + <dependency> + <groupId>org.apache.ambari</groupId> + <artifactId>ambari-logsearch-solr-client</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.ambari</groupId> + <artifactId>ambari-metrics-common</artifactId> + <version>${project.version}</version> + </dependency> </dependencies> </project> http://git-wip-us.apache.org/repos/asf/ambari/blob/dcd19817/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/LogSearch.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/LogSearch.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/LogSearch.java index 14c0c1e..2ebf981 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/LogSearch.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/LogSearch.java @@ -24,6 +24,7 @@ import java.util.List; import java.util.Timer; import org.apache.ambari.logsearch.common.ManageStartEndTime; +import org.apache.ambari.logsearch.solr.metrics.SolrMetricsLoader; import org.apache.ambari.logsearch.util.ConfigUtil; import org.apache.hadoop.http.HttpServer2; import org.apache.log4j.Logger; @@ -52,5 +53,7 @@ public class LogSearch { } catch (Throwable e) { logger.error("Error running logsearch server", e); } + + SolrMetricsLoader.startSolrMetricsLoaderTasks(); } } http://git-wip-us.apache.org/repos/asf/ambari/blob/dcd19817/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/solr/metrics/SolrAmsClient.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/solr/metrics/SolrAmsClient.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/solr/metrics/SolrAmsClient.java new file mode 100644 index 0000000..1a0c573 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/solr/metrics/SolrAmsClient.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ambari.logsearch.solr.metrics; + +import org.apache.hadoop.metrics2.sink.timeline.AbstractTimelineMetricsSink; +import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics; + +public class SolrAmsClient extends AbstractTimelineMetricsSink { + private final String collectorHost; + + public SolrAmsClient(String collectorHost) { + this.collectorHost = collectorHost; + } + + @Override + public String getCollectorUri() { + return collectorHost; + } + + @Override + protected int getTimeoutSeconds() { + return 10; + } + + @Override + protected void emitMetrics(TimelineMetrics metrics) { + super.emitMetrics(metrics); + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/dcd19817/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/solr/metrics/SolrJmxAdapter.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/solr/metrics/SolrJmxAdapter.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/solr/metrics/SolrJmxAdapter.java new file mode 100644 index 0000000..fc58661 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/solr/metrics/SolrJmxAdapter.java @@ -0,0 +1,134 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.solr.metrics; + +import java.io.IOException; +import java.lang.management.MemoryMXBean; +import java.net.MalformedURLException; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +import javax.management.JMX; +import javax.management.MBeanServerConnection; +import javax.management.MalformedObjectNameException; +import javax.management.ObjectName; +import javax.management.remote.JMXConnector; +import javax.management.remote.JMXConnectorFactory; +import javax.management.remote.JMXServiceURL; + +import com.sun.management.OperatingSystemMXBean; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@SuppressWarnings("restriction") +public class SolrJmxAdapter { + private static final Logger LOG = LoggerFactory.getLogger(SolrJmxAdapter.class); + + private static final String JMX_SERVICE_URL = "service:jmx:rmi:///jndi/rmi://%s:%d/jmxrmi"; + + private final JMXServiceURL jmxServiceUrl; + + private JMXConnector jmxConnector; + private MBeanServerConnection conn; + + public SolrJmxAdapter(String host, int port) throws MalformedURLException { + String url = String.format(JMX_SERVICE_URL, host, port); + jmxServiceUrl = new JMXServiceURL(url); + } + + public double getProcessCpuLoad() throws MalformedObjectNameException { + ObjectName objectName = new ObjectName("java.lang:type=OperatingSystem"); + OperatingSystemMXBean mxBean = JMX.newMXBeanProxy(conn, objectName, OperatingSystemMXBean.class); + return mxBean.getProcessCpuLoad(); + } + + public Map<String, Long> getMemoryData() throws MalformedObjectNameException { + Map<String, Long> memoryData = new HashMap<>(); + + ObjectName objectName = new ObjectName("java.lang:type=Memory"); + MemoryMXBean mxBean = JMX.newMXBeanProxy(conn, objectName, MemoryMXBean.class); + + memoryData.put("heapMemoryUsed", mxBean.getHeapMemoryUsage().getUsed()); + memoryData.put("heapMemoryCommitted", mxBean.getHeapMemoryUsage().getCommitted()); + memoryData.put("heapMemoryMax", mxBean.getHeapMemoryUsage().getMax()); + memoryData.put("nonHeapMemoryUsed", mxBean.getNonHeapMemoryUsage().getUsed()); + memoryData.put("nonHeapMemoryCommitted", mxBean.getNonHeapMemoryUsage().getCommitted()); + memoryData.put("nonHeapMemoryMax", mxBean.getNonHeapMemoryUsage().getMax()); + + return memoryData; + } + + public long getIndexSize() throws Exception { + long indexSize = 0; + + ObjectName objectNamePattern = new ObjectName( + "solr/*shard*replica*:type=/replication,id=org.apache.solr.handler.ReplicationHandler"); + Set<ObjectName> objectNames = conn.queryNames(objectNamePattern, null); + for (ObjectName objectName : objectNames) { + String indexSizeString = (String) conn.getAttribute(objectName, "indexSize"); + indexSize += getIndexSizeInBytes(indexSizeString); + } + + return indexSize; + } + + private long getIndexSizeInBytes(String indexSizeString) { + String[] tokens = indexSizeString.split(" "); + double number = Double.parseDouble(tokens[0]); + + long multiplier = 0; + switch (tokens[1]) { + case "bytes": + multiplier = 1; + break; + case "KB": + multiplier = 1024; + break; + case "MB": + multiplier = 1024 * 1024; + break; + case "GB": + multiplier = 1024 * 1024 * 1024; + break; + default: + throw new IllegalArgumentException("Unknown unit: " + tokens[1]); + } + + return (long)(number * multiplier); + } + + public void reConnect() throws IOException { + if (jmxConnector != null) { + try { + jmxConnector.close(); + } catch (IOException e) { + LOG.info("Could not close jmxConnector", e); + } + } + + connect(); + } + + public void connect() throws IOException { + jmxConnector = JMXConnectorFactory.connect(jmxServiceUrl); + conn = jmxConnector.getMBeanServerConnection(); + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/dcd19817/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/solr/metrics/SolrMetricsLoader.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/solr/metrics/SolrMetricsLoader.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/solr/metrics/SolrMetricsLoader.java new file mode 100644 index 0000000..ac0b669 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/solr/metrics/SolrMetricsLoader.java @@ -0,0 +1,168 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.solr.metrics; + +import java.io.IOException; +import java.util.Collection; +import java.util.Map; +import java.util.Timer; +import java.util.TimerTask; + +import javax.management.MalformedObjectNameException; + +import org.apache.ambari.logsearch.solr.AmbariSolrCloudClient; +import org.apache.ambari.logsearch.solr.AmbariSolrCloudClientBuilder; +import org.apache.ambari.logsearch.util.PropertiesUtil; +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric; +import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class SolrMetricsLoader extends TimerTask { + private static final Logger LOG = LoggerFactory.getLogger(SolrMetricsLoader.class); + + private final String solrHost; + private final SolrJmxAdapter solrJmxAdapter; + private final SolrAmsClient solrAmsClient; + + public SolrMetricsLoader(String solrHost, int solrJmxPort, String collectorHost) throws IOException { + this.solrHost = solrHost; + this.solrJmxAdapter = new SolrJmxAdapter(solrHost, solrJmxPort); + this.solrAmsClient = new SolrAmsClient(collectorHost); + + solrJmxAdapter.connect(); + } + + @Override + public void run() { + LOG.info("Loading Solr Metrics for the host " + solrHost); + + TimelineMetrics metrics = new TimelineMetrics(); + + addCpuUsageMetric(metrics); + addHeapMemoryUsageMetric(metrics); + addIndexSizeMetric(metrics); + + solrAmsClient.emitMetrics(metrics); + } + + private void addCpuUsageMetric(TimelineMetrics metrics) { + Exception lastException = null; + for (int retries = 0; retries < 3; retries++) { + try { + double processCpuLoad = solrJmxAdapter.getProcessCpuLoad(); + addMetric("logsearch.solr.cpu.usage", "Float", processCpuLoad, metrics); + return; + } catch (MalformedObjectNameException e) { + lastException = e; + try { + solrJmxAdapter.reConnect(); + } catch (IOException e1) { + } + } + } + + LOG.info("Could not load solr cpu usage metric, last exception:", lastException); + } + + private void addHeapMemoryUsageMetric(TimelineMetrics metrics) { + Exception lastException = null; + for (int retries = 0; retries < 3; retries++) { + try { + Map<String, Long> memoryData = solrJmxAdapter.getMemoryData(); + addMetric("jvm.JvmMetrics.MemHeapUsedM", "Long", memoryData.get("heapMemoryUsed").doubleValue() / 1024 / 1024, metrics); + addMetric("jvm.JvmMetrics.MemHeapCommittedM", "Long", memoryData.get("heapMemoryCommitted").doubleValue() / 1024 / 1024, metrics); + addMetric("jvm.JvmMetrics.MemHeapMaxM", "Long", memoryData.get("heapMemoryMax").doubleValue() / 1024 / 1024, metrics); + addMetric("jvm.JvmMetrics.MemNonHeapUsedM", "Long", memoryData.get("nonHeapMemoryUsed").doubleValue() / 1024 / 1024, metrics); + addMetric("jvm.JvmMetrics.MemNonHeapCommittedM", "Long", memoryData.get("nonHeapMemoryCommitted").doubleValue() / 1024 / 1024, metrics); + addMetric("jvm.JvmMetrics.MemNonHeapMaxM", "Long", memoryData.get("nonHeapMemoryMax").doubleValue() / 1024 / 1024, metrics); + return; + } catch (MalformedObjectNameException e) { + lastException = e; + try { + solrJmxAdapter.reConnect(); + } catch (IOException e1) { + } + } + } + + LOG.info("Could not load solr heap memory usage metric, last exception:", lastException); + } + + private void addIndexSizeMetric(TimelineMetrics metrics) { + Exception lastException = null; + for (int retries = 0; retries < 3; retries++) { + try { + double indexSize = solrJmxAdapter.getIndexSize(); + addMetric("logsearch.solr.index.size", "Long", indexSize / 1024 / 1024 / 1024, metrics); + return; + } catch (Exception e) { + lastException = e; + try { + solrJmxAdapter.reConnect(); + } catch (IOException e1) { + } + } + } + + LOG.info("Could not load solr index size metric, last exception:", lastException); + } + + private void addMetric(String metricName, String type, Double value, TimelineMetrics metrics) { + Long currMS = System.currentTimeMillis(); + + TimelineMetric metric = new TimelineMetric(); + metric.setMetricName(metricName); + metric.setHostName(solrHost); + metric.setAppId("logsearch-solr"); + metric.setStartTime(currMS); + metric.setType(type); + metric.setTimestamp(currMS); + metric.getMetricValues().put(currMS, value); + + metrics.addOrMergeTimelineMetric(metric); + } + + public static void startSolrMetricsLoaderTasks() { + try { + String collectorHosts = PropertiesUtil.getProperty("metrics.collector.hosts"); + if (StringUtils.isEmpty(collectorHosts)) { + LOG.warn("No Ambari Metrics service is available, no Solr metrics will be loaded!"); + return; + } + + int solrJmxPort = PropertiesUtil.getIntProperty("solr.jmx.port"); + + String zkHosts = PropertiesUtil.getProperty("solr.zkhosts"); + AmbariSolrCloudClient ambariSolrCloudClient = new AmbariSolrCloudClientBuilder() + .withZookeeperHosts(zkHosts) + .build(); + + Collection<String> solrHosts = ambariSolrCloudClient.getSolrHosts(); + for (String solrHost : solrHosts) { + SolrMetricsLoader sml = new SolrMetricsLoader(solrHost, solrJmxPort, collectorHosts); + Timer timer = new Timer("Solr Metrics Loader - " + solrHost, false); + timer.scheduleAtFixedRate(sml, 0, 10000); + } + } catch (Exception e) { + LOG.warn("Could not start solr metric loader tasks", e); + } + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/dcd19817/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/AmbariSolrCloudClient.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/AmbariSolrCloudClient.java b/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/AmbariSolrCloudClient.java index 7cfed89..2805b0b 100644 --- a/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/AmbariSolrCloudClient.java +++ b/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/AmbariSolrCloudClient.java @@ -23,19 +23,17 @@ import org.apache.ambari.logsearch.solr.commands.CreateCollectionCommand; import org.apache.ambari.logsearch.solr.commands.CreateShardCommand; import org.apache.ambari.logsearch.solr.commands.DownloadConfigZkCommand; import org.apache.ambari.logsearch.solr.commands.GetShardsCommand; +import org.apache.ambari.logsearch.solr.commands.GetSolrHostsCommand; import org.apache.ambari.logsearch.solr.commands.ListCollectionCommand; import org.apache.ambari.logsearch.solr.commands.UploadConfigZkCommand; import org.apache.ambari.logsearch.solr.util.ShardUtils; import org.apache.solr.client.solrj.impl.CloudSolrClient; -import org.apache.solr.common.cloud.Replica; import org.apache.solr.common.cloud.Slice; import org.apache.solr.common.cloud.SolrZkClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.Collection; -import java.util.HashSet; -import java.util.Iterator; import java.util.List; /** @@ -88,7 +86,7 @@ public class AmbariSolrCloudClient { * Create Solr collection if exists */ public String createCollection() throws Exception { - List <String> collections = listCollections(); + List<String> collections = listCollections(); if (!collections.contains(getCollection())) { String collection = new CreateCollectionCommand(getRetryTimes(), getInterval()).run(this); LOG.info("Collection '{}' created.", collection); @@ -133,9 +131,11 @@ public class AmbariSolrCloudClient { } /** - * Create shard in collection - create a new one if shard name specified, if not create based on - * the number of shards logic (with shard_# suffix) - * @param shard name of the created shard + * Create shard in collection - create a new one if shard name specified, if + * not create based on the number of shards logic (with shard_# suffix) + * + * @param shard + * name of the created shard */ public Collection<String> createShard(String shard) throws Exception { Collection<String> existingShards = getShardNames(); @@ -163,6 +163,13 @@ public class AmbariSolrCloudClient { return ShardUtils.getShardNamesFromSlices(slices, this.getCollection()); } + /** + * Get Solr Hosts + */ + public Collection<String> getSolrHosts() throws Exception { + return new GetSolrHostsCommand(getRetryTimes(), getInterval()).run(this); + } + public String getZookeeperHosts() { return zookeeperHosts; } http://git-wip-us.apache.org/repos/asf/ambari/blob/dcd19817/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/GetSolrHostsCommand.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/GetSolrHostsCommand.java b/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/GetSolrHostsCommand.java new file mode 100644 index 0000000..f814678 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/GetSolrHostsCommand.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.solr.commands; + +import org.apache.ambari.logsearch.solr.AmbariSolrCloudClient; +import org.apache.zookeeper.ZooKeeper; + +import java.net.InetAddress; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +public class GetSolrHostsCommand extends AbstractRetryCommand<Collection<String>> { + + public GetSolrHostsCommand(int maxRetries, int interval) { + super(maxRetries, interval); + } + + @Override + public Collection<String> createAndProcessRequest(AmbariSolrCloudClient solrCloudClient) throws Exception { + List<String> solrHosts = new ArrayList<>(); + + ZooKeeper zk = new ZooKeeper(solrCloudClient.getZookeeperHosts(), 10000, null); + List<String> ids = zk.getChildren("/live_nodes", false); + for (String id : ids) { + if (id.endsWith("_solr")) { + String hostAndPort = id.substring(0, id.length() - 5); + String[] tokens = hostAndPort.split(":"); + String host = InetAddress.getByName(tokens[0]).getHostName(); + + solrHosts.add(host); + } + } + + return solrHosts; + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/dcd19817/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/alerts.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/alerts.json b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/alerts.json index d58fdf8..0ab2b19 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/alerts.json +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/alerts.json @@ -63,62 +63,78 @@ "description": "This host-level alert is triggered if the Solr index size is over a limit.", "interval": 1, "scope": "ANY", - "enabled" : false, "source": { - "parameters" : [ - { - "name" : "solr.index.size.warning", - "display_name" : "Solr index size warning threshold", - "units" : "GB", - "value" : 50, - "description" : "The maximum size of the Solr index before this alert is considered to be WARNING", - "type" : "NUMERIC", - "threshold" : "WARNING" + "type": "AMS", + "uri": { + "http": "{{ams-site/timeline.metrics.service.webapp.address}}", + "https": "{{ams-site/timeline.metrics.service.webapp.address}}", + "https_property": "{{ams-site/timeline.metrics.service.http.policy}}", + "https_property_value": "HTTPS_ONLY", + "connection_timeout": 5.0 + }, + "reporting": { + "ok": { + "text": "Solr index size is {0:.1f}GB" }, - { - "name" : "solr.index.size.critical", - "display_name" : "Solr index size critical threshold", - "units" : "GB", - "value" : 100, - "description" : "The maximum size of the Solr index before this alert is considered to be CRITICAL", - "type" : "NUMERIC", - "threshold" : "CRITICAL" - } - ], - "path" : "LOGSEARCH/0.5.0/package/alerts/alert_solr_index_size.py", - "type" : "SCRIPT" + "warning": { + "text": "Solr index size is {0:.1f}GB", + "value": 50 + }, + "critical": { + "text": "Solr index size is {0:.1f}GB", + "value": 100 + }, + "units" : "GB" + }, + "ams": { + "app_id": "logsearch-solr", + "interval": 1, + "metric_list": [ + "logsearch.solr.index.size" + ], + "value": "{0}", + "compute": "mean" + } } }, { - "name": "logsearch_solr_cpu", + "name": "logsearch_solr_cpu_usage", "label": "LogSearch Solr CPU Utilization", "description": "This host-level alert is triggered if CPU utilization of the LogSearch Solr exceeds certain warning and critical thresholds. It checks the LogSearch Solr JMX Servlet for the SystemCPULoad property. The threshold values are in percent.", "interval": 5, "scope": "ANY", - "enabled": true, "source": { - "parameters" : [ - { - "name" : "solr.cpu.usage.warning", - "display_name" : "Solr cpu usage warning threshold", - "units" : "%", - "value" : 200, - "description" : "The maximum cpu usage of Solr before this alert is considered to be WARNING", - "type" : "NUMERIC", - "threshold" : "WARNING" + "type": "AMS", + "uri": { + "http": "{{ams-site/timeline.metrics.service.webapp.address}}", + "https": "{{ams-site/timeline.metrics.service.webapp.address}}", + "https_property": "{{ams-site/timeline.metrics.service.http.policy}}", + "https_property_value": "HTTPS_ONLY", + "connection_timeout": 5.0 + }, + "reporting": { + "ok": { + "text": "CPU load {0:.1%}" }, - { - "name" : "solr.cpu.usage.critical", - "display_name" : "Solr cpu usage critical threshold", - "units" : "%", - "value" : 250, - "description" : "The maximum cpu usage of Solr index before this alert is considered to be CRITICAL", - "type" : "NUMERIC", - "threshold" : "CRITICAL" - } - ], - "path" : "LOGSEARCH/0.5.0/package/alerts/alert_solr_cpu_usage.py", - "type" : "SCRIPT" + "warning": { + "text": "CPU load {0:.1%}", + "value": 200 + }, + "critical": { + "text": "CPU load {0:.1%}", + "value": 250 + }, + "units" : "%" + }, + "ams": { + "app_id": "logsearch-solr", + "interval": 1, + "metric_list": [ + "logsearch.solr.cpu.usage" + ], + "value": "{0}", + "compute": "mean" + } } }, { @@ -127,30 +143,39 @@ "description": "This host-level alert is triggered if CPU utilization of the LogSearch Solr exceeds certain warning and critical thresholds. It checks the LogSearch Solr JMX Servlet for the SystemCPULoad property. The threshold values are in percent.", "interval": 5, "scope": "ANY", - "enabled": true, "source": { - "parameters" : [ - { - "name" : "solr.memory.usage.warning", - "display_name" : "Solr memory usage warning threshold", - "units" : "%", - "value" : 200, - "description" : "The maximum memory usage of Solr before this alert is considered to be WARNING", - "type" : "NUMERIC", - "threshold" : "WARNING" + "type": "AMS", + "uri": { + "http": "{{ams-site/timeline.metrics.service.webapp.address}}", + "https": "{{ams-site/timeline.metrics.service.webapp.address}}", + "https_property": "{{ams-site/timeline.metrics.service.http.policy}}", + "https_property_value": "HTTPS_ONLY", + "connection_timeout": 5.0 + }, + "reporting": { + "ok": { + "text": "Memory usage is {0:.1%}" }, - { - "name" : "solr.memory.usage.critical", - "display_name" : "Solr memory usage critical threshold", - "units" : "%", - "value" : 250, - "description" : "The maximum memory usage of Solr index before this alert is considered to be CRITICAL", - "type" : "NUMERIC", - "threshold" : "CRITICAL" - } - ], - "path" : "LOGSEARCH/0.5.0/package/alerts/alert_solr_memory_usage.py", - "type" : "SCRIPT" + "warning": { + "text": "Memory usage is {0:.1%}", + "value": 200 + }, + "critical": { + "text": "Memory usage is {0:.1%}", + "value": 250 + }, + "units" : "%" + }, + "ams": { + "app_id": "logsearch-solr", + "interval": 1, + "metric_list": [ + "jvm.JvmMetrics.MemHeapUsedM", + "jvm.JvmMetrics.MemHeapMaxM" + ], + "value": "{0} / {1}", + "compute": "mean" + } } } ] http://git-wip-us.apache.org/repos/asf/ambari/blob/dcd19817/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-properties.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-properties.xml b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-properties.xml index 85f0e4f..f04a5c4 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-properties.xml +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-properties.xml @@ -120,4 +120,11 @@ <display-name>Logfeeder log levels</display-name> </property> + <property> + <name>logsearch.solr.metrics.collector.hosts</name> + <value>{metrics_collector_hosts}</value> + <description>Metrics collector hosts for pushing metrics by LogSearch Solr</description> + <display-name>Metrics collector hosts</display-name> + </property> + </configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/dcd19817/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-solr-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-solr-env.xml b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-solr-env.xml index f08e57d..ce53692 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-solr-env.xml +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-solr-env.xml @@ -30,6 +30,13 @@ </property> <property> + <name>logsearch_solr_jmx_port</name> + <value>18886</value> + <description>Solr jmx port</description> + <display-name>Logsearch Solr jmx port</display-name> + </property> + + <property> <name>logsearch_solr_pid_dir</name> <value>/var/run/ambari-logsearch-solr</value> <description>Solr Process ID Directory</description> http://git-wip-us.apache.org/repos/asf/ambari/blob/dcd19817/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml index b497c67..7d8959d 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml @@ -74,6 +74,7 @@ <component> <name>LOGSEARCH_SOLR</name> + <timelineAppid>logsearch-solr</timelineAppid> <displayName>Log Search Solr Instance</displayName> <category>MASTER</category> <cardinality>1+</cardinality> http://git-wip-us.apache.org/repos/asf/ambari/blob/dcd19817/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metrics.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metrics.json b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metrics.json index 7ffb77e..1e1a388 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metrics.json +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metrics.json @@ -49,5 +49,56 @@ } } ] + }, + "LOGSEARCH_SOLR": { + "Component": [ + { + "type": "ganglia", + "metrics": { + "default": { + "metrics/logsearch/solr/cpu/usage": { + "metric": "logsearch.solr.cpu.usage", + "pointInTime": true, + "temporal": true + }, + "metrics/jvm/memHeapUsedM": { + "metric": "jvm.JvmMetrics.MemHeapUsedM", + "pointInTime": true, + "temporal": true + }, + "metrics/jvm/memHeapCommittedM": { + "metric": "jvm.JvmMetrics.MemHeapCommittedM", + "pointInTime": true, + "temporal": true + }, + "metrics/jvm/memHeapMaxM": { + "metric": "jvm.JvmMetrics.MemHeapMaxM", + "pointInTime": true, + "temporal": true + }, + "metrics/jvm/memNonHeapUsedM": { + "metric": "jvm.JvmMetrics.MemNonHeapUsedM", + "pointInTime": true, + "temporal": true + }, + "metrics/jvm/memNonHeapCommittedM": { + "metric": "jvm.JvmMetrics.MemNonHeapCommittedM", + "pointInTime": true, + "temporal": true + }, + "metrics/jvm/memNonHeapMaxM": { + "metric": "jvm.JvmMetrics.MemNonHeapMaxM", + "pointInTime": true, + "temporal": true + }, + "metrics/logsearch/solr/index/size": { + "metric": "logsearch.solr.index.size", + "pointInTime": true, + "temporal": true + } + } + } + } + ] } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/dcd19817/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/alerts/alert_solr_cpu_usage.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/alerts/alert_solr_cpu_usage.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/alerts/alert_solr_cpu_usage.py deleted file mode 100644 index 3e070b1..0000000 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/alerts/alert_solr_cpu_usage.py +++ /dev/null @@ -1,104 +0,0 @@ -#!/usr/bin/env python - -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" - -import traceback -import urllib2 -import ambari_simplejson as json # simplejson is much faster comparing to Python 2.6 json module and has the same functions set. - -RESULT_CODE_OK = 'OK' -RESULT_CODE_WARNING = 'WARNING' -RESULT_CODE_CRITICAL = 'CRITICAL' -RESULT_CODE_UNKNOWN = 'UNKNOWN' - -SOLR_PORT = '{{logsearch-solr-env/logsearch_solr_port}}' - -MESSAGE = '{1} CPU, load {0:.1%}' - -SOLR_CPU_USAGE_WARNING_KEY = 'solr.cpu.usage.warning' -SOLR_CPU_USAGE_WARNING_DEFAULT = 200 - -SOLR_CPU_USAGE_CRITICAL_KEY = 'solr.cpu.usage.critical' -SOLR_CPU_USAGE_CRITICAL_DEFAULT = 250 - -CRITICAL_CONNECTION_MESSAGE = 'Could not execute query {0}\n{1}' - -def get_tokens(): - """ - Returns a tuple of tokens in the format {{site/property}} that will be used - to build the dictionary passed into execute - """ - return (SOLR_PORT,) - - -def execute(configurations={}, parameters={}, host_name=None): - """ - Returns a tuple containing the result code and a pre-formatted result label - - Keyword arguments: - configurations (dictionary): a mapping of configuration key to value - parameters (dictionary): a mapping of script parameter key to value - host_name (string): the name of this host where the alert is running - """ - - if configurations is None: - return (RESULT_CODE_UNKNOWN, ['There were no configurations supplied to the script.']) - - if SOLR_PORT in configurations: - solr_port = configurations[SOLR_PORT] - else: - return (RESULT_CODE_UNKNOWN, ['No Solr port specified']) - - # parse script arguments - solr_cpu_usage_warning = SOLR_CPU_USAGE_WARNING_DEFAULT - if SOLR_CPU_USAGE_WARNING_KEY in parameters: - solr_cpu_usage_warning = float(parameters[SOLR_CPU_USAGE_WARNING_KEY]) - - solr_cpu_usage_critical = SOLR_CPU_USAGE_CRITICAL_DEFAULT - if SOLR_CPU_USAGE_CRITICAL_KEY in parameters: - solr_cpu_usage_critical = float(parameters[SOLR_CPU_USAGE_CRITICAL_KEY]) - - - try: - query = "http://localhost:"; + str(solr_port) + "/solr/admin/cores?action=STATUS&indexInfo=false&wt=json" - shard_response = urllib2.urlopen(query) - shard_raw_data = shard_response.read() - shard_json_data = json.loads(shard_raw_data) - - shard_name = shard_json_data["status"].keys()[0] - query = "http://localhost:"; + str(solr_port) + "/solr/" + shard_name + "/admin/system?wt=json" - shard_details_response = urllib2.urlopen(query) - shard_details_raw_data = shard_details_response.read() - shard_details_json_data = json.loads(shard_details_raw_data) - cpu_load = shard_details_json_data["system"]["processCpuLoad"] - available_processors = shard_details_json_data["system"]["availableProcessors"] - except: - label = CRITICAL_CONNECTION_MESSAGE.format(query, traceback.format_exc()) - return (RESULT_CODE_CRITICAL, [label]) - - cpu_percent = cpu_load * 100.0 - label = MESSAGE.format(cpu_load, available_processors) - if cpu_percent <= solr_cpu_usage_warning: - result_code = RESULT_CODE_OK - elif cpu_percent <= solr_cpu_usage_critical: - result_code = RESULT_CODE_WARNING - else: - result_code = RESULT_CODE_CRITICAL - - return (result_code, [label]) http://git-wip-us.apache.org/repos/asf/ambari/blob/dcd19817/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/alerts/alert_solr_index_size.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/alerts/alert_solr_index_size.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/alerts/alert_solr_index_size.py deleted file mode 100644 index 63e4edf..0000000 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/alerts/alert_solr_index_size.py +++ /dev/null @@ -1,100 +0,0 @@ -#!/usr/bin/env python - -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" - -import traceback -import urllib2 -import ambari_simplejson as json # simplejson is much faster comparing to Python 2.6 json module and has the same functions set. - -RESULT_CODE_OK = 'OK' -RESULT_CODE_WARNING = 'WARNING' -RESULT_CODE_CRITICAL = 'CRITICAL' -RESULT_CODE_UNKNOWN = 'UNKNOWN' - -SOLR_PORT = '{{logsearch-solr-env/logsearch_solr_port}}' - -MESSAGE = 'Solr index size is {0:.1f}GB' - -SOLR_INDEX_SIZE_WARNING_KEY = 'solr.index.size.warning' -SOLR_INDEX_SIZE_WARNING_DEFAULT = 50 - -SOLR_INDEX_SIZE_CRITICAL_KEY = 'solr.index.size.critical' -SOLR_INDEX_SIZE_CRITICAL_DEFAULT = 100 - -CRITICAL_CONNECTION_MESSAGE = 'Could not execute query {0}\n{1}' - -def get_tokens(): - """ - Returns a tuple of tokens in the format {{site/property}} that will be used - to build the dictionary passed into execute - """ - return (SOLR_PORT,) - - -def execute(configurations={}, parameters={}, host_name=None): - """ - Returns a tuple containing the result code and a pre-formatted result label - - Keyword arguments: - configurations (dictionary): a mapping of configuration key to value - parameters (dictionary): a mapping of script parameter key to value - host_name (string): the name of this host where the alert is running - """ - - if configurations is None: - return (RESULT_CODE_UNKNOWN, ['There were no configurations supplied to the script.']) - - if SOLR_PORT in configurations: - solr_port = configurations[SOLR_PORT] - else: - return (RESULT_CODE_UNKNOWN, ['No Solr port specified']) - - # parse script arguments - solr_index_size_warning = SOLR_INDEX_SIZE_WARNING_DEFAULT - if SOLR_INDEX_SIZE_WARNING_KEY in parameters: - solr_index_size_warning = float(parameters[SOLR_INDEX_SIZE_WARNING_KEY]) - - solr_index_size_critical = SOLR_INDEX_SIZE_CRITICAL_DEFAULT - if SOLR_INDEX_SIZE_CRITICAL_KEY in parameters: - solr_index_size_critical = float(parameters[SOLR_INDEX_SIZE_CRITICAL_KEY]) - - - try: - query = "http://localhost:"; + str(solr_port) + "/solr/admin/cores?action=STATUS&wt=json" - response = urllib2.urlopen(query) - raw_data = response.read() - json_data = json.loads(raw_data) - - size_in_bytes = 0 - for shard_data in json_data["status"].itervalues(): - size_in_bytes += shard_data["index"]["sizeInBytes"] - except: - label = CRITICAL_CONNECTION_MESSAGE.format(query, traceback.format_exc()) - return (RESULT_CODE_CRITICAL, [label]) - - size_in_gb = float(size_in_bytes) / float(1024 * 1024 * 1024) - label = MESSAGE.format(size_in_gb) - if size_in_gb <= solr_index_size_warning: - result_code = RESULT_CODE_OK - elif size_in_gb <= solr_index_size_critical: - result_code = RESULT_CODE_WARNING - else: - result_code = RESULT_CODE_CRITICAL - - return (result_code, [label]) http://git-wip-us.apache.org/repos/asf/ambari/blob/dcd19817/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/alerts/alert_solr_memory_usage.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/alerts/alert_solr_memory_usage.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/alerts/alert_solr_memory_usage.py deleted file mode 100644 index b36af34..0000000 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/alerts/alert_solr_memory_usage.py +++ /dev/null @@ -1,103 +0,0 @@ -#!/usr/bin/env python - -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" - -import traceback -import urllib2 -import ambari_simplejson as json # simplejson is much faster comparing to Python 2.6 json module and has the same functions set. - -RESULT_CODE_OK = 'OK' -RESULT_CODE_WARNING = 'WARNING' -RESULT_CODE_CRITICAL = 'CRITICAL' -RESULT_CODE_UNKNOWN = 'UNKNOWN' - -SOLR_PORT = '{{logsearch-solr-env/logsearch_solr_port}}' - -MESSAGE = 'Memory usage is {0:.1%}' - -SOLR_MEMORY_USAGE_WARNING_KEY = 'solr.memory.usage.warning' -SOLR_MEMORY_USAGE_WARNING_DEFAULT = 200 - -SOLR_MEMORY_USAGE_CRITICAL_KEY = 'solr.memory.usage.critical' -SOLR_MEMORY_USAGE_CRITICAL_DEFAULT = 250 - -CRITICAL_CONNECTION_MESSAGE = 'Could not execute query {0}\n{1}' - -def get_tokens(): - """ - Returns a tuple of tokens in the format {{site/property}} that will be used - to build the dictionary passed into execute - """ - return (SOLR_PORT,) - - -def execute(configurations={}, parameters={}, host_name=None): - """ - Returns a tuple containing the result code and a pre-formatted result label - - Keyword arguments: - configurations (dictionary): a mapping of configuration key to value - parameters (dictionary): a mapping of script parameter key to value - host_name (string): the name of this host where the alert is running - """ - - if configurations is None: - return (RESULT_CODE_UNKNOWN, ['There were no configurations supplied to the script.']) - - if SOLR_PORT in configurations: - solr_port = configurations[SOLR_PORT] - else: - return (RESULT_CODE_UNKNOWN, ['No Solr port specified']) - - # parse script arguments - solr_memory_usage_warning = SOLR_MEMORY_USAGE_WARNING_DEFAULT - if SOLR_MEMORY_USAGE_WARNING_KEY in parameters: - solr_memory_usage_warning = float(parameters[SOLR_MEMORY_USAGE_WARNING_KEY]) - - solr_memory_usage_critical = SOLR_MEMORY_USAGE_CRITICAL_DEFAULT - if SOLR_MEMORY_USAGE_CRITICAL_KEY in parameters: - solr_memory_usage_critical = float(parameters[SOLR_MEMORY_USAGE_CRITICAL_KEY]) - - - try: - query = "http://localhost:"; + str(solr_port) + "/solr/admin/cores?action=STATUS&indexInfo=false&wt=json" - shard_response = urllib2.urlopen(query) - shard_raw_data = shard_response.read() - shard_json_data = json.loads(shard_raw_data) - - shard_name = shard_json_data["status"].keys()[0] - query = "http://localhost:"; + str(solr_port) + "/solr/" + shard_name + "/admin/system?wt=json" - shard_details_response = urllib2.urlopen(query) - shard_details_raw_data = shard_details_response.read() - shard_details_json_data = json.loads(shard_details_raw_data) - memory_percent = shard_details_json_data["jvm"]["memory"]["raw"]["used%"] - except: - label = CRITICAL_CONNECTION_MESSAGE.format(query, traceback.format_exc()) - return (RESULT_CODE_CRITICAL, [label]) - - memory_load = memory_percent / 100.0 - label = MESSAGE.format(memory_load) - if memory_percent <= solr_memory_usage_warning: - result_code = RESULT_CODE_OK - elif memory_percent <= solr_memory_usage_critical: - result_code = RESULT_CODE_WARNING - else: - result_code = RESULT_CODE_CRITICAL - - return (result_code, [label]) http://git-wip-us.apache.org/repos/asf/ambari/blob/dcd19817/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py index 6c26afc..183425a 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py @@ -98,6 +98,9 @@ logsearch_service_logs_merge_factor = config['configurations']['logsearch-servic logsearch_audit_logs_max_retention = config['configurations']['logsearch-audit_logs-solrconfig']['logsearch_audit_logs_max_retention'] logsearch_audit_logs_merge_factor = config['configurations']['logsearch-audit_logs-solrconfig']['logsearch_audit_logs_merge_factor'] +logsearch_solr_metrics_collector_hosts = format(config['configurations']['logsearch-properties']['logsearch.solr.metrics.collector.hosts']) +logsearch_solr_jmx_port = config['configurations']['logsearch-solr-env']['logsearch_solr_jmx_port'] + logsearch_service_logs_fields = config['configurations']['logsearch-properties']['logsearch.service.logs.fields'] audit_logs_collection_splits_interval_mins = config['configurations']['logsearch-properties']['logsearch.audit.logs.split.interval.mins'] http://git-wip-us.apache.org/repos/asf/ambari/blob/dcd19817/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/logsearch.properties.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/logsearch.properties.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/logsearch.properties.j2 index 3813a0a..4d248b4 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/logsearch.properties.j2 +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/logsearch.properties.j2 @@ -39,7 +39,11 @@ solr.core.history=history solr.history.config_name=history solr.history.replication_factor=1 -#Logfeeder Settings +# Metrics +metrics.collector.hosts={{logsearch_solr_metrics_collector_hosts}} +solr.jmx.port={{logsearch_solr_jmx_port}} + +# Logfeeder Settings logfeeder.include.default.level={{logsearch_logfeeder_log_level_include}} @@ -50,3 +54,4 @@ logsearch.login.credentials.file={{logsearch_admin_credential_file}} {% for key, value in logsearch_custom_properties.items() %} {{key}}={{value}} {% endfor %} + http://git-wip-us.apache.org/repos/asf/ambari/blob/dcd19817/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/logsearch-solr-env.sh.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/logsearch-solr-env.sh.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/logsearch-solr-env.sh.j2 index 357a429..f59ba49 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/logsearch-solr-env.sh.j2 +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/logsearch-solr-env.sh.j2 @@ -60,10 +60,10 @@ ZK_CLIENT_TIMEOUT="60000" # Set to true to activate the JMX RMI connector to allow remote JMX client applications # to monitor the JVM hosting Solr; set to "false" to disable that behavior # (false is recommended in production environments) -ENABLE_REMOTE_JMX_OPTS="false" +ENABLE_REMOTE_JMX_OPTS="true" # The script will use SOLR_PORT+10000 for the RMI_PORT or you can set it here -# RMI_PORT=18983 +RMI_PORT={{logsearch_solr_jmx_port}} # Anything you add to the SOLR_OPTS variable will be included in the java # start command line as-is, in ADDITION to other options. If you specify the http://git-wip-us.apache.org/repos/asf/ambari/blob/dcd19817/ambari-server/src/test/python/stacks/2.4/configs/default.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.4/configs/default.json b/ambari-server/src/test/python/stacks/2.4/configs/default.json index e28b372..63d16e3 100644 --- a/ambari-server/src/test/python/stacks/2.4/configs/default.json +++ b/ambari-server/src/test/python/stacks/2.4/configs/default.json @@ -254,6 +254,7 @@ "logsearch_debug_enabled": "false", "logsearch_debug_port": "5005", "logsearch_solr_audit_logs_use_ranger": "false", + "logsearch_solr_jmx_port": "18886", "content": "# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements. See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License. You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#solr.url=http://{{solr_host}}:{{solr_port}}/solr\n\n#Service Logs and History colletion\nsolr.zkhosts={{zookeeper_quorum}}{{solr_znode}}\nsolr.core.logs={{logsearch_collection_s ervice_logs}}\nsolr.core.history=history\n\nsolr.service_logs.split_interval_mins={{service_logs_collection_splits_interval_mins}}\nsolr.service_logs.shards={{logsearch_service_logs_numshards}}\nsolr.service_logs.replication_factor={{logsearch_service_logs_repfactor}}\n\nsolr.servicelogs.fields={{logsearch_service_logs_fields}}\n\n#Audit logs\nauditlog.solr.zkhosts={{solr_audit_logs_zk_quorum}}{{solr_audit_logs_zk_node}}\nauditlog.solr.core.logs={{solr_collection_audit_logs}}\nauditlog.solr.url={{solr_audit_logs_url}}\n\nsolr.audit_logs.split_interval_mins={{audit_logs_collection_splits_interval_mins}}\nsolr.audit_logs.shards={{logsearch_audit_logs_numshards}}\nsolr.audit_logs.replication_factor={{logsearch_audit_logs_repfactor}}" }, "logsearch-service_logs-solrconfig": { @@ -270,7 +271,8 @@ "logsearch.collection.service.logs.replication.factor": "1", "logsearch.collection.audit.logs.numshards": "10", "logsearch.collection.audit.logs.replication.factor": "1", - "logsearch.app.max.memory": "1g" + "logsearch.app.max.memory": "1g", + "logsearch.solr.metrics.collector.hosts" : "{metrics_collector_hosts}" }, "logfeeder-properties": { "logfeeder.metrics.collector.hosts" : "{metrics_collector_hosts}", http://git-wip-us.apache.org/repos/asf/ambari/blob/dcd19817/ambari-web/app/data/HDP2/site_properties.js ---------------------------------------------------------------------- diff --git a/ambari-web/app/data/HDP2/site_properties.js b/ambari-web/app/data/HDP2/site_properties.js index 024118f..f1ad3cf 100644 --- a/ambari-web/app/data/HDP2/site_properties.js +++ b/ambari-web/app/data/HDP2/site_properties.js @@ -2079,6 +2079,13 @@ var hdp2properties = [ "category": "Advanced logsearch-properties", "index": 6 }, + { + "name": "logsearch.solr.metrics.collector.hosts", + "serviceName": "LOGSEARCH", + "filename": "logsearch-properties.xml", + "category": "Advanced logsearch-properties", + "index": 9 + }, /*logsearch-solr-env*/ { "name": "logsearch_solr_datadir", @@ -2123,26 +2130,33 @@ var hdp2properties = [ "index": 6 }, { - "name": "logsearch_solr_minmem", + "name": "logsearch_solr_jmx_port", "serviceName": "LOGSEARCH", "filename": "logsearch-solr-env.xml", "category": "Advanced logsearch-solr-env", "index": 7 }, { - "name": "logsearch_solr_maxmem", + "name": "logsearch_solr_minmem", "serviceName": "LOGSEARCH", "filename": "logsearch-solr-env.xml", "category": "Advanced logsearch-solr-env", "index": 8 }, { - "name": "content", + "name": "logsearch_solr_maxmem", "serviceName": "LOGSEARCH", "filename": "logsearch-solr-env.xml", "category": "Advanced logsearch-solr-env", "index": 9 }, + { + "name": "content", + "serviceName": "LOGSEARCH", + "filename": "logsearch-solr-env.xml", + "category": "Advanced logsearch-solr-env", + "index": 10 + }, /*logsearch-solr-log4j*/ { "name": "content",
