AMBARI-15679. Stack definition for Logsearch (oleewre)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/4a2dcfbe Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/4a2dcfbe Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/4a2dcfbe Branch: refs/heads/branch-dev-logsearch Commit: 4a2dcfbe0780a0ed543d5c47a859fd4aec693e63 Parents: f729469 Author: oleewere <[email protected]> Authored: Sat Apr 9 11:59:06 2016 +0200 Committer: oleewere <[email protected]> Committed: Sat Apr 9 12:50:54 2016 +0200 ---------------------------------------------------------------------- .../libraries/functions/package_conditions.py | 4 + .../common-services/LOGSEARCH/0.5.0/alerts.json | 63 + .../0.5.0/configuration/logfeeder-env.xml | 118 ++ .../0.5.0/configuration/logfeeder-log4j.xml | 80 + .../0.5.0/configuration/logfeeder-site.xml | 87 + .../logsearch-audit_logs-solrconfig.xml | 1929 ++++++++++++++++++ .../0.5.0/configuration/logsearch-env.xml | 143 ++ .../0.5.0/configuration/logsearch-log4j.xml | 110 + .../logsearch-service_logs-solrconfig.xml | 1928 +++++++++++++++++ .../0.5.0/configuration/logsearch-site.xml | 80 + .../0.5.0/configuration/logsearch-solr-env.xml | 193 ++ .../configuration/logsearch-solr-log4j.xml | 57 + .../0.5.0/configuration/logsearch-solr-xml.xml | 41 + .../LOGSEARCH/0.5.0/kerberos.json | 17 + .../LOGSEARCH/0.5.0/metainfo.xml | 119 ++ .../0.5.0/package/scripts/logfeeder.py | 67 + .../0.5.0/package/scripts/logsearch.py | 65 + .../LOGSEARCH/0.5.0/package/scripts/params.py | 231 +++ .../0.5.0/package/scripts/setup_logfeeder.py | 67 + .../0.5.0/package/scripts/setup_logsearch.py | 93 + .../0.5.0/package/scripts/setup_solr.py | 67 + .../LOGSEARCH/0.5.0/package/scripts/solr.py | 66 + .../0.5.0/package/scripts/status_params.py | 36 + .../package/templates/global.config.json.j2 | 28 + .../0.5.0/package/templates/grok-patterns.j2 | 144 ++ .../templates/input.config-accumulo.json.j2 | 105 + .../templates/input.config-ambari.json.j2 | 93 + .../package/templates/input.config-ams.json.j2 | 92 + .../templates/input.config-atlas.json.j2 | 55 + .../templates/input.config-falcon.json.j2 | 55 + .../templates/input.config-hbase.json.j2 | 62 + .../package/templates/input.config-hdfs.json.j2 | 246 +++ .../package/templates/input.config-hive.json.j2 | 62 + .../templates/input.config-kafka.json.j2 | 105 + .../package/templates/input.config-knox.json.j2 | 68 + .../templates/input.config-logsearch.json.j2 | 68 + .../templates/input.config-oozie.json.j2 | 56 + .../templates/input.config-ranger.json.j2 | 122 ++ .../templates/input.config-storm.json.j2 | 86 + .../package/templates/input.config-yarn.json.j2 | 86 + .../templates/input.config-zookeeper.json.j2 | 56 + .../package/templates/logfeeder.properties.j2 | 18 + .../package/templates/logsearch.properties.j2 | 40 + .../package/templates/output.config.json.j2 | 97 + .../0.5.0/package/templates/zoo.cfg.j2 | 31 + .../LOGSEARCH/0.5.0/quicklinks/quicklinks.json | 28 + .../stacks/HDP/2.2/upgrades/upgrade-2.3.xml | 3 + .../stacks/HDP/2.2/upgrades/upgrade-2.4.xml | 3 + .../stacks/HDP/2.3/role_command_order.json | 5 +- .../HDP/2.3/services/LOGSEARCH/metainfo.xml | 26 + .../stacks/HDP/2.3/upgrades/upgrade-2.3.xml | 3 + .../stacks/HDP/2.3/upgrades/upgrade-2.4.xml | 3 + .../stacks/HDP/2.4/upgrades/upgrade-2.4.xml | 3 + .../stacks/2.4/LOGSEARCH/test_logfeeder.py | 152 ++ .../stacks/2.4/LOGSEARCH/test_logsearch.py | 164 ++ .../python/stacks/2.4/LOGSEARCH/test_solr.py | 155 ++ .../test/python/stacks/2.4/configs/default.json | 398 ++++ .../src/test/python/stacks/utils/RMFTestCase.py | 14 +- .../HDP/2.2.0/upgrades/upgrade_test_checks.xml | 1 + .../app/mappers/service_metrics_mapper.js | 3 +- ambari-web/app/models/quick_links.js | 13 + .../app/views/common/quick_view_link_view.js | 6 +- .../test/views/common/quick_link_view_test.js | 5 + 63 files changed, 8417 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/4a2dcfbe/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py b/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py index df46ce0..e164f56 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py @@ -46,6 +46,10 @@ def should_install_ams_grafana(): config = Script.get_config() return 'role' in config and config['role'] == "METRICS_GRAFANA" +def should_install_logsearch_solr(): + config = Script.get_config() + return 'role' in config and config['role'] != "LOGSEARCH_LOGFEEDER" + def should_install_mysql(): config = Script.get_config() hive_database = config['configurations']['hive-env']['hive_database'] http://git-wip-us.apache.org/repos/asf/ambari/blob/4a2dcfbe/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/alerts.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/alerts.json b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/alerts.json new file mode 100644 index 0000000..32bbf67 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/alerts.json @@ -0,0 +1,63 @@ +{ + "LOGSEARCH": { + "LOGSEARCH_SERVER": [ + { + "name": "logsearch_ui", + "label": "LogSearch Web UI", + "description": "This host-level alert is triggered if the LogSearch UI is unreachable.", + "interval": 1, + "scope": "ANY", + "source": { + "type": "WEB", + "uri": { + "http": "{{logsearch-site/logsearch.ui.port}}", + "https": "{{logsearch-site/logsearch.ui.port}}", + "connection_timeout": 5.0, + "default_port": 61888 + }, + "reporting": { + "ok": { + "text": "HTTP {0} response in {2:.3f}s" + }, + "warning":{ + "text": "HTTP {0} response from {1} in {2:.3f}s ({3})" + }, + "critical": { + "text": "Connection failed to {1} ({3})" + } + } + } + } + ], + "LOGSEARCH_SOLR": [ + { + "name": "logsearch_solr", + "label": "Solr Web UI", + "description": "This host-level alert is triggered if the Solr Cloud Instance is unreachable.", + "interval": 1, + "scope": "ANY", + "source": { + "type": "WEB", + "uri": { + "http": "{{logsearch-solr-env/logsearch_solr_port}}", + "https": "{{logsearch-solr-env/logsearch_solr_port}}", + "connection_timeout": 5.0, + "default_port": 8886 + }, + "reporting": { + "ok": { + "text": "HTTP {0} response in {2:.3f}s" + }, + "warning":{ + "text": "HTTP {0} response from {1} in {2:.3f}s ({3})" + }, + "critical": { + "text": "Connection failed to {1} ({3})" + } + } + } + } + ] + + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/4a2dcfbe/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-env.xml b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-env.xml new file mode 100644 index 0000000..2722be8 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-env.xml @@ -0,0 +1,118 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> + + +<!-- This is a special config file for properties used to monitor status of the service --> +<configuration supports_adding_forbidden="true"> + + <property> + <name>logfeeder_pid_dir</name> + <value>/var/run/ambari-logsearch-logfeeder</value> + <description>logfeeder Process ID Directory</description> + <display-name>Logfeeder pid dir</display-name> + </property> + + <property> + <name>logfeeder_log_dir</name> + <value>/var/log/ambari-logsearch-logfeeder</value> + <description>Log dir for logfeeder</description> + <display-name>Logfeeder log dir</display-name> + </property> + + <property> + <name>logfeeder_checkpoint_folder</name> + <value>/etc/ambari-logsearch-logfeeder/conf/checkpoints</value> + <description>Checkpoint folder for logfeeder</description> + <display-name>Logfeeder checkpoint dir</display-name> + </property> + + <property> + <name>logfeeder_config_files</name> + <value>{{logfeeder_config_files}}</value> + <description>Comma separated config files in grok format</description> + <display-name>Logfeeder config files</display-name> + </property> + + <property> + <name>logfeeder_user</name> + <value>logfeeder</value> + <property-type>USER</property-type> + <description>logfeeder user</description> + <display-name>Logsfeeder User</display-name> + </property> + + <property> + <name>logfeeder_group</name> + <value>logfeeder</value> + <property-type>GROUP</property-type> + <description>logfeeder group</description> + <display-name>Logsfeeder Group</display-name> + </property> + + <property> + <name>logfeeder_max_mem</name> + <value>512m</value> + <description>Max memory for LogFeeder</description> + <display-name>Logsfeeder max memory</display-name> + </property> + + <property> + <name>content</name> + <description>This is the jinja template for config.json file</description> + <value>#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +export LOGFEEDER_PATH={{logfeeder_dir}} + +export LOGFEEDER_CONF_DIR={{logsearch_logfeeder_conf}} + +#Logfile e.g. /var/log/logfeeder.log +export LOGFILE={{logfeeder_log}} + +#pid file e.g. /var/run/logfeeder.pid +export PID_FILE={{logfeeder_pid_file}} + +export JAVA_HOME={{java64_home}} + +if [ "$LOGFEEDER_JAVA_MEM" = "" ]; then + export LOGFEEDER_JAVA_MEM=-Xmx{{logfeeder_max_mem}} +fi + </value> + </property> + + +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/4a2dcfbe/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-log4j.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-log4j.xml b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-log4j.xml new file mode 100644 index 0000000..5a3331e --- /dev/null +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-log4j.xml @@ -0,0 +1,80 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<!-- This is a special config file for properties used to monitor status of the service --> +<configuration supports_adding_forbidden="true"> + + <!-- solrconfig.xml --> + + <property> + <name>content</name> + <description>This is the jinja template for solrconfig.xml file for service logs</description> + <value><?xml version="1.0" encoding="UTF-8" ?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd"> +<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/"> + <appender name="console" class="org.apache.log4j.ConsoleAppender"> + <param name="Target" value="System.out" /> + <layout class="org.apache.log4j.PatternLayout"> + <param name="ConversionPattern" value="%d [%t] %-5p %C{6} (%F:%L) - %m%n" /> + </layout> + </appender> + + <appender name="rolling_file" class="org.apache.log4j.RollingFileAppender"> + <param name="file" value="{{logfeeder_log_dir}}/logfeeder.log" /> + <param name="append" value="true" /> + <param name="maxFileSize" value="10MB" /> + <param name="maxBackupIndex" value="10" /> + <layout class="org.apache.log4j.PatternLayout"> + <param name="ConversionPattern" value="%d [%t] %-5p %C{6} (%F:%L) - %m%n"/> + </layout> + </appender> + + <category name="org.apache.ambari.logfeeder" additivity="false"> + <priority value="info" /> + <appender-ref ref="rolling_file" /> + </category> + + <root> + <priority value="warn" /> + <appender-ref ref="rolling_file" /> + </root> +</log4j:configuration> + </value> + </property> + + +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/4a2dcfbe/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-site.xml b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-site.xml new file mode 100644 index 0000000..40ee374 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-site.xml @@ -0,0 +1,87 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration> + + <property> + <name>logfeeder.solr.service.logs.enable</name> + <value>true</value> + <display-name>Service Logs enabled</display-name> + <description>Enable service logs to Solr</description> + </property> + + <property> + <name>logfeeder.solr.audit.logs.enable</name> + <value>true</value> + <display-name>Audit Logs enabled</display-name> + <description>Enable audit logs to Solr</description> + </property> + + <property> + <name>logfeeder.kafka.service.logs.enable</name> + <value>false</value> + <display-name>Kafka Service Logs enabled</display-name> + <description>Enable service logs to Kafka</description> + </property> + + <property> + <name>logfeeder.kafka.topic.service.logs</name> + <value>service_logs</value> + <display-name>Kafka Service Logs topic</display-name> + <description>Kafka topic for service logs</description> + </property> + + <property> + <name>logfeeder.kafka.audit.logs.enable</name> + <value>false</value> + <display-name>Audit Logs enabled</display-name> + <description>Enable audit logs to Kafka</description> + </property> + + <property> + <name>logfeeder.kafka.topic.audit.logs</name> + <value>audit_logs</value> + <display-name>Kafka Audit Logs topic</display-name> + <description>Kafka topic for audit logs</description> + </property> + + <property> + <name>logfeeder.kafka.broker.list</name> + <value>NONE</value> + <display-name>Kafka broker list</display-name> + <description>List of brokers. e.g. broker1_host:port,broker2_host:port</description> + </property> + + <property> + <name>logfeeder.kafka.security.protocol</name> + <value>NONE</value> + <display-name>Kafka security protocol</display-name> + <description>If security is enabled, then set this value. E.g. SASL_PLAINTEXT</description> + </property> + + <property> + <name>logfeeder.kafka.kerberos.service.name</name> + <value>NONE</value> + <display-name>Kafka Kerberos service name</display-name> + <description>If security is enabled, then set this value. E.g. kafka</description> + </property> + +</configuration>
