http://git-wip-us.apache.org/repos/asf/ambari/blob/4a2dcfbe/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-solr-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-solr-env.xml b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-solr-env.xml new file mode 100644 index 0000000..de67420 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-solr-env.xml @@ -0,0 +1,193 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<!-- This is a special config file for properties used to monitor status of the service --> +<configuration supports_adding_forbidden="true"> + + <property> + <name>logsearch_solr_port</name> + <value>8886</value> + <description>Solr port</description> + <display-name>Logsearch Solr port</display-name> + </property> + + <property> + <name>logsearch_solr_pid_dir</name> + <value>/var/run/ambari-logsearch-solr</value> + <description>Solr Process ID Directory</description> + <display-name>Logsearch Solr pid dir</display-name> + </property> + + <property> + <name>logsearch_solr_log_dir</name> + <value>/var/log/ambari-logsearch-solr</value> + <description>Directory for Solr logs</description> + <display-name>Logsearch Solr log dir</display-name> + </property> + + <property> + <name>logsearch_solr_user</name> + <value>solr</value> + <property-type>USER</property-type> + <description>Solr user</description> + <display-name>Logsearch Solr User</display-name> + <value-attributes> + <type>user</type> + <overridable>false</overridable> + </value-attributes> + </property> + + <property> + <name>logsearch_solr_group</name> + <value>solr</value> + <property-type>GROUP</property-type> + <description>Solr group</description> + <display-name>Logsearch Solr Group</display-name> + </property> + + <property> + <name>logsearch_solr_datadir</name> + <value>/opt/logsearch_solr/data</value> + <display-name>Logsearch Solr data dir</display-name> + <description>Directory for storting Solr index. Make sure you have enough disk space</description> + </property> + + <property> + <name>logsearch_solr_znode</name> + <value>/logsearch</value> + <description>Zookeeper znode</description> + <display-name>Logsearch Solr ZNode</display-name> + </property> + + <property> + <name>logsearch_solr_minmem</name> + <value>512m</value> + <display-name>Logsearch minimum heap size</display-name> + <description>Solr minimum heap size e.g.512m </description> + </property> + + <property> + <name>logsearch_solr_maxmem</name> + <value>512m</value> + <description>Solr maximum heap size e.g. 512m</description> + <display-name>Logsearch maximum heap size</display-name> + </property> + + + <!-- logsearch-solr-env.sh --> + + <property> + <name>content</name> + <description>This is the jinja template for logsearch-solr-env.sh file</description> + <value> +# By default the script will use JAVA_HOME to determine which java +# to use, but you can set a specific path for Solr to use without +# affecting other Java applications on your server/workstation. +SOLR_JAVA_HOME={{java64_home}} + +# Increase Java Min/Max Heap as needed to support your indexing / query needs +SOLR_JAVA_MEM="-Xms{{logsearch_solr_min_mem}} -Xmx{{logsearch_solr_max_mem}}" + +# Enable verbose GC logging +GC_LOG_OPTS="-verbose:gc -XX:+PrintHeapAtGC -XX:+PrintGCDetails \ +-XX:+PrintGCDateStamps -XX:+PrintGCTimeStamps -XX:+PrintTenuringDistribution -XX:+PrintGCApplicationStoppedTime" + +# These GC settings have shown to work well for a number of common Solr workloads +GC_TUNE="-XX:NewRatio=3 \ +-XX:SurvivorRatio=4 \ +-XX:TargetSurvivorRatio=90 \ +-XX:MaxTenuringThreshold=8 \ +-XX:+UseConcMarkSweepGC \ +-XX:+UseParNewGC \ +-XX:ConcGCThreads=4 -XX:ParallelGCThreads=4 \ +-XX:+CMSScavengeBeforeRemark \ +-XX:PretenureSizeThreshold=64m \ +-XX:+UseCMSInitiatingOccupancyOnly \ +-XX:CMSInitiatingOccupancyFraction=50 \ +-XX:CMSMaxAbortablePrecleanTime=6000 \ +-XX:+CMSParallelRemarkEnabled \ +-XX:+ParallelRefProcEnabled" + +# Set the ZooKeeper connection string if using an external ZooKeeper ensemble +# e.g. host1:2181,host2:2181/chroot +# Leave empty if not using SolrCloud +ZK_HOST="{{zookeeper_quorum}}{{logsearch_solr_znode}}" + +# Set the ZooKeeper client timeout (for SolrCloud mode) +ZK_CLIENT_TIMEOUT="60000" + +# By default the start script uses "localhost"; override the hostname here +# for production SolrCloud environments to control the hostname exposed to cluster state +#SOLR_HOST="192.168.1.1" + +# By default the start script uses UTC; override the timezone if needed +#SOLR_TIMEZONE="UTC" + +# Set to true to activate the JMX RMI connector to allow remote JMX client applications +# to monitor the JVM hosting Solr; set to "false" to disable that behavior +# (false is recommended in production environments) +ENABLE_REMOTE_JMX_OPTS="false" + +# The script will use SOLR_PORT+10000 for the RMI_PORT or you can set it here +# RMI_PORT=18983 + +# Anything you add to the SOLR_OPTS variable will be included in the java +# start command line as-is, in ADDITION to other options. If you specify the +# -a option on start script, those options will be appended as well. Examples: +#SOLR_OPTS="$SOLR_OPTS -Dsolr.autoSoftCommit.maxTime=3000" +#SOLR_OPTS="$SOLR_OPTS -Dsolr.autoCommit.maxTime=60000" +#SOLR_OPTS="$SOLR_OPTS -Dsolr.clustering.enabled=true" + +# Location where the bin/solr script will save PID files for running instances +# If not set, the script will create PID files in $SOLR_TIP/bin +SOLR_PID_DIR={{logsearch_solr_piddir}} + +# Path to a directory where Solr creates index files, the specified directory +# must contain a solr.xml; by default, Solr will use server/solr +SOLR_HOME={{logsearch_solr_datadir}} + +# Solr provides a default Log4J configuration properties file in server/resources +# however, you may want to customize the log settings and file appender location +# so you can point the script to use a different log4j.properties file +LOG4J_PROPS={{logsearch_solr_conf}}/log4j.properties + +# Location where Solr should write logs to; should agree with the file appender +# settings in server/resources/log4j.properties +SOLR_LOGS_DIR={{logsearch_solr_log_dir}} + +# Sets the port Solr binds to, default is 8983 +SOLR_PORT={{logsearch_solr_port}} + +# Uncomment to set SSL-related system properties +# Be sure to update the paths to the correct keystore for your environment +#SOLR_SSL_OPTS="-Djavax.net.ssl.keyStore=etc/solr-ssl.keystore.jks \ +#-Djavax.net.ssl.keyStorePassword=secret \ +#-Djavax.net.ssl.trustStore=etc/solr-ssl.keystore.jks \ +#-Djavax.net.ssl.trustStorePassword=secret" + +# Uncomment to set a specific SSL port (-Djetty.ssl.port=N); if not set +# and you are using SSL, then the start script will use SOLR_PORT for the SSL port +#SOLR_SSL_PORT= + </value> + </property> + + +</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/4a2dcfbe/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-solr-log4j.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-solr-log4j.xml b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-solr-log4j.xml new file mode 100644 index 0000000..1f19c8d --- /dev/null +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-solr-log4j.xml @@ -0,0 +1,57 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration supports_adding_forbidden="true"> + + <property> + <name>content</name> + <description>This is the jinja template for log4j.properties</description> + <value> +# Logging level +solr.log={{logsearch_solr_log_dir}} +#log4j.rootLogger=INFO, file, CONSOLE +log4j.rootLogger=INFO, file + +log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender + +log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout +log4j.appender.CONSOLE.layout.ConversionPattern=%-4r [%t] %-5p %c %x [%X{collection} %X{shard} %X{replica} %X{core}] \u2013 %m%n + +#- size rotation with log cleanup. +log4j.appender.file=org.apache.log4j.RollingFileAppender +log4j.appender.file.MaxFileSize=10MB +log4j.appender.file.MaxBackupIndex=9 + +#- File to log to and log format +log4j.appender.file.File=${solr.log}/solr.log +log4j.appender.file.layout=org.apache.log4j.PatternLayout +log4j.appender.file.layout.ConversionPattern=%d{ISO8601} [%t] %-5p [%X{collection} %X{shard} %X{replica} %X{core}] %C (%F:%L) - %m%n + +log4j.logger.org.apache.zookeeper=WARN +log4j.logger.org.apache.hadoop=WARN + +# set to INFO to enable infostream log messages +log4j.logger.org.apache.solr.update.LoggingInfoStream=OFF + </value> + </property> + + +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/4a2dcfbe/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-solr-xml.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-solr-xml.xml b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-solr-xml.xml new file mode 100644 index 0000000..01b32b5 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-solr-xml.xml @@ -0,0 +1,41 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration supports_adding_forbidden="true"> + + <property> + <name>content</name> + <description>This is the jinja template for logsearch solr.xml file</description> + <value> +<solr> + <solrcloud> + <str name="host">${host:}</str> + <int name="hostPort">${jetty.port:}</int> + <str name="hostContext">${hostContext:solr}</str> + <int name="zkClientTimeout">${zkClientTimeout:15000}</int> + <bool name="genericCoreNodeNames">${genericCoreNodeNames:true}</bool> + </solrcloud> +</solr> + </value> + </property> + + +</configuration> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/4a2dcfbe/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/kerberos.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/kerberos.json b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/kerberos.json new file mode 100644 index 0000000..6dd4aa7 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/kerberos.json @@ -0,0 +1,17 @@ +{ + "services": [ + { + "name": "LOGSEARCH", + "identities": [ + { + "name": "/smokeuser" + } + ], + "components": [ + { + "name": "LOGSEARCH_SERVER" + } + ] + } + ] +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/4a2dcfbe/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml new file mode 100644 index 0000000..9bf486b --- /dev/null +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml @@ -0,0 +1,119 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<metainfo> + <schemaVersion>2.0</schemaVersion> + <services> + <service> + <name>LOGSEARCH</name> + <displayName>Log Search</displayName> + <comment>Log aggregation, analysis, and visualization for Ambari managed services</comment> + <version>0.5.0</version> + + <components> + + <component> + <name>LOGSEARCH_SERVER</name> + <timelineAppid>logsearch</timelineAppid> + <displayName>Log Search Server</displayName> + <category>MASTER</category> + <cardinality>1</cardinality> + <commandScript> + <script>scripts/logsearch.py</script> + <scriptType>PYTHON</scriptType> + </commandScript> + <logs> + <log> + <logId>logsearch_app</logId> + </log> + <log> + <logId>logsearch_perf</logId> + </log> + </logs> + </component> + + <component> + <name>LOGSEARCH_SOLR</name> + <displayName>Solr Instance</displayName> + <category>MASTER</category> + <cardinality>1+</cardinality> + <commandScript> + <script>scripts/solr.py</script> + <scriptType>PYTHON</scriptType> + </commandScript> + </component> + + <component> + <name>LOGSEARCH_LOGFEEDER</name> + <timelineAppid>logfeeder</timelineAppid> + <displayName>Log Feeder</displayName> + <category>SLAVE</category> + <cardinality>ALL</cardinality> + <commandScript> + <script>scripts/logfeeder.py</script> + <scriptType>PYTHON</scriptType> + </commandScript> + <logs> + <log> + <logId>logsearch_feeder</logId> + </log> + </logs> + </component> + + </components> + + <osSpecifics> + <osSpecific> + <osFamily>redhat6</osFamily> + <packages> + <package> + <name>ambari-logsearch-logfeeder</name> + <skipUpgrade>true</skipUpgrade> + </package> + <package> + <name>ambari-logsearch-portal</name> + <skipUpgrade>true</skipUpgrade> + </package> + <package> + <name>ambari-logsearch-solr</name> + <skipUpgrade>true</skipUpgrade> + <condition>should_install_logsearch_solr</condition> + </package> + </packages> + </osSpecific> + </osSpecifics> + + <requiredServices> + <service>ZOOKEEPER</service> + </requiredServices> + + <configuration-dependencies> + <config-type>logsearch-site</config-type> + <config-type>solr-site</config-type> + <config-type>logfeeder-site</config-type> + </configuration-dependencies> + <restartRequiredAfterChange>false</restartRequiredAfterChange> + + <quickLinksConfigurations> + <quickLinksConfiguration> + <fileName>quicklinks.json</fileName> + <default>true</default> + </quickLinksConfiguration> + </quickLinksConfigurations> + </service> + </services> +</metainfo> http://git-wip-us.apache.org/repos/asf/ambari/blob/4a2dcfbe/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logfeeder.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logfeeder.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logfeeder.py new file mode 100644 index 0000000..ce0155d --- /dev/null +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logfeeder.py @@ -0,0 +1,67 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management import * +from setup_logfeeder import setup_logfeeder + +class LogFeeder(Script): + + def install(self, env): + import params + env.set_params(params) + + self.install_packages(env) + + def configure(self, env, upgrade_type=None): + import params + env.set_params(params) + + setup_logfeeder() + + def start(self, env, upgrade_type=None): + import params + env.set_params(params) + self.configure(env) + + Execute(format("{logfeeder_dir}/run.sh"), + environment = {'LOGFEEDER_INCLUDE': format('{logsearch_logfeeder_conf}/logfeeder-env.sh')}, + user=params.logfeeder_user + ) + + def stop(self, env, upgrade_type=None): + import params + env.set_params(params) + + Execute (format("kill `cat {logfeeder_pid_file}`"), + user=params.logfeeder_user , + only_if = format("test -f {logfeeder_pid_file}") + ) + File(params.logfeeder_pid_file, + action="delete" + ) + + def status(self, env): + import status_params + env.set_params(status_params) + + check_process_status(status_params.logfeeder_pid_file) + + +if __name__ == "__main__": + LogFeeder().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/4a2dcfbe/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logsearch.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logsearch.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logsearch.py new file mode 100644 index 0000000..ac37788 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logsearch.py @@ -0,0 +1,65 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management import * +from setup_logsearch import setup_logsearch + +class LogSearch(Script): + + def install(self, env): + import params + env.set_params(params) + self.install_packages(env) + + def configure(self, env, upgrade_type=None): + import params + env.set_params(params) + + setup_logsearch() + + def start(self, env, upgrade_type=None): + import params + env.set_params(params) + self.configure(env) + + Execute(format("{logsearch_dir}/run.sh {logsearch_ui_port}"), + environment = {'LOGSEARCH_INCLUDE': format('{logsearch_server_conf}/logsearch-env.sh')}, + user=params.logsearch_user + ) + + def stop(self, env, upgrade_type=None): + import params + env.set_params(params) + + Execute (format('kill `cat {logsearch_pid_file}` >/dev/null 2>&1'), + user=params.logsearch_user, + only_if=format("test -f {logsearch_pid_file}") # TODO: check if process is up + ) + File(params.logsearch_pid_file, + action="delete" + ) + + def status(self, env): + import status_params + env.set_params(status_params) + + check_process_status(status_params.logsearch_pid_file) + +if __name__ == "__main__": + LogSearch().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/4a2dcfbe/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py new file mode 100644 index 0000000..ee39629 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py @@ -0,0 +1,231 @@ +#!/usr/bin/env python + +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" +from ambari_commons.constants import AMBARI_SUDO_BINARY +from resource_management import * +import os +from resource_management.libraries.functions.default import default +import status_params + +def get_port_from_url(address): + if not is_empty(address): + return address.split(':')[-1] + else: + return address + +# config object that holds the configurations declared in the -site.xml file +config = Script.get_config() +tmp_dir = Script.get_tmp_dir() + +stack_version = default("/commandParams/version", None) +sudo = AMBARI_SUDO_BINARY + +logsearch_solr_conf = "/etc/ambari-logsearch-solr/conf" +logsearch_server_conf = "/etc/ambari-logsearch-portal/conf" +logsearch_logfeeder_conf = "/etc/ambari-logsearch-logfeeder/conf" + +logsearch_solr_port = status_params.logsearch_solr_port +logsearch_solr_piddir = status_params.logsearch_solr_piddir +logsearch_solr_pidfile = status_params.logsearch_solr_pidfile + +#logsearch pid file +logsearch_pid_dir = status_params.logsearch_pid_dir +logsearch_pid_file = status_params.logsearch_pid_file + +#logfeeder pid file +logfeeder_pid_dir = status_params.logfeeder_pid_dir +logfeeder_pid_file = status_params.logfeeder_pid_file + +#shared configs +java64_home = config['hostLevelParams']['java_home'] +#get comma separated list of zookeeper hosts from clusterHostInfo +zookeeper_hosts = ",".join(config['clusterHostInfo']['zookeeper_hosts']) +cluster_name = str(config['clusterName']) + +#for now just pick first collector +if 'metrics_collector_hosts' in config['clusterHostInfo']: + metrics_collector_hosts_list = ",".join(config['clusterHostInfo']['metrics_collector_hosts']) + metrics_collector_port = str(get_port_from_url(config['configurations']['ams-site']['timeline.metrics.service.webapp.address'])) + metrics_collector_hosts = format('http://{metrics_collector_hosts_list}:{metrics_collector_port}/ws/v1/timeline/metrics') +else: + metrics_collector_hosts = '' + +##################################### +#Solr configs +##################################### + +# Only supporting SolrCloud mode - so hardcode those options +solr_cloudmode = 'true' +solr_dir = '/usr/lib/ambari-logsearch-solr' +solr_bindir = solr_dir + '/bin' +cloud_scripts = solr_dir +'/server/scripts/cloud-scripts' + +logsearch_solr_znode = config['configurations']['logsearch-solr-env']['logsearch_solr_znode'] +logsearch_solr_min_mem = format(config['configurations']['logsearch-solr-env']['logsearch_solr_minmem']) +logsearch_solr_max_mem = format(config['configurations']['logsearch-solr-env']['logsearch_solr_maxmem']) +logsearch_solr_instance_count = len(config['clusterHostInfo']['logsearch_solr_hosts']) +logsearch_solr_datadir = format(config['configurations']['logsearch-solr-env']['logsearch_solr_datadir']) +logsearch_solr_data_resources_dir = os.path.join(logsearch_solr_datadir,'resources') +logsearch_service_logs_max_retention = config['configurations']['logsearch-service_logs-solrconfig']['logsearch_service_logs_max_retention'] +logsearch_audit_logs_max_retention = config['configurations']['logsearch-audit_logs-solrconfig']['logsearch_audit_logs_max_retention'] + +logsearch_service_logs_fields = config['configurations']['logsearch-site']['logsearch.service.logs.fields'] + +logsearch_ui_port = config['configurations']['logsearch-site']["logsearch.ui.port"] +logsearch_debug_enabled = str(config['configurations']['logsearch-env']["logsearch_debug_enabled"]).lower() +logsearch_debug_port = config['configurations']['logsearch-env']["logsearch_debug_port"] +logsearch_app_max_memory = config['configurations']['logsearch-env']['logsearch_app_max_memory'] + +audit_logs_collection_splits_interval_mins = config['configurations']['logsearch-site']['logsearch.audit.logs.split.interval.mins'] +service_logs_collection_splits_interval_mins = config['configurations']['logsearch-site']['logsearch.service.logs.split.interval.mins'] + +zookeeper_port=default('/configurations/zoo.cfg/clientPort', None) +#get comma separated list of zookeeper hosts from clusterHostInfo +index = 0 +zookeeper_quorum = "" +for host in config['clusterHostInfo']['zookeeper_hosts']: + zookeeper_quorum += host + ":"+str(zookeeper_port) + index += 1 + if index < len(config['clusterHostInfo']['zookeeper_hosts']): + zookeeper_quorum += "," + +logsearch_solr_user = config['configurations']['logsearch-solr-env']['logsearch_solr_user'] +logsearch_solr_group = config['configurations']['logsearch-solr-env']['logsearch_solr_group'] +logsearch_solr_log_dir = config['configurations']['logsearch-solr-env']['logsearch_solr_log_dir'] +logsearch_solr_log = format("{logsearch_solr_log_dir}/solr-install.log") + +solr_env_content = config['configurations']['logsearch-solr-env']['content'] + +solr_xml_content = config['configurations']['logsearch-solr-xml']['content'] + +solr_log4j_content = config['configurations']['logsearch-solr-log4j']['content'] + + +##################################### +#Logsearch configs +##################################### +logsearch_dir = '/usr/lib/ambari-logsearch-portal' +logsearch_numshards_config = config['configurations']['logsearch-site']['logsearch.collection.numshards'] + +if logsearch_numshards_config > 0: + logsearch_numshards = str(logsearch_numshards_config) +else: + logsearch_numshards = format(str(logsearch_solr_instance_count)) + +logsearch_repfactor = str(config['configurations']['logsearch-site']['logsearch.collection.replication.factor']) + +logsearch_solr_collection_service_logs = default('/configurations/logsearch-site/logsearch.solr.collection.service.logs', 'hadoop_logs') +logsearch_solr_collection_audit_logs = default('/configurations/logsearch-site/logsearch.solr.collection.audit.logs', 'audit_logs') + +solr_audit_logs_use_ranger = default('/configurations/logsearch-env/logsearch_solr_audit_logs_use_ranger', False) +solr_audit_logs_url = '' + +if solr_audit_logs_use_ranger: + #In Ranger, this contain the /zkNode also + ranger_audit_solr_zookeepers = default('/configurations/ranger-admin-site/ranger.audit.solr.zookeepers', None) + #TODO: ranger property already has zk node appended. We need to remove it. + #For now, let's assume it is going to be URL + solr_audit_logs_url = default('/configurations/ranger-admin-site/ranger.audit.solr.urls', solr_audit_logs_url) +else: + solr_audit_logs_zk_node = default('/configurations/logsearch-env/logsearch_solr_audit_logs_zk_node', None) + solr_audit_logs_zk_quorum = default('/configurations/logsearch-env/logsearch_solr_audit_logs_zk_quorum', None) + + if not(solr_audit_logs_zk_quorum): + solr_audit_logs_zk_quorum=zookeeper_quorum + if not(solr_audit_logs_zk_node): + solr_audit_logs_zk_node=logsearch_solr_znode + + solr_audit_logs_zk_node = format(solr_audit_logs_zk_node) + solr_audit_logs_zk_quorum = format(solr_audit_logs_zk_quorum) + + +# logsearch-env configs +logsearch_user = config['configurations']['logsearch-env']['logsearch_user'] +logsearch_group = config['configurations']['logsearch-env']['logsearch_group'] +logsearch_log_dir = config['configurations']['logsearch-env']['logsearch_log_dir'] +logsearch_log = logsearch_log_dir + '/logsearch.out' + +# store the log file for the service from the 'solr.log' property of the 'logsearch-env.xml' file +logsearch_env_content = config['configurations']['logsearch-env']['content'] +logsearch_service_logs_solrconfig_content = config['configurations']['logsearch-service_logs-solrconfig']['content'] +logsearch_audit_logs_solrconfig_content = config['configurations']['logsearch-audit_logs-solrconfig']['content'] +logsearch_app_log4j_content = config['configurations']['logsearch-log4j']['content'] + +#Log dirs +ambari_server_log_dir = '/var/log/ambari-server' +ambari_agent_log_dir = '/var/log/ambari-agent' +knox_log_dir = '/var/log/knox' + +metrics_collector_log_dir = default('/configurations/ams-env/metrics_collector_log_dir', '/var/log') +metrics_monitor_log_dir = default('/configurations/ams-env/metrics_monitor_log_dir', '/var/log') + + +atlas_log_dir = default('/configurations/atlas-env/metadata_log_dir', '/var/log/atlas') +accumulo_log_dir = default('/configurations/accumulo-env/accumulo_log_dir', '/var/log/accumulo') +falcon_log_dir = default('/configurations/falcon-env/falcon_log_dir','/var/log/falcon') +hbase_log_dir = default('/configurations/hbase-env/hbase_log_dir','/var/log/hbase') +hdfs_log_dir_prefix = default('/configurations/hadoop-env/hdfs_log_dir_prefix','/var/log/hadoop') +hive_log_dir = default('/configurations/hive-env/hive_log_dir','/var/log/hive') +kafka_log_dir = default('/configurations/kafka-env/kafka_log_dir','/var/log/kafka') +oozie_log_dir = default('/configurations/oozie-env/oozie_log_dir','/var/log/oozie') +ranger_usersync_log_dir = default('/configurations/ranger-env/ranger_usersync_log_dir', '/var/log/ranger/usersync') +ranger_admin_log_dir = default('/configurations/ranger-env/ranger_admin_log_dir', '/var/log/ranger/admin') +ranger_kms_log_dir = default('/configurations/kms-env/kms_log_dir', '/var/log/ranger/kms') +storm_log_dir = default('/configurations/storm-env/storm_log_dir','/var/log/storm') +yarn_log_dir_prefix = default('/configurations/yarn-env/yarn_log_dir_prefix','/var/log/hadoop') +mapred_log_dir_prefix = default('/configurations/mapred-env/mapred_log_dir_prefix','/var/log/hadoop') +zk_log_dir = default('/configurations/zookeeper-env/zk_log_dir','/var/log/zookeeper') + + +##################################### +#Logfeeder configs +##################################### + +logfeeder_dir = "/usr/lib/ambari-logsearch-logfeeder" + +# logfeeder-site configs +solr_service_logs_enable = default('/configurations/logfeeder-site/logfeeder.solr.service.logs.enable',True) +solr_audit_logs_enable = default('/configurations/logfeeder-site/logfeeder.solr.audit.logs_enable',True) + +kafka_broker_list = default('/configurations/logfeeder-site/logfeeder.kafka.broker.list', '') +kafka_security_protocol= default('/configurations/logfeeder-site/logfeeder.kafka.security.protocol', '') +kafka_kerberos_service_name=default('/configurations/logfeeder-site/logfeeder.kafka.kerberos.service.name', '') + +kafka_service_logs_enable = default('/configurations/logfeeder-site/logfeeder.kafka.service.logs.enable',False) +kafka_audit_logs_enable = default('/configurations/logfeeder-site/logfeeder.kafka.audit.logs.enable',False) +kafka_topic_service_logs = default('/configurations/logfeeder-site/logfeeder.kafka.topic.service.logs','service_logs') +kafka_topic_audit_logs = default('/configurations/logfeeder-site/logfeeder.kafka.topic.audit.logs','audit_logs') + +# logfeeder-env configs +logfeeder_user = config['configurations']['logfeeder-env']['logfeeder_user'] +logfeeder_group = config['configurations']['logfeeder-env']['logfeeder_group'] +logfeeder_log_dir = config['configurations']['logfeeder-env']['logfeeder_log_dir'] +logfeeder_log = logfeeder_log_dir + '/logfeeder.out' +logfeeder_max_mem = config['configurations']['logfeeder-env']['logfeeder_max_mem'] +logfeeder_env_content = config['configurations']['logfeeder-env']['content'] +logfeeder_log4j_content = config['configurations']['logfeeder-log4j']['content'] + +logfeeder_checkpoint_folder = default('/configurations/logfeeder-env/logfeeder.checkpoint.folder', '/etc/ambari-logsearch-logfeeder/conf/checkpoints') +logfeeder_supported_services = ['accumulo', 'ambari', 'ams', 'atlas', 'falcon', 'hbase', 'hdfs', 'hive', 'kafka', + 'knox', 'logsearch', 'oozie', 'ranger', 'storm', 'yarn', 'zookeeper'] + +logfeeder_config_file_names = ['global.config.json', 'output.config.json'] + ['input.config-%s.json' % (tag) for tag in logfeeder_supported_services] + +logfeeder_config_files = ','.join(logfeeder_config_file_names) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/4a2dcfbe/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py new file mode 100644 index 0000000..0c1cb6d --- /dev/null +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py @@ -0,0 +1,67 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management import * + +def setup_logfeeder(): + import params + + Directory([params.logfeeder_log_dir, params.logfeeder_pid_dir, params.logfeeder_dir, + params.logsearch_logfeeder_conf, params.logfeeder_checkpoint_folder], + mode = 0755, + cd_access = 'a', + owner = params.logfeeder_user, + group = params.logfeeder_group, + create_parents = True + ) + + + File(params.logfeeder_log, + mode = 0644, + owner = params.logfeeder_user, + group = params.logfeeder_group, + content = '' + ) + + File(format("{logsearch_logfeeder_conf}/logfeeder.properties"), + content=Template("logfeeder.properties.j2"), + owner=params.logfeeder_user + ) + + File(format("{logsearch_logfeeder_conf}/logfeeder-env.sh"), + content = InlineTemplate(params.logfeeder_env_content), + owner = params.logfeeder_user + ) + + File(format("{logsearch_logfeeder_conf}/log4j.xml"), + content = InlineTemplate(params.logfeeder_log4j_content), + owner = params.logfeeder_user + ) + + File(format("{logsearch_logfeeder_conf}/grok-patterns"), + content = Template("grok-patterns.j2"), + owner = params.logfeeder_user, + encoding = "utf-8" + ) + + for file_name in params.logfeeder_config_file_names: + File(format("{logsearch_logfeeder_conf}/" + file_name), + content = Template(file_name + ".j2"), + owner = params.logfeeder_user + ) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/4a2dcfbe/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py new file mode 100644 index 0000000..7cab1d9 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py @@ -0,0 +1,93 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +import random +from resource_management import * +from resource_management.libraries.functions.decorator import retry + +def setup_logsearch(): + import params + + Directory([params.logsearch_log_dir, params.logsearch_pid_dir, params.logsearch_dir, params.logsearch_server_conf], + mode=0755, + cd_access='a', + owner=params.logsearch_user, + group=params.logsearch_group, + create_parents=True + ) + + File(params.logsearch_log, + mode=0644, + owner=params.logsearch_user, + group=params.logsearch_group, + content='' + ) + + File(format("{logsearch_server_conf}/logsearch.properties"), + content=Template("logsearch.properties.j2"), + owner=params.logsearch_user + ) + + File(format("{logsearch_server_conf}/log4j.xml"), + content=InlineTemplate(params.logsearch_app_log4j_content), + owner=params.logsearch_user + ) + + File(format("{logsearch_server_conf}/logsearch-env.sh"), + content=InlineTemplate(params.logsearch_env_content), + mode=0755, + owner=params.logsearch_user + ) + + File(format("{logsearch_server_conf}/solr_configsets/hadoop_logs/conf/solrconfig.xml"), + content=InlineTemplate(params.logsearch_service_logs_solrconfig_content), + owner=params.logsearch_user + ) + + File(format("{logsearch_server_conf}/solr_configsets/audit_logs/conf/solrconfig.xml"), + content=InlineTemplate(params.logsearch_audit_logs_solrconfig_content), + owner=params.logsearch_user + ) + + random_num = random.random() + + upload_configuration_dir_to_zk('hadoop_logs', random_num) + + upload_configuration_dir_to_zk('history', random_num) + + upload_configuration_dir_to_zk('audit_logs', random_num) + + Execute(("chmod", "-R", "ugo+r", format("{logsearch_server_conf}/solr_configsets")), + sudo=True + ) + +def upload_configuration_dir_to_zk(config_set, random_num): + import params + + tmp_config_set_folder=format('{tmp_dir}/solr_config_{config_set}_{random_num}') + zk_cli_prefix = format('export JAVA_HOME={java64_home} ; {cloud_scripts}/zkcli.sh -zkhost {zookeeper_quorum}{logsearch_solr_znode}') + + Execute(format('{zk_cli_prefix} -cmd downconfig -confdir {tmp_config_set_folder} -confname {config_set}'), + only_if = format("{zk_cli_prefix} -cmd get /configs/{config_set}"), + ) + + Execute(format('{zk_cli_prefix} -cmd upconfig -confdir {logsearch_server_conf}/solr_configsets/{config_set}/conf -confname {config_set}'), + not_if = format("test -d {tmp_config_set_folder}"), + ) + http://git-wip-us.apache.org/repos/asf/ambari/blob/4a2dcfbe/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_solr.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_solr.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_solr.py new file mode 100644 index 0000000..0192277 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_solr.py @@ -0,0 +1,67 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management import * + +def setup_solr(): + import params + + Directory([params.solr_dir, params.logsearch_solr_log_dir, params.logsearch_solr_piddir, params.logsearch_solr_conf, + params.logsearch_solr_datadir, params.logsearch_solr_data_resources_dir], + mode=0755, + cd_access='a', + owner=params.logsearch_solr_user, + group=params.logsearch_solr_group, + create_parents=True + ) + + File(params.logsearch_solr_log, + mode=0644, + owner=params.logsearch_solr_user, + group=params.logsearch_solr_group, + content='' + ) + + File(format("{logsearch_solr_conf}/logsearch-solr-env.sh"), + content=InlineTemplate(params.solr_env_content), + mode = 0755, + owner=params.logsearch_solr_user + ) + + File(format("{logsearch_solr_datadir}/solr.xml"), + content=InlineTemplate(params.solr_xml_content), + owner=params.logsearch_solr_user + ) + + File(format("{logsearch_solr_conf}/log4j.properties"), + content=InlineTemplate(params.solr_log4j_content), + owner=params.logsearch_solr_user + ) + + File(format("{logsearch_solr_datadir}/zoo.cfg"), + content=Template("zoo.cfg.j2"), + owner=params.logsearch_solr_user + ) + + zk_cli_prefix = format('export JAVA_HOME={java64_home}; {cloud_scripts}/zkcli.sh -zkhost {zookeeper_hosts}') + Execute (format('{zk_cli_prefix} -cmd makepath {logsearch_solr_znode}'), + not_if = format("{zk_cli_prefix} -cmd get /logsearch"), + ignore_failures = True, + user=params.logsearch_solr_user + ) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/4a2dcfbe/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/solr.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/solr.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/solr.py new file mode 100644 index 0000000..545505a --- /dev/null +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/solr.py @@ -0,0 +1,66 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management import * +from setup_solr import setup_solr + +class Solr(Script): + + def install(self, env): + import params + env.set_params(params) + self.install_packages(env) + + def configure(self, env, upgrade_type=None): + import params + env.set_params(params) + + setup_solr() + + def start(self, env, upgrade_type=None): + import params + env.set_params(params) + self.configure(env) + + Execute(format('{solr_bindir}/solr start -cloud -noprompt -s {logsearch_solr_datadir} >> {logsearch_solr_log} 2>&1'), + environment = {'SOLR_INCLUDE': format('{logsearch_solr_conf}/logsearch-solr-env.sh')}, + user=params.logsearch_solr_user + ) + + def stop(self, env, upgrade_type=None): + import params + env.set_params(params) + + Execute (format('{solr_bindir}/solr stop -all >> {logsearch_solr_log}'), + environment = {'SOLR_INCLUDE': format('{logsearch_solr_conf}/logsearch-solr-env.sh')}, + user=params.logsearch_solr_user, + only_if = format("test -f {logsearch_solr_pidfile}") + ) + File(params.logsearch_solr_pidfile, + action="delete" + ) + + def status(self, env): + import status_params + env.set_params(status_params) + + check_process_status(status_params.logsearch_solr_pidfile) + +if __name__ == "__main__": + Solr().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/4a2dcfbe/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/status_params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/status_params.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/status_params.py new file mode 100644 index 0000000..3a293fd --- /dev/null +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/status_params.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python + +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management import * + +config = Script.get_config() + +logsearch_solr_port = config['configurations']['logsearch-solr-env']['logsearch_solr_port'] +logsearch_solr_piddir = config['configurations']['logsearch-solr-env']['logsearch_solr_pid_dir'] +logsearch_solr_pidfile = format("{logsearch_solr_piddir}/solr-{logsearch_solr_port}.pid") + +#logsearch pid file +logsearch_pid_dir = config['configurations']['logsearch-env']['logsearch_pid_dir'] +logsearch_pid_file = format("{logsearch_pid_dir}/logsearch.pid") + +#logfeeder pid file +logfeeder_pid_dir = config['configurations']['logfeeder-env']['logfeeder_pid_dir'] +logfeeder_pid_file = format("{logfeeder_pid_dir}/logfeeder.pid") http://git-wip-us.apache.org/repos/asf/ambari/blob/4a2dcfbe/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/global.config.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/global.config.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/global.config.json.j2 new file mode 100644 index 0000000..f337527 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/global.config.json.j2 @@ -0,0 +1,28 @@ +{# + # Licensed to the Apache Software Foundation (ASF) under one + # or more contributor license agreements. See the NOTICE file + # distributed with this work for additional information + # regarding copyright ownership. The ASF licenses this file + # to you under the Apache License, Version 2.0 (the + # "License"); you may not use this file except in compliance + # with the License. You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + #} +{ + "global":{ + "add_fields":{ + "cluster":"{{cluster_name}}" + }, + "source":"file", + "tail":"true", + "gen_event_md5":"true", + "start_position":"beginning" + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/4a2dcfbe/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/grok-patterns.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/grok-patterns.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/grok-patterns.j2 new file mode 100644 index 0000000..c85a06a --- /dev/null +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/grok-patterns.j2 @@ -0,0 +1,144 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#Updated JAVACLASS to be same as JAVAFILE. Because if class doesn't have package, then it doesn't work. +JAVACLASS (?:[A-Za-z$0-9_. -]+) +#JAVACLASS (?:[a-zA-Z$_][a-zA-Z$_0-9]*\.)*[a-zA-Z$_][a-zA-Z$_0-9]* +#JAVACLASS (?:[a-zA-Z0-9-]+\.)+[A-Za-z0-9$]+ + +#Space is an allowed character to match special cases like 'Native Method' or 'Unknown Source' +JAVAFILE (?:[A-Za-z0-9_. -]+) +#Allow special <init> or <clinit> method +JAVAMETHOD (?:(<init>)|(<clinit>)|[a-zA-Z$_][a-zA-Z$_0-9]*) +#Line number is optional in special cases 'Native method' or 'Unknown source' +JAVASTACKTRACEPART %{SPACE}at %{JAVACLASS:class}\.%{JAVAMETHOD:method}\(%{JAVAFILE:file}(?::%{NUMBER:line})?\) +# Java Logs +JAVATHREAD (?:[A-Z]{2}-Processor[\d]+) + +JAVASTACKTRACEPART at %{JAVACLASS:class}\.%{WORD:method}\(%{JAVAFILE:file}:%{NUMBER:line}\) +JAVALOGMESSAGE (.*) +# MMM dd, yyyy HH:mm:ss eg: Jan 9, 2014 7:13:13 AM +CATALINA_DATESTAMP %{MONTH} %{MONTHDAY}, 20%{YEAR} %{HOUR}:?%{MINUTE}(?::?%{SECOND}) (?:AM|PM) +# yyyy-MM-dd HH:mm:ss,SSS ZZZ eg: 2014-01-09 17:32:25,527 -0800 +TOMCAT_DATESTAMP 20%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{HOUR}:?%{MINUTE}(?::?%{SECOND}) %{ISO8601_TIMEZONE} +CATALINALOG %{CATALINA_DATESTAMP:timestamp} %{JAVACLASS:class} %{JAVALOGMESSAGE:logmessage} +# 2014-01-09 20:03:28,269 -0800 | ERROR | com.example.service.ExampleService - something compeletely unexpected happened... +TOMCATLOG %{TOMCAT_DATESTAMP:timestamp} \| %{LOGLEVEL:level} \| %{JAVACLASS:class} - %{JAVALOGMESSAGE:logmessage} + +USERNAME [a-zA-Z0-9._-]+ +USER %{USERNAME} +EMAILLOCALPART [a-zA-Z][a-zA-Z0-9_.+-=:]+ +EMAILADDRESS %{EMAILLOCALPART}@%{HOSTNAME} +HTTPDUSER %{EMAILADDRESS}|%{USER} +INT (?:[+-]?(?:[0-9]+)) +BASE10NUM (?<![0-9.+-])(?>[+-]?(?:(?:[0-9]+(?:\.[0-9]+)?)|(?:\.[0-9]+))) +NUMBER (?:%{BASE10NUM}) +BASE16NUM (?<![0-9A-Fa-f])(?:[+-]?(?:0x)?(?:[0-9A-Fa-f]+)) +BASE16FLOAT \b(?<![0-9A-Fa-f.])(?:[+-]?(?:0x)?(?:(?:[0-9A-Fa-f]+(?:\.[0-9A-Fa-f]*)?)|(?:\.[0-9A-Fa-f]+)))\b + +POSINT \b(?:[1-9][0-9]*)\b +NONNEGINT \b(?:[0-9]+)\b +WORD \b\w+\b +NOTSPACE \S+ +SPACE \s* +DATA .*? +GREEDYDATA .* +QUOTEDSTRING (?>(?<!\\)(?>"(?>\\.|[^\\"]+)+"|""|(?>'(?>\\.|[^\\']+)+')|''|(?>`(?>\\.|[^\\`]+)+`)|``)) +UUID [A-Fa-f0-9]{8}-(?:[A-Fa-f0-9]{4}-){3}[A-Fa-f0-9]{12} + +# Networking +MAC (?:%{CISCOMAC}|%{WINDOWSMAC}|%{COMMONMAC}) +CISCOMAC (?:(?:[A-Fa-f0-9]{4}\.){2}[A-Fa-f0-9]{4}) +WINDOWSMAC (?:(?:[A-Fa-f0-9]{2}-){5}[A-Fa-f0-9]{2}) +COMMONMAC (?:(?:[A-Fa-f0-9]{2}:){5}[A-Fa-f0-9]{2}) +IPV6 ((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5 ]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)? +IPV4 (?<![0-9])(?:(?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5])[.](?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5])[.](?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5])[.](?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5]))(?![0-9]) +IP (?:%{IPV6}|%{IPV4}) +HOSTNAME \b(?:[0-9A-Za-z][0-9A-Za-z-]{0,62})(?:\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\.?|\b) +IPORHOST (?:%{IP}|%{HOSTNAME}) +HOSTPORT %{IPORHOST}:%{POSINT} + +# paths +PATH (?:%{UNIXPATH}|%{WINPATH}) +UNIXPATH (/([\w_%!$@:.,~-]+|\\.)*)+ +TTY (?:/dev/(pts|tty([pq])?)(\w+)?/?(?:[0-9]+)) +WINPATH (?>[A-Za-z]+:|\\)(?:\\[^\\?*]*)+ +URIPROTO [A-Za-z]+(\+[A-Za-z+]+)? +URIHOST %{IPORHOST}(?::%{POSINT:port})? +# uripath comes loosely from RFC1738, but mostly from what Firefox +# doesn't turn into %XX +URIPATH (?:/[A-Za-z0-9$.+!*'(){},~:;=@#%_\-]*)+ +#URIPARAM \?(?:[A-Za-z0-9]+(?:=(?:[^&]*))?(?:&(?:[A-Za-z0-9]+(?:=(?:[^&]*))?)?)*)? +URIPARAM \?[A-Za-z0-9$.+!*'|(){},~@#%&/=:;_?\-\[\]<>]* +URIPATHPARAM %{URIPATH}(?:%{URIPARAM})? +URI %{URIPROTO}://(?:%{USER}(?::[^@]*)?@)?(?:%{URIHOST})?(?:%{URIPATHPARAM})? + +# Months: January, Feb, 3, 03, 12, December +MONTH \b(?:Jan(?:uary|uar)?|Feb(?:ruary|ruar)?|M(?:a|ä)?r(?:ch|z)?|Apr(?:il)?|Ma(?:y|i)?|Jun(?:e|i)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|O(?:c|k)?t(?:ober)?|Nov(?:ember)?|De(?:c|z)(?:ember)?)\b +MONTHNUM (?:0?[1-9]|1[0-2]) +MONTHNUM2 (?:0[1-9]|1[0-2]) +MONTHDAY (?:(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9]) + +# Days: Monday, Tue, Thu, etc... +DAY (?:Mon(?:day)?|Tue(?:sday)?|Wed(?:nesday)?|Thu(?:rsday)?|Fri(?:day)?|Sat(?:urday)?|Sun(?:day)?) + +# Years? +YEAR (?>\d\d){1,2} +HOUR (?:2[0123]|[01]?[0-9]) +MINUTE (?:[0-5][0-9]) +# '60' is a leap second in most time standards and thus is valid. +SECOND (?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?) +TIME (?!<[0-9])%{HOUR}:%{MINUTE}(?::%{SECOND})(?![0-9]) +# datestamp is YYYY/MM/DD-HH:MM:SS.UUUU (or something like it) +DATE_US %{MONTHNUM}[/-]%{MONTHDAY}[/-]%{YEAR} +DATE_EU %{MONTHDAY}[./-]%{MONTHNUM}[./-]%{YEAR} +ISO8601_TIMEZONE (?:Z|[+-]%{HOUR}(?::?%{MINUTE})) +ISO8601_SECOND (?:%{SECOND}|60) +TIMESTAMP_ISO8601 %{YEAR}-%{MONTHNUM}-%{MONTHDAY}[T ]%{HOUR}:?%{MINUTE}(?::?%{SECOND})?%{ISO8601_TIMEZONE}? +DATE %{DATE_US}|%{DATE_EU} +DATESTAMP %{DATE}[- ]%{TIME} +TZ (?:[PMCE][SD]T|UTC) +DATESTAMP_RFC822 %{DAY} %{MONTH} %{MONTHDAY} %{YEAR} %{TIME} %{TZ} +DATESTAMP_RFC2822 %{DAY}, %{MONTHDAY} %{MONTH} %{YEAR} %{TIME} %{ISO8601_TIMEZONE} +DATESTAMP_OTHER %{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{TZ} %{YEAR} +DATESTAMP_EVENTLOG %{YEAR}%{MONTHNUM2}%{MONTHDAY}%{HOUR}%{MINUTE}%{SECOND} +HTTPDERROR_DATE %{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR} + +# Syslog Dates: Month Day HH:MM:SS +SYSLOGTIMESTAMP %{MONTH} +%{MONTHDAY} %{TIME} +PROG [\x21-\x5a\x5c\x5e-\x7e]+ +SYSLOGPROG %{PROG:program}(?:\[%{POSINT:pid}\])? +SYSLOGHOST %{IPORHOST} +SYSLOGFACILITY <%{NONNEGINT:facility}.%{NONNEGINT:priority}> +HTTPDATE %{MONTHDAY}/%{MONTH}/%{YEAR}:%{TIME} %{INT} + +# Shortcuts +QS %{QUOTEDSTRING} + +# Log formats +SYSLOGBASE %{SYSLOGTIMESTAMP:timestamp} (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:logsource} %{SYSLOGPROG}: +COMMONAPACHELOG %{IPORHOST:clientip} %{HTTPDUSER:ident} %{USER:auth} \[%{HTTPDATE:timestamp}\] "(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})" %{NUMBER:response} (?:%{NUMBER:bytes}|-) +COMBINEDAPACHELOG %{COMMONAPACHELOG} %{QS:referrer} %{QS:agent} +HTTPD20_ERRORLOG \[%{HTTPDERROR_DATE:timestamp}\] \[%{LOGLEVEL:loglevel}\] (?:\[client %{IPORHOST:clientip}\] ){0,1}%{GREEDYDATA:errormsg} +HTTPD24_ERRORLOG \[%{HTTPDERROR_DATE:timestamp}\] \[%{WORD:module}:%{LOGLEVEL:loglevel}\] \[pid %{POSINT:pid}:tid %{NUMBER:tid}\]( \(%{POSINT:proxy_errorcode}\)%{DATA:proxy_errormessage}:)?( \[client %{IPORHOST:client}:%{POSINT:clientport}\])? %{DATA:errorcode}: %{GREEDYDATA:message} +HTTPD_ERRORLOG %{HTTPD20_ERRORLOG}|%{HTTPD24_ERRORLOG} + + +# Log Levels +LOGLEVEL ([Aa]lert|ALERT|[Tt]race|TRACE|[Dd]ebug|DEBUG|[Nn]otice|NOTICE|[Ii]nfo|INFO|[Ww]arn?(?:ing)?|WARN?(?:ING)?|[Ee]rr?(?:or)?|ERR?(?:OR)?|[Cc]rit?(?:ical)?|CRIT?(?:ICAL)?|[Ff]atal|FATAL|[Ss]evere|SEVERE|EMERG(?:ENCY)?|[Ee]merg(?:ency)?) + + +# Custom +USER_SYNC_DATE %{MONTHDAY} %{MONTH} %{YEAR} %{TIME} http://git-wip-us.apache.org/repos/asf/ambari/blob/4a2dcfbe/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-accumulo.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-accumulo.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-accumulo.json.j2 new file mode 100644 index 0000000..a6c7ab8 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-accumulo.json.j2 @@ -0,0 +1,105 @@ +{# + # Licensed to the Apache Software Foundation (ASF) under one + # or more contributor license agreements. See the NOTICE file + # distributed with this work for additional information + # regarding copyright ownership. The ASF licenses this file + # to you under the Apache License, Version 2.0 (the + # "License"); you may not use this file except in compliance + # with the License. You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + #} +{ + "input":[ + { + "type":"accumulo_gc", + "rowtype":"service", + "path":"{{accumulo_log_dir}}/gc_*.log" + }, + { + "type":"accumulo_master", + "rowtype":"service", + "path":"{{accumulo_log_dir}}/master_*.log" + }, + { + "type":"accumulo_monitor", + "rowtype":"service", + "path":"{{accumulo_log_dir}}/monitor_*.log" + }, + { + "type":"accumulo_tracer", + "rowtype":"service", + "path":"{{accumulo_log_dir}}/tracer_*.log" + }, + { + "type":"accumulo_tserver", + "rowtype":"service", + "path":"{{accumulo_log_dir}}/tserver_*.log" + } + + ], + "filter":[ + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "accumulo_master" + ] + + } + + }, + "log4j_format":"%d{ISO8601} [%-8c{2}] %-5p: %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}\\[%{JAVACLASS:logger_name}\\]%{SPACE}%{LOGLEVEL:level}:%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + + } + + } + + }, + { + "filter":"grok", + "comment":"This one has one extra space after LEVEL", + "conditions":{ + "fields":{ + "type":[ + "accumulo_gc", + "accumulo_monitor", + "accumulo_tracer", + "accumulo_tserver" + ] + + } + + }, + "log4j_format":"%d{ISO8601} [%-8c{2}] %-5p: %X{application} %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}\\[%{JAVACLASS:logger_name}\\]%{SPACE}%{LOGLEVEL:level}%{SPACE}:%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + + } + + } + + } + + ] + +} http://git-wip-us.apache.org/repos/asf/ambari/blob/4a2dcfbe/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-ambari.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-ambari.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-ambari.json.j2 new file mode 100644 index 0000000..273b880 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-ambari.json.j2 @@ -0,0 +1,93 @@ +{# + # Licensed to the Apache Software Foundation (ASF) under one + # or more contributor license agreements. See the NOTICE file + # distributed with this work for additional information + # regarding copyright ownership. The ASF licenses this file + # to you under the Apache License, Version 2.0 (the + # "License"); you may not use this file except in compliance + # with the License. You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + #} +{ + "input":[ + { + "type":"ambari_agent", + "rowtype":"service", + "path":"{{ambari_agent_log_dir}}/ambari-agent.log" + }, + { + "type":"ambari_server", + "rowtype":"service", + "path":"{{ambari_server_log_dir}}/ambari-server.log" + } + + ], + "filter":[ + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "ambari_agent" + ] + + } + + }, + "log4j_format":"", + "multiline_pattern":"^(%{LOGLEVEL:level} %{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{LOGLEVEL:level} %{TIMESTAMP_ISO8601:logtime} %{JAVAFILE:file}:%{INT:line_number} - %{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + + }, + "level":{ + "map_fieldvalue":{ + "pre_value":"WARNING", + "post_value":"WARN" + } + + } + + } + + }, + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "ambari_server" + ] + + } + + }, + "log4j_format":"%d{DATE} %5p [%t] %c{1}:%L - %m%n", + "multiline_pattern":"^(%{USER_SYNC_DATE:logtime})", + "message_pattern":"(?m)^%{USER_SYNC_DATE:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{JAVACLASS:logger_name}:%{INT:line_number}%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "date_pattern":"dd MMM yyyy HH:mm:ss" + } + + } + + } + + } + + ] + +} http://git-wip-us.apache.org/repos/asf/ambari/blob/4a2dcfbe/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-ams.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-ams.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-ams.json.j2 new file mode 100644 index 0000000..d2f3f1a --- /dev/null +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-ams.json.j2 @@ -0,0 +1,92 @@ +{# + # Licensed to the Apache Software Foundation (ASF) under one + # or more contributor license agreements. See the NOTICE file + # distributed with this work for additional information + # regarding copyright ownership. The ASF licenses this file + # to you under the Apache License, Version 2.0 (the + # "License"); you may not use this file except in compliance + # with the License. You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + #} +{ + "input":[ + { + "type":"ams_hbase_master", + "rowtype":"service", + "path":"{{metrics_collector_log_dir}}/hbase-ams-master-*.log" + }, + { + "type":"ams_hbase_regionserver", + "rowtype":"service", + "path":"{{metrics_collector_log_dir}}/hbase-ams-regionserver-*.log" + }, + { + "type":"ams_collector", + "rowtype":"service", + "path":"{{metrics_collector_log_dir}}/ambari-metrics-collector.log" + } + + ], + "filter":[ + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "ams_collector" + ] + + } + + }, + "log4j_format":"%d{ISO8601} %p %c: %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}:%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + + } + + } + + }, + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "ams_hbase_master", + "ams_hbase_regionserver" + ] + + } + + }, + "log4j_format":"%d{ISO8601} %-5p [%t] %c{2}: %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{JAVACLASS:logger_name}:%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + + } + + } + + } + + ] + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/4a2dcfbe/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-atlas.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-atlas.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-atlas.json.j2 new file mode 100644 index 0000000..84bc84c --- /dev/null +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-atlas.json.j2 @@ -0,0 +1,55 @@ +{# + # Licensed to the Apache Software Foundation (ASF) under one + # or more contributor license agreements. See the NOTICE file + # distributed with this work for additional information + # regarding copyright ownership. The ASF licenses this file + # to you under the Apache License, Version 2.0 (the + # "License"); you may not use this file except in compliance + # with the License. You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + #} +{ + "input":[ + { + "type":"atlas_app", + "rowtype":"service", + "path":"{{atlas_log_dir}}/application.log" + } + ], + "filter":[ + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "atlas_app" + ] + + } + + }, + "log4j_format":"%d %-5p - [%t:%x] ~ %m (%c{1}:%L)%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{SPACE}-%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}~%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + + } + + } + + } + + ] + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/4a2dcfbe/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-falcon.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-falcon.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-falcon.json.j2 new file mode 100644 index 0000000..2e24e3f --- /dev/null +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-falcon.json.j2 @@ -0,0 +1,55 @@ +{# + # Licensed to the Apache Software Foundation (ASF) under one + # or more contributor license agreements. See the NOTICE file + # distributed with this work for additional information + # regarding copyright ownership. The ASF licenses this file + # to you under the Apache License, Version 2.0 (the + # "License"); you may not use this file except in compliance + # with the License. You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + #} +{ + "input":[ + { + "type":"falcon_app", + "rowtype":"service", + "path":"{{falcon_log_dir}}/falcon.application.log" + } + ], + "filter":[ + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "falcon_app" + ] + + } + + }, + "log4j_format":"%d %-5p - [%t:%x] ~ %m (%c{1}:%L)%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{SPACE}-%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}~%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + + } + + } + + } + + ] + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/4a2dcfbe/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hbase.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hbase.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hbase.json.j2 new file mode 100644 index 0000000..fe28f89 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hbase.json.j2 @@ -0,0 +1,62 @@ +{# + # Licensed to the Apache Software Foundation (ASF) under one + # or more contributor license agreements. See the NOTICE file + # distributed with this work for additional information + # regarding copyright ownership. The ASF licenses this file + # to you under the Apache License, Version 2.0 (the + # "License"); you may not use this file except in compliance + # with the License. You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + #} +{ + "input":[ + { + "type":"hbase_master", + "rowtype":"service", + "path":"{{hbase_log_dir}}/hbase-hbase-master-*.log" + }, + { + "type":"hbase_regionserver", + "rowtype":"service", + "path":"{{hbase_log_dir}}/hbase-hbase-regionserver-*.log" + } + + ], + "filter":[ + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "hbase_master", + "hbase_regionserver" + ] + + } + + }, + "log4j_format":"%d{ISO8601} %-5p [%t] %c{2}: %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{JAVACLASS:logger_name}:%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + + } + + } + + } + + ] + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/4a2dcfbe/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hdfs.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hdfs.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hdfs.json.j2 new file mode 100644 index 0000000..5697012 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hdfs.json.j2 @@ -0,0 +1,246 @@ +{# + # Licensed to the Apache Software Foundation (ASF) under one + # or more contributor license agreements. See the NOTICE file + # distributed with this work for additional information + # regarding copyright ownership. The ASF licenses this file + # to you under the Apache License, Version 2.0 (the + # "License"); you may not use this file except in compliance + # with the License. You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + #} +{ + "input":[ + { + "type":"hdfs_datanode", + "rowtype":"service", + "path":"{{hdfs_log_dir_prefix}}/hdfs/hadoop-hdfs-datanode-*.log" + }, + { + "type":"hdfs_namenode", + "rowtype":"service", + "path":"{{hdfs_log_dir_prefix}}/hdfs/hadoop-hdfs-namenode-*.log" + }, + { + "type":"hdfs_journalnode", + "rowtype":"service", + "path":"{{hdfs_log_dir_prefix}}/hdfs/hadoop-hdfs-journalnode-*.log" + }, + { + "type":"hdfs_secondarynamenode", + "rowtype":"service", + "path":"{{hdfs_log_dir_prefix}}/hdfs/hadoop-hdfs-secondarynamenode-*.log" + }, + { + "type":"hdfs_zkfc", + "rowtype":"service", + "path":"{{hdfs_log_dir_prefix}}/hdfs/hadoop-hdfs-zkfc-*.log" + }, + { + "type":"hdfs_audit", + "rowtype":"audit", + "is_enabled":"true", + "add_fields":{ + "logType":"HDFSAudit", + "enforcer":"hadoop-acl", + "repoType":"1", + "repo":"hdfs" + }, + "path":"{{hdfs_log_dir_prefix}}/hdfs/hdfs-audit.log" + } + + ], + "filter":[ + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "hdfs_datanode", + "hdfs_journalnode", + "hdfs_secondarynamenode", + "hdfs_namenode", + "hdfs_zkfc" + ] + + } + + }, + "log4j_format":"%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\(%{JAVAFILE:file}:%{JAVAMETHOD:method}\\(%{INT:line_number}\\)\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + + } + + } + + }, + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "hdfs_audit" + ] + + } + + }, + "log4j_format":"%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:evtTime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:evtTime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}:%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "evtTime":{ + "map_date":{ + "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + + } + + } + + }, + { + "filter":"keyvalue", + "sort_order":1, + "conditions":{ + "fields":{ + "type":[ + "hdfs_audit" + ] + + } + + }, + "source_field":"log_message", + "value_split":"=", + "field_split":"\t", + "post_map_values":{ + "src":{ + "map_fieldname":{ + "new_fieldname":"resource" + } + + }, + "ip":{ + "map_fieldname":{ + "new_fieldname":"cliIP" + } + + }, + "allowed":[ + { + "map_fieldvalue":{ + "pre_value":"true", + "post_value":"1" + } + + }, + { + "map_fieldvalue":{ + "pre_value":"false", + "post_value":"0" + } + + }, + { + "map_fieldname":{ + "new_fieldname":"result" + } + + } + + ], + "cmd":{ + "map_fieldname":{ + "new_fieldname":"action" + } + + }, + "proto":{ + "map_fieldname":{ + "new_fieldname":"cliType" + } + + }, + "callerContext":{ + "map_fieldname":{ + "new_fieldname":"req_caller_id" + } + + } + + } + + }, + { + "filter":"grok", + "sort_order":2, + "source_field":"ugi", + "remove_source_field":"false", + "conditions":{ + "fields":{ + "type":[ + "hdfs_audit" + ] + + } + + }, + "message_pattern":"%{USERNAME:p_user}.+auth:%{USERNAME:p_authType}.+via %{USERNAME:k_user}.+auth:%{USERNAME:k_authType}|%{USERNAME:user}.+auth:%{USERNAME:authType}|%{USERNAME:x_user}", + "post_map_values":{ + "user":{ + "map_fieldname":{ + "new_fieldname":"reqUser" + } + + }, + "x_user":{ + "map_fieldname":{ + "new_fieldname":"reqUser" + } + + }, + "p_user":{ + "map_fieldname":{ + "new_fieldname":"reqUser" + } + + }, + "k_user":{ + "map_fieldname":{ + "new_fieldname":"proxyUsers" + } + + }, + "p_authType":{ + "map_fieldname":{ + "new_fieldname":"authType" + } + + }, + "k_authType":{ + "map_fieldname":{ + "new_fieldname":"proxyAuthType" + } + + } + + } + + } + + ] + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/4a2dcfbe/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hive.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hive.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hive.json.j2 new file mode 100644 index 0000000..1b8d9a1 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hive.json.j2 @@ -0,0 +1,62 @@ +{# + # Licensed to the Apache Software Foundation (ASF) under one + # or more contributor license agreements. See the NOTICE file + # distributed with this work for additional information + # regarding copyright ownership. The ASF licenses this file + # to you under the Apache License, Version 2.0 (the + # "License"); you may not use this file except in compliance + # with the License. You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + #} +{ + "input":[ + { + "type":"hive_hiveserver2", + "rowtype":"service", + "path":"{{hive_log_dir}}/hiveserver2.log" + }, + { + "type":"hive_metastore", + "rowtype":"service", + "path":"{{hive_log_dir}}/hivemetastore.log" + } + + ], + "filter":[ + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "hive_hiveserver2", + "hive_metastore" + ] + + } + + }, + "log4j_format":"%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]:%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\(%{JAVAFILE:file}:%{JAVAMETHOD:method}\\(%{INT:line_number}\\)\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + + } + + } + + } + + ] + +} \ No newline at end of file
