http://git-wip-us.apache.org/repos/asf/ambari/blob/e734a7bf/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/configuration/knox-logsearch-conf.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/configuration/knox-logsearch-conf.xml b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/configuration/knox-logsearch-conf.xml new file mode 100644 index 0000000..528b011 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/configuration/knox-logsearch-conf.xml @@ -0,0 +1,93 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration supports_final="false" supports_adding_forbidden="true"> + <property> + <name>service_name</name> + <display-name>Service name</display-name> + <description>Service name for Logsearch Portal (label)</description> + <value>Knox</value> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>component_mappings</name> + <display-name>Component mapping</display-name> + <description>Logsearch component logid mapping list (e.g.: COMPONENT1:logid1,logid2;COMPONENT2:logid3)</description> + <value>KNOX_GATEWAY:knox_gateway,knox_cli,knox_ldap</value> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>content</name> + <display-name>Logfeeder Config</display-name> + <description>Metadata jinja template for Logfeeder which contains grok patterns for reading service specific logs.</description> + <value> +{ + "input":[ + { + "type":"knox_gateway", + "rowtype":"service", + "path":"/var/log/knox/gateway.log" + }, + { + "type":"knox_cli", + "rowtype":"service", + "path":"/var/log/knox/knoxcli.log" + }, + { + "type":"knox_ldap", + "rowtype":"service", + "path":"/var/log/knox/ldap.log" + } + ], + "filter":[ + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "knox_gateway", + "knox_cli", + "knox_ldap" + ] + + } + }, + "log4j_format":"%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\(%{JAVAFILE:file}:%{JAVAMETHOD:method}\\(%{INT:line_number}\\)\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + } + } + } + ] + } + </value> + <value-attributes> + <type>content</type> + <show-property-name>false</show-property-name> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> +</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/e734a7bf/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-system_log-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-system_log-env.xml b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-system_log-env.xml new file mode 100644 index 0000000..b8dd4b9 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-system_log-env.xml @@ -0,0 +1,59 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration supports_adding_forbidden="true"> + + <property> + <name>logfeeder_system_messages_content</name> + <display-name>System messages grok filter</display-name> + <description>This is the jinja template for input.config.system_messages.json file</description> + <value/> + <property-type>VALUE_FROM_PROPERTY_FILE</property-type> + <value-attributes> + <property-file-name>system_messages.json</property-file-name> + <property-file-type>text</property-file-type> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + + <property> + <name>logfeeder_secure_log_content</name> + <display-name>Secure log grok filter</display-name> + <description>This is the jinja template for input.config.secure_log.json file</description> + <value/> + <property-type>VALUE_FROM_PROPERTY_FILE</property-type> + <value-attributes> + <property-file-name>secure_log.json</property-file-name> + <property-file-type>text</property-file-type> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + + <property> + <name>logfeeder_system_log_enabled</name> + <value>false</value> + <value-attributes> + <type>boolean</type> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + +</configuration> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/e734a7bf/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logfeeder.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logfeeder.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logfeeder.py index cfc6c91..fbc29c8 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logfeeder.py +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logfeeder.py @@ -25,7 +25,6 @@ from resource_management.libraries.script.script import Script from setup_logfeeder import setup_logfeeder from logsearch_common import kill_process - class LogFeeder(Script): def install(self, env): import params http://git-wip-us.apache.org/repos/asf/ambari/blob/e734a7bf/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logsearch.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logsearch.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logsearch.py index 9c5215d..4410712 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logsearch.py +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logsearch.py @@ -24,7 +24,6 @@ from resource_management.libraries.script.script import Script from setup_logsearch import setup_logsearch from logsearch_common import kill_process - class LogSearch(Script): def install(self, env): import params http://git-wip-us.apache.org/repos/asf/ambari/blob/e734a7bf/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logsearch_config_aggregator.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logsearch_config_aggregator.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logsearch_config_aggregator.py new file mode 100644 index 0000000..ad515ee --- /dev/null +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logsearch_config_aggregator.py @@ -0,0 +1,67 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" +from resource_management import Logger + +def __parse_component_mappings(component_mappings): + components = list() + component_mappings_list = component_mappings.split(';') + if component_mappings_list and len(component_mappings_list) > 0: + metadata_list = map(lambda x : x.split(':'), component_mappings_list) + if metadata_list and len(metadata_list) > 0: + for metadata in metadata_list: + if (len(metadata) == 2): + logids = metadata[1].split(',') + components.extend(logids) + Logger.info("Found logids for logsearch component %s - (%s) " % (metadata[0], metadata[1])) + return components + +def get_logsearch_meta_configs(configurations): + logsearch_meta_configs = {} + for key, value in configurations.iteritems(): # iter on both keys and values + if str(key).endswith('logsearch-conf'): + logsearch_meta_configs[key] = value + Logger.info("Found logsearch config entry : " + key) + return logsearch_meta_configs + +def get_logfeeder_metadata(logsearch_meta_configs): + """ + get logfeeder pattern metadata list, an element: (e.g.) : + ['service_config_name' : 'pattern json content'] + """ + logfeeder_contents = {} + for key, value in logsearch_meta_configs.iteritems(): + if 'content' in logsearch_meta_configs[key]: + logfeeder_contents[key] = logsearch_meta_configs[key]['content'] + Logger.info("Found logfeeder pattern content in " + key) + return logfeeder_contents + +def get_logsearch_metadata(logsearch_meta_configs): + """ + get logsearch metadata list, an element (e.g.) : + ['service_name_key' : {component1 : [logid1, logid2]}, {component2 : [logid1, logid2]}] + """ + logsearch_service_component_mappings = {} + for key, value in logsearch_meta_configs.iteritems(): + if 'service_name' in logsearch_meta_configs[key] and 'component_mappings' in logsearch_meta_configs[key]: + service_name = logsearch_meta_configs[key]['service_name'] + component_mappings = __parse_component_mappings(logsearch_meta_configs[key]['component_mappings']) + logsearch_service_component_mappings[service_name] = component_mappings + + return logsearch_service_component_mappings + http://git-wip-us.apache.org/repos/asf/ambari/blob/e734a7bf/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py index 4526640..1f5064a 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py @@ -19,6 +19,7 @@ limitations under the License. """ from ambari_commons.constants import AMBARI_SUDO_BINARY +from logsearch_config_aggregator import get_logfeeder_metadata, get_logsearch_metadata, get_logsearch_meta_configs from resource_management.libraries.functions.default import default from resource_management.libraries.functions.format import format from resource_management.libraries.functions.is_empty import is_empty @@ -66,6 +67,11 @@ zookeeper_hosts = ",".join(zookeeper_hosts_list) cluster_name = str(config['clusterName']) availableServices = config['availableServices'] +configurations = config['configurations'] # need reference inside logfeeder jinja templates +logserch_meta_configs = get_logsearch_meta_configs(configurations) +logsearch_metadata = get_logsearch_metadata(logserch_meta_configs) +logfeeder_metadata = get_logfeeder_metadata(logserch_meta_configs) + # for now just pick first collector if 'metrics_collector_hosts' in config['clusterHostInfo']: metrics_collector_hosts_list = ",".join(config['clusterHostInfo']['metrics_collector_hosts']) @@ -139,41 +145,13 @@ logsearch_app_log4j_content = config['configurations']['logsearch-log4j']['conte # Log dirs ambari_server_log_dir = '/var/log/ambari-server' ambari_agent_log_dir = '/var/log/ambari-agent' -knox_log_dir = '/var/log/knox' hst_log_dir = '/var/log/hst' hst_activity_log_dir = '/var/log/smartsense-activity' -metrics_collector_log_dir = default('/configurations/ams-env/metrics_collector_log_dir', '/var/log/ambari-metrics-collector') -metrics_monitor_log_dir = default('/configurations/ams-env/metrics_monitor_log_dir', '/var/log/ambari-metrics-monitor') -metrics_grafana_log_dir = default('/configurations/ams-grafana-env/metrics_grafana_log_dir', '/var/log/ambari-metrics-grafana') - -atlas_log_dir = default('/configurations/atlas-env/metadata_log_dir', '/var/log/atlas') -accumulo_log_dir = default('/configurations/accumulo-env/accumulo_log_dir', '/var/log/accumulo') -falcon_log_dir = default('/configurations/falcon-env/falcon_log_dir', '/var/log/falcon') -flume_log_dir = default('/configurations/flume-env/flume_log_dir', '/var/log/flume') -hbase_log_dir = default('/configurations/hbase-env/hbase_log_dir', '/var/log/hbase') -hdfs_log_dir_prefix = default('/configurations/hadoop-env/hdfs_log_dir_prefix', '/var/log/hadoop') -hive_log_dir = default('/configurations/hive-env/hive_log_dir', '/var/log/hive') -hcat_log_dir = default('configurations/hive-env/hcat_log_dir', '/var/log/webhcat') -infra_solr_log_dir = default('configurations/infra-solr-env/infra_solr_log_dir', '/var/log/ambari-infra-solr') -kafka_log_dir = default('/configurations/kafka-env/kafka_log_dir', '/var/log/kafka') -nifi_log_dir = default('/configurations/nifi-env/nifi_node_log_dir', '/var/log/nifi') -oozie_log_dir = default('/configurations/oozie-env/oozie_log_dir', '/var/log/oozie') -ranger_usersync_log_dir = default('/configurations/ranger-env/ranger_usersync_log_dir', '/var/log/ranger/usersync') -ranger_admin_log_dir = default('/configurations/ranger-env/ranger_admin_log_dir', '/var/log/ranger/admin') -ranger_kms_log_dir = default('/configurations/kms-env/kms_log_dir', '/var/log/ranger/kms') -storm_log_dir = default('/configurations/storm-env/storm_log_dir', '/var/log/storm') -yarn_log_dir_prefix = default('/configurations/yarn-env/yarn_log_dir_prefix', '/var/log/hadoop') -mapred_log_dir_prefix = default('/configurations/mapred-env/mapred_log_dir_prefix', '/var/log/hadoop') -zeppelin_log_dir = default('/configurations/zeppelin-env/zeppelin_log_dir', '/var/log/zeppelin') -zk_log_dir = default('/configurations/zookeeper-env/zk_log_dir', '/var/log/zookeeper') -spark_log_dir = default('/configurations/spark-env/spark_log_dir', '/var/log/spark') -livy_log_dir = default('/configurations/livy-env/livy_log_dir', '/var/log/livy') -spark2_log_dir = default('/configurations/spark2-env/spark_log_dir', '/var/log/spark2') - -hdfs_user = default('configurations/hadoop-env/hdfs_user', 'hdfs') -mapred_user = default('configurations/mapred-env/mapred_user', 'mapred') -yarn_user = default('configurations/yarn-env/yarn_user', 'yarn') +# System logs +logfeeder_system_messages_content = config['configurations']['logfeeder-system_log-env']['logfeeder_system_messages_content'] +logfeeder_secure_log_content = config['configurations']['logfeeder-system_log-env']['logfeeder_secure_log_content'] +logfeeder_system_log_enabled = default('/configurations/logfeeder-system_log-env/logfeeder_system_log_enabled', False) # Logsearch auth configs @@ -272,14 +250,17 @@ logfeeder_truststore_location = config['configurations']['logfeeder-env']['logfe logfeeder_truststore_password = config['configurations']['logfeeder-env']['logfeeder_truststore_password'] logfeeder_truststore_type = config['configurations']['logfeeder-env']['logfeeder_truststore_type'] -logfeeder_supported_services = ['accumulo', 'ambari', 'ams', 'atlas', 'falcon', 'flume', 'hbase', 'hdfs', 'hive', 'hst', 'infra', - 'kafka', 'knox', 'logsearch', 'nifi', 'oozie', 'ranger', 'spark', 'spark2', 'storm', 'yarn', - 'zeppelin', 'zookeeper'] +logfeeder_default_services = ['ambari', 'logsearch'] +logfeeder_default_config_file_names = ['global.config.json', 'output.config.json'] + ['input.config-%s.json' % (tag) for tag in logfeeder_default_services] +logfeeder_custom_config_file_names = ['input.config-%s.json' % (tag.replace('-logsearch-conf', '')) + for tag, content in logfeeder_metadata.iteritems() if any(logfeeder_metadata)] -logfeeder_config_file_names = \ - ['global.config.json', 'output.config.json'] + ['input.config-%s.json' % (tag) for tag in logfeeder_supported_services] +if logfeeder_system_log_enabled: + default_config_files = ','.join(logfeeder_default_config_file_names + logfeeder_custom_config_file_names + + ['input.config-system_messages.json', 'input.config-secure_log.json']) +else: + default_config_files = ','.join(logfeeder_default_config_file_names + logfeeder_custom_config_file_names) -default_config_files = ','.join(logfeeder_config_file_names) logfeeder_grok_patterns = config['configurations']['logfeeder-grok']['default_grok_patterns'] if config['configurations']['logfeeder-grok']['custom_grok_patterns'].strip(): @@ -292,8 +273,6 @@ if config['configurations']['logfeeder-grok']['custom_grok_patterns'].strip(): '\n' + \ config['configurations']['logfeeder-grok']['custom_grok_patterns'] -logfeeder_properties = dict(config['configurations']['logfeeder-properties']) - # logfeeder properties # load default values http://git-wip-us.apache.org/repos/asf/ambari/blob/e734a7bf/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py index ad16958..e90d623 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py @@ -17,6 +17,7 @@ limitations under the License. """ +from resource_management.libraries.functions.default import default from resource_management.core.resources.system import Directory, File from resource_management.libraries.functions.format import format from resource_management.core.source import InlineTemplate, Template @@ -61,11 +62,25 @@ def setup_logfeeder(): encoding="utf-8" ) - for file_name in params.logfeeder_config_file_names: + for file_name in params.logfeeder_default_config_file_names: File(format("{logsearch_logfeeder_conf}/" + file_name), content=Template(file_name + ".j2") ) + for service, pattern_content in params.logfeeder_metadata.iteritems(): + File(format("{logsearch_logfeeder_conf}/input.config-" + service.replace('-logsearch-conf', '') + ".json"), + content=InlineTemplate(pattern_content, extra_imports=[default]) + ) + + if params.logfeeder_system_log_enabled: + File(format("{logsearch_logfeeder_conf}/input.config-system_messages.json"), + content=params.logfeeder_system_messages_content + ) + File(format("{logsearch_logfeeder_conf}/input.config-secure_log.json"), + content=params.logfeeder_secure_log_content + ) + + if params.security_enabled: File(format("{logfeeder_jaas_file}"), content=Template("logfeeder_jaas.conf.j2") http://git-wip-us.apache.org/repos/asf/ambari/blob/e734a7bf/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/HadoopServiceConfig.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/HadoopServiceConfig.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/HadoopServiceConfig.json.j2 index 7982631..7e9153f 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/HadoopServiceConfig.json.j2 +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/HadoopServiceConfig.json.j2 @@ -15,520 +15,86 @@ # See the License for the specific language governing permissions and # limitations under the License. #} -{ - "service": { -{% if "ACCUMULO" in availableServices %} - "accumulo": { - "label": "Accumulo", + { + "service": { +{% if logsearch_metadata and logsearch_metadata.keys()|length > 0 %} + {% for metadata_key, component_list in logsearch_metadata.iteritems() %} + "{{ metadata_key.lower() }}": { + "label": "{{ metadata_key }}", "components": [ + {% for component in component_list %} { - "name": "accumulo_gc" - }, - { - "name": "accumulo_master" - }, - { - "name": "accumulo_monitor" - }, - { - "name": "accumulo_tracer" - }, - { - "name": "accumulo_tserver" - } - ], - "dependencies": [ - ] - }, -{% endif %} -{% if "ATLAS" in availableServices %} - "atlas": { - "label": "Atlas", - "components": [ - { - "name": "atlas_app" - } - ], - "dependencies": [ - ] - }, -{% endif %} - "ambari": { - "label": "Ambari", - "components": [ - { - "name": "ambari_agent" - }, - { - "name": "ambari_server" - }, - { - "name": "ambari_alerts" - }, - { - "name": "ambari_audit" - }, - { - "name": "ambari_config_changes" - }, - { - "name": "ambari_eclipselink" - }, - { - "name": "ambari_server_check_database" - } - ], - "dependencies": [ - ] - }, -{% if "AMBARI_METRICS" in availableServices %} - "ams": { - "label": "AMS", - "components": [ - { - "name": "ams_hbase_master" - }, - { - "name": "ams_hbase_regionserver" - }, - { - "name": "ams_collector" - }, - { - "name": "ams_monitor" - }, - { - "name": "ams_grafana" - } - ], - "dependencies": [ - ] - }, -{% endif %} -{% if "FALCON" in availableServices %} - "falcon": { - "label": "Falcon", - "components": [ - { - "name": "falcon_app" - } - ], - "dependencies": [ - ] - }, -{% endif %} -{% if "FLUME" in availableServices %} - "flume": { - "label": "Flume", - "components": [ - { - "name": "flume_handler" - } - ], - "dependencies": [ - ] - }, -{% endif %} -{% if "HBASE" in availableServices %} - "hbase": { - "label": "HBase", - "components": [ - { - "name": "hbase_master" - }, - { - "name": "hbase_regionserver" - }, - { - "name": "hbase_phoenix_server" - } - ], - "dependencies": [ - { - "service": "hdfs", - "components": [ - "hdfs_namenode" - ] - } - ] - }, -{% endif %} -{% if "HDFS" in availableServices %} - "hdfs": { - "label": "HDFS", - "components": [ - { - "name": "hdfs_datanode" - }, - { - "name": "hdfs_namenode" - }, - { - "name": "hdfs_journalnode" - }, - { - "name": "hdfs_secondarynamenode" - }, - { - "name": "hdfs_zkfc" - }, - { - "name": "hdfs_nfs3" - }, - { - "name": "hdfs_audit", - "rowtype": "audit" - } - ], - "dependencies": [ - ] - }, -{% endif %} -{% if "HIVE" in availableServices %} - "hive": { - "label": "Hive", - "components": [ - { - "name": "hive_hiveserver2" - }, - { - "name": "hive_metastore" - }, - { - "name": "webhcat_server" - } - ], - "dependencies": [ - { - "service": "hdfs", - "components": [ - "hdfs_namenode" - ] - } - ] - }, -{% endif %} -{% if "KAFKA" in availableServices %} - "kafka": { - "label": "Kafka", - "components": [ - { - "name": "kafka_controller" - }, - { - "name": "kafka_request" - }, - { - "name": "kafka_logcleaner" - }, - { - "name": "kafka_server" - }, - { - "name": "kafka_statechange" - } - ], - "dependencies": [ - { - "service": "zookeeper", - "components": [ - "zookeeper" - ] - } - ] - }, -{% endif %} - "ambari_infra" : { - "label": "Infra", - "components" : [ - { - "name": "infra_solr" - } - ], - "dependencies": [ - ] - }, -{% if "KNOX" in availableServices %} - "knox": { - "label": "Knox", - "components": [ - { - "name": "knox_gateway" - }, - { - "name": "knox_cli" - }, - { - "name": "knox_ldap" - } - ], - "dependencies": [ - ] - }, -{% endif %} -{% if "NIFI" in availableServices %} - "nifi": { - "label": "NiFi", - "components": [ - { - "name": "nifi_app" - }, - { - "name": "nifi_bootstrap" - }, - { - "name": "nifi_setup" - }, - { - "name": "nifi_user" - }, - { - "name": "nifi_ca" - } - ], - "dependencies": [ - ] - }, -{% endif %} -{% if "MAPREDUCE2" in availableServices %} - "mapred": { - "label": "MapReduce", - "components": [ - { - "name": "mapred_historyserver" - } - ], - "dependencies": [ - ] - }, -{% endif %} - "logsearch": { - "label": "Logsearch", - "components": [ - { - "name": "logsearch_app" - }, - { - "name": "logsearch_feeder" - }, - { - "name": "logsearch_perf" - } - ], - "dependencies": [ - ] - }, -{% if "RANGER" in availableServices %} - "ranger": { - "label": "Ranger", - "components": [ - { - "name": "ranger_admin" - }, - { - "name": "ranger_dbpatch" - }, - { - "name": "ranger_kms" - }, - { - "name": "ranger_usersync" - } - ], - "dependencies": [ - { - "service": "hdfs", - "required": "optional", - "components": [ - "hdfs_namenode" - ] - }, - { - "service": "hbase", - "required": "optional", - "components": [ - "hbase_master", - "hbase_regionserver" - ] - }, - { - "service": "hive", - "required": "optional", - "components": [ - "hive_hiveserver2" - ] - }, - { - "service": "kafka", - "required": "optional", - "components": [ - "kafka_ranger" - ] - }, - { - "service": "knox", - "required": "optional", - "components": [ - "knox_gateway" - ] - }, - { - "service": "storm", - "required": "optional", - "components": [ - "storm_supervisor" - ] - }, - { - "service": "yarn", - "required": "optional", - "components": [ - "yarn_resourcemanager" - ] - } - ] - }, -{% endif %} -{% if "OOZIE" in availableServices %} - "oozie": { - "label": "Oozie", - "components": [ - { - "name": "oozie_app" - } - ], - "dependencies": [ - ] - }, -{% endif %} -{% if "SMARTSENSE" in availableServices %} - "hst": { - "label": "SmartSense", - "components": [ - { - "name": "hst_server" - }, - { - "name": "hst_agent" - }, - { - "name": "activity_analyzer" - }, - { - "name": "activity_explorer" - } - ], - "dependencies": [ - ] - }, -{% endif %} -{% if "SPARK" in availableServices %} - "spark": { - "label": "Spark", - "components": [ - { - "name": "spark_jobhistory_server" - }, - { - "name": "spark_thriftserver" - }, - { - "name": "livy_server" - } - ], - "dependencies": [ - ] - }, -{% endif %} -{% if "SPARK2" in availableServices %} - "spark2": { - "label": "Spark 2", - "components": [ - { - "name": "spark2_jobhistory_server" - }, - { - "name": "spark2_thriftserver" - } - ], - "dependencies": [ - ] - }, -{% endif %} -{% if "STORM" in availableServices %} - "storm": { - "label": "Storm", - "components": [ - { - "name": "storm_drpc" - }, - { - "name": "storm_logviewer" - }, - { - "name": "storm_nimbus" - }, - { - "name": "storm_supervisor" - }, - { - "name": "storm_ui" - }, - { - "name": "storm_worker" - } - ], - "dependencies": [ - ] - }, -{% endif %} -{% if "YARN" in availableServices %} - "yarn": { - "label": "YARN", - "components": [ - { - "name": "yarn_nodemanager" - }, - { - "name": "yarn_resourcemanager" - }, - { - "name": "yarn_timelineserver" - }, - { - "name": "yarn_historyserver" - }, - { - "name": "yarn_jobsummary" - } - ], - "dependencies": [ - ] - }, -{% endif %} -{% if "ZEPPELIN" in availableServices %} - "zeppelin": { - "label": "Zeppelin", - "components": [ - { - "name": "zeppelin" - } - ], - "dependencies": [ - ] - }, -{% endif %} -{% if "ZOOKEEPER" in availableServices %} - "zookeeper": { - "label": "ZooKeeper", - "components": [ - { - "name": "zookeeper" + "name": "{{ component }}" } + {% if not loop.last %} + , + {% endif %} + {% endfor %} ], "dependencies": [ ] } -{% endif %} + , + {% endfor %} +{% endif %} +{% if logfeeder_system_log_enabled %} + "system" : { + "label" : "System", + "components" : [ + { + "name": "system_message" + }, + { + "name": "secure_log" + } + ], + "dependencies": [ + ] + }, +{% endif %} + "ambari": { + "label": "Ambari", + "components": [ + { + "name": "ambari_agent" + }, + { + "name": "ambari_server" + }, + { + "name": "ambari_alerts" + }, + { + "name": "ambari_audit" + }, + { + "name": "ambari_config_changes" + }, + { + "name": "ambari_eclipselink" + }, + { + "name": "ambari_server_check_database" + } + ], + "dependencies": [ + ] + }, + "logsearch": { + "label": "Logsearch", + "components": [ + { + "name": "logsearch_app" + }, + { + "name": "logsearch_feeder" + }, + { + "name": "logsearch_perf" + } + ], + "dependencies": [ + ] + } } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/e734a7bf/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-accumulo.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-accumulo.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-accumulo.json.j2 deleted file mode 100644 index 1fd93cf..0000000 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-accumulo.json.j2 +++ /dev/null @@ -1,105 +0,0 @@ -{# - # Licensed to the Apache Software Foundation (ASF) under one - # or more contributor license agreements. See the NOTICE file - # distributed with this work for additional information - # regarding copyright ownership. The ASF licenses this file - # to you under the Apache License, Version 2.0 (the - # "License"); you may not use this file except in compliance - # with the License. You may obtain a copy of the License at - # - # http://www.apache.org/licenses/LICENSE-2.0 - # - # Unless required by applicable law or agreed to in writing, software - # distributed under the License is distributed on an "AS IS" BASIS, - # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - # See the License for the specific language governing permissions and - # limitations under the License. - #} -{ - "input":[ - { - "type":"accumulo_gc", - "rowtype":"service", - "path":"{{accumulo_log_dir}}/gc_*.log" - }, - { - "type":"accumulo_master", - "rowtype":"service", - "path":"{{accumulo_log_dir}}/master_*.log" - }, - { - "type":"accumulo_monitor", - "rowtype":"service", - "path":"{{accumulo_log_dir}}/monitor_*.log" - }, - { - "type":"accumulo_tracer", - "rowtype":"service", - "path":"{{accumulo_log_dir}}/tracer_*.log" - }, - { - "type":"accumulo_tserver", - "rowtype":"service", - "path":"{{accumulo_log_dir}}/tserver_*.log" - } - - ], - "filter":[ - { - "filter":"grok", - "conditions":{ - "fields":{ - "type":[ - "accumulo_master" - ] - - } - - }, - "log4j_format":"%d{ISO8601} [%-8c{2}] %-5p: %m%n", - "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", - "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}\\[%{JAVACLASS:logger_name}\\]%{SPACE}%{LOGLEVEL:level}:%{SPACE}%{GREEDYDATA:log_message}", - "post_map_values":{ - "logtime":{ - "map_date":{ - "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" - } - - } - - } - - }, - { - "filter":"grok", - "comment":"This one has one extra space after LEVEL", - "conditions":{ - "fields":{ - "type":[ - "accumulo_gc", - "accumulo_monitor", - "accumulo_tracer", - "accumulo_tserver" - ] - - } - - }, - "log4j_format":"%d{ISO8601} [%-8c{2}] %-5p: %X{application} %m%n", - "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", - "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}\\[%{JAVACLASS:logger_name}\\]%{SPACE}%{LOGLEVEL:level}%{SPACE}:%{SPACE}%{GREEDYDATA:log_message}", - "post_map_values":{ - "logtime":{ - "map_date":{ - "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" - } - - } - - } - - } - - ] - -} http://git-wip-us.apache.org/repos/asf/ambari/blob/e734a7bf/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-ambari.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-ambari.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-ambari.json.j2 index 36507c9..ad4adb2 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-ambari.json.j2 +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-ambari.json.j2 @@ -31,7 +31,7 @@ "type":"ambari_alerts", "rowtype":"service", "add_fields":{ - "level":"INFO" + "level":"UNKNOWN" }, "path":"{{ambari_server_log_dir}}/ambari-alerts.log" }, http://git-wip-us.apache.org/repos/asf/ambari/blob/e734a7bf/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-ams.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-ams.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-ams.json.j2 deleted file mode 100644 index 0bcaf56..0000000 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-ams.json.j2 +++ /dev/null @@ -1,191 +0,0 @@ -{# - # Licensed to the Apache Software Foundation (ASF) under one - # or more contributor license agreements. See the NOTICE file - # distributed with this work for additional information - # regarding copyright ownership. The ASF licenses this file - # to you under the Apache License, Version 2.0 (the - # "License"); you may not use this file except in compliance - # with the License. You may obtain a copy of the License at - # - # http://www.apache.org/licenses/LICENSE-2.0 - # - # Unless required by applicable law or agreed to in writing, software - # distributed under the License is distributed on an "AS IS" BASIS, - # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - # See the License for the specific language governing permissions and - # limitations under the License. - #} -{ - "input":[ - { - "type":"ams_hbase_master", - "rowtype":"service", - "path":"{{metrics_collector_log_dir}}/hbase-ams-master-*.log" - }, - { - "type":"ams_hbase_regionserver", - "rowtype":"service", - "path":"{{metrics_collector_log_dir}}/hbase-ams-regionserver-*.log" - }, - { - "type":"ams_collector", - "rowtype":"service", - "path":"{{metrics_collector_log_dir}}/ambari-metrics-collector.log" - }, - { - "type":"ams_monitor", - "rowtype":"service", - "path":"{{metrics_monitor_log_dir}}/ambari-metrics-monitor.out" - }, - { - "type":"ams_grafana", - "rowtype":"service", - "path":"{{metrics_grafana_log_dir}}/grafana.log" - } - ], - "filter":[ - { - "filter":"grok", - "conditions":{ - "fields":{ - "type":[ - "ams_collector" - ] - - } - - }, - "log4j_format":"%d{ISO8601} %p %c: %m%n", - "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", - "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}:%{SPACE}%{GREEDYDATA:log_message}", - "post_map_values":{ - "logtime":{ - "map_date":{ - "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" - } - - } - - } - - }, - { - "filter":"grok", - "conditions":{ - "fields":{ - "type":[ - "ams_hbase_master", - "ams_hbase_regionserver" - ] - - } - - }, - "log4j_format":"%d{ISO8601} %-5p [%t] %c{2}: %m%n", - "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", - "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{JAVACLASS:logger_name}:%{SPACE}%{GREEDYDATA:log_message}", - "post_map_values":{ - "logtime":{ - "map_date":{ - "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" - } - - } - - } - - }, - { - "filter":"grok", - "conditions":{ - "fields":{ - "type":[ - "ams_grafana" - ] - - } - - }, - "log4j_format":"%d{ISO8601} %-5p [%t] %c{2}: %m%n", - "multiline_pattern":"^(%{DATESTAMP:logtime})", - "message_pattern":"(?m)^%{DATESTAMP:logtime}%{SPACE}\\[%{WORD:level}\\]%{SPACE}%{GREEDYDATA:log_message}", - "post_map_values":{ - "logtime":{ - "map_date":{ - "target_date_pattern":"yyyy/MM/dd HH:mm:ss" - } - - }, - "level":[ - { - "map_fieldvalue":{ - "pre_value":"I", - "post_value":"INFO" - } - }, - { - "map_fieldvalue":{ - "pre_value":"W", - "post_value":"WARN" - } - }, - { - "map_fieldvalue":{ - "pre_value":"D", - "post_value":"DEBUG" - } - }, - { - "map_fieldvalue":{ - "pre_value":"E", - "post_value":"ERROR" - } - }, - { - "map_fieldvalue":{ - "pre_value":"F", - "post_value":"FATAL" - } - } - ] - - } - - }, - { - "filter":"grok", - "conditions":{ - "fields":{ - "type":[ - "ams_monitor" - ] - - } - - }, - "log4j_format":"", - "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", - "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}\\[%{LOGLEVEL:level}\\]%{SPACE}%{JAVAFILE:file}:%{INT:line_number}%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", - "post_map_values":{ - "logtime":{ - "map_date":{ - "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" - } - - } - - }, - "level":[ - { - "map_fieldvalue":{ - "pre_value":"WARNING", - "post_value":"WARN" - } - } - ] - - } - - ] - -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/e734a7bf/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-atlas.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-atlas.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-atlas.json.j2 deleted file mode 100644 index 53912c8..0000000 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-atlas.json.j2 +++ /dev/null @@ -1,55 +0,0 @@ -{# - # Licensed to the Apache Software Foundation (ASF) under one - # or more contributor license agreements. See the NOTICE file - # distributed with this work for additional information - # regarding copyright ownership. The ASF licenses this file - # to you under the Apache License, Version 2.0 (the - # "License"); you may not use this file except in compliance - # with the License. You may obtain a copy of the License at - # - # http://www.apache.org/licenses/LICENSE-2.0 - # - # Unless required by applicable law or agreed to in writing, software - # distributed under the License is distributed on an "AS IS" BASIS, - # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - # See the License for the specific language governing permissions and - # limitations under the License. - #} -{ - "input":[ - { - "type":"atlas_app", - "rowtype":"service", - "path":"{{atlas_log_dir}}/application.log" - } - ], - "filter":[ - { - "filter":"grok", - "conditions":{ - "fields":{ - "type":[ - "atlas_app" - ] - - } - - }, - "log4j_format":"%d %-5p - [%t:%x] ~ %m (%c{1}:%L)%n", - "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", - "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{SPACE}-%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}~%{SPACE}%{GREEDYDATA:log_message}", - "post_map_values":{ - "logtime":{ - "map_date":{ - "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" - } - - } - - } - - } - - ] - -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/e734a7bf/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-falcon.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-falcon.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-falcon.json.j2 deleted file mode 100644 index a40e52d..0000000 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-falcon.json.j2 +++ /dev/null @@ -1,55 +0,0 @@ -{# - # Licensed to the Apache Software Foundation (ASF) under one - # or more contributor license agreements. See the NOTICE file - # distributed with this work for additional information - # regarding copyright ownership. The ASF licenses this file - # to you under the Apache License, Version 2.0 (the - # "License"); you may not use this file except in compliance - # with the License. You may obtain a copy of the License at - # - # http://www.apache.org/licenses/LICENSE-2.0 - # - # Unless required by applicable law or agreed to in writing, software - # distributed under the License is distributed on an "AS IS" BASIS, - # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - # See the License for the specific language governing permissions and - # limitations under the License. - #} -{ - "input":[ - { - "type":"falcon_app", - "rowtype":"service", - "path":"{{falcon_log_dir}}/falcon.application.log" - } - ], - "filter":[ - { - "filter":"grok", - "conditions":{ - "fields":{ - "type":[ - "falcon_app" - ] - - } - - }, - "log4j_format":"%d %-5p - [%t:%x] ~ %m (%c{1}:%L)%n", - "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", - "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{SPACE}-%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}~%{SPACE}%{GREEDYDATA:log_message}", - "post_map_values":{ - "logtime":{ - "map_date":{ - "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" - } - - } - - } - - } - - ] - -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/e734a7bf/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-flume.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-flume.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-flume.json.j2 deleted file mode 100644 index b31054b..0000000 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-flume.json.j2 +++ /dev/null @@ -1,56 +0,0 @@ -{# - # Licensed to the Apache Software Foundation (ASF) under one - # or more contributor license agreements. See the NOTICE file - # distributed with this work for additional information - # regarding copyright ownership. The ASF licenses this file - # to you under the Apache License, Version 2.0 (the - # "License"); you may not use this file except in compliance - # with the License. You may obtain a copy of the License at - # - # http://www.apache.org/licenses/LICENSE-2.0 - # - # Unless required by applicable law or agreed to in writing, software - # distributed under the License is distributed on an "AS IS" BASIS, - # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - # See the License for the specific language governing permissions and - # limitations under the License. - #} -{ - "input":[ - { - "type":"flume_handler", - "rowtype":"service", - "path":"{{flume_log_dir}}/flume.log" - } - - ], - "filter":[ - { - "filter":"grok", - "conditions":{ - "fields":{ - "type":[ - "flume_handler" - ] - - } - - }, - "log4j_format":"", - "multiline_pattern":"^(%{USER_SYNC_DATE:logtime})", - "message_pattern":"(?m)^%{USER_SYNC_DATE:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}\\(%{JAVACLASS:class}\\.%{JAVAMETHOD:method}:%{INT:line_number}\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", - "post_map_values":{ - "logtime":{ - "map_date":{ - "target_date_pattern":"dd MMM yyyy HH:mm:ss,SSS" - } - - } - - } - - } - - ] - -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/e734a7bf/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hbase.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hbase.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hbase.json.j2 deleted file mode 100644 index 0c04052..0000000 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hbase.json.j2 +++ /dev/null @@ -1,91 +0,0 @@ -{# - # Licensed to the Apache Software Foundation (ASF) under one - # or more contributor license agreements. See the NOTICE file - # distributed with this work for additional information - # regarding copyright ownership. The ASF licenses this file - # to you under the Apache License, Version 2.0 (the - # "License"); you may not use this file except in compliance - # with the License. You may obtain a copy of the License at - # - # http://www.apache.org/licenses/LICENSE-2.0 - # - # Unless required by applicable law or agreed to in writing, software - # distributed under the License is distributed on an "AS IS" BASIS, - # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - # See the License for the specific language governing permissions and - # limitations under the License. - #} -{ - "input":[ - { - "type":"hbase_master", - "rowtype":"service", - "path":"{{hbase_log_dir}}/hbase-*-master-*.log" - }, - { - "type":"hbase_regionserver", - "rowtype":"service", - "path":"{{hbase_log_dir}}/hbase-*-regionserver-*.log" - }, - { - "type":"hbase_phoenix_server", - "rowtype":"service", - "path":"{{hbase_log_dir}}/phoenix-*-server.log" - } - - ], - "filter":[ - { - "filter":"grok", - "conditions":{ - "fields":{ - "type":[ - "hbase_master", - "hbase_regionserver" - ] - - } - - }, - "log4j_format":"%d{ISO8601} %-5p [%t] %c{2}: %m%n", - "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", - "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{JAVACLASS:logger_name}:%{SPACE}%{GREEDYDATA:log_message}", - "post_map_values":{ - "logtime":{ - "map_date":{ - "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" - } - - } - - } - - }, - { - "filter":"grok", - "conditions":{ - "fields":{ - "type":[ - "hbase_phoenix_server" - ] - - } - }, - "log4j_format":"%d{ISO8601} %-5p [%t] %c{2}: %m%n", - "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", - "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}:%{SPACE}%{GREEDYDATA:log_message}", - "post_map_values":{ - "logtime":{ - "map_date":{ - "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" - } - - } - - } - - } - - ] - -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/e734a7bf/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hdfs.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hdfs.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hdfs.json.j2 deleted file mode 100644 index 3dff6f5..0000000 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hdfs.json.j2 +++ /dev/null @@ -1,251 +0,0 @@ -{# - # Licensed to the Apache Software Foundation (ASF) under one - # or more contributor license agreements. See the NOTICE file - # distributed with this work for additional information - # regarding copyright ownership. The ASF licenses this file - # to you under the Apache License, Version 2.0 (the - # "License"); you may not use this file except in compliance - # with the License. You may obtain a copy of the License at - # - # http://www.apache.org/licenses/LICENSE-2.0 - # - # Unless required by applicable law or agreed to in writing, software - # distributed under the License is distributed on an "AS IS" BASIS, - # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - # See the License for the specific language governing permissions and - # limitations under the License. - #} -{ - "input":[ - { - "type":"hdfs_datanode", - "rowtype":"service", - "path":"{{hdfs_log_dir_prefix}}/{{hdfs_user}}/hadoop-{{hdfs_user}}-datanode-*.log" - }, - { - "type":"hdfs_namenode", - "rowtype":"service", - "path":"{{hdfs_log_dir_prefix}}/{{hdfs_user}}/hadoop-{{hdfs_user}}-namenode-*.log" - }, - { - "type":"hdfs_journalnode", - "rowtype":"service", - "path":"{{hdfs_log_dir_prefix}}/{{hdfs_user}}/hadoop-{{hdfs_user}}-journalnode-*.log" - }, - { - "type":"hdfs_secondarynamenode", - "rowtype":"service", - "path":"{{hdfs_log_dir_prefix}}/{{hdfs_user}}/hadoop-{{hdfs_user}}-secondarynamenode-*.log" - }, - { - "type":"hdfs_zkfc", - "rowtype":"service", - "path":"{{hdfs_log_dir_prefix}}/{{hdfs_user}}/hadoop-{{hdfs_user}}-zkfc-*.log" - }, - { - "type":"hdfs_nfs3", - "rowtype":"service", - "path":"{{hdfs_log_dir_prefix}}/{{hdfs_user}}/hadoop-{{hdfs_user}}-nfs3-*.log" - }, - { - "type":"hdfs_audit", - "rowtype":"audit", - "is_enabled":"true", - "add_fields":{ - "logType":"HDFSAudit", - "enforcer":"hadoop-acl", - "repoType":"1", - "repo":"hdfs" - }, - "path":"{{hdfs_log_dir_prefix}}/{{hdfs_user}}/hdfs-audit.log" - } - - ], - "filter":[ - { - "filter":"grok", - "conditions":{ - "fields":{ - "type":[ - "hdfs_datanode", - "hdfs_journalnode", - "hdfs_secondarynamenode", - "hdfs_namenode", - "hdfs_zkfc", - "hdfs_nfs3" - ] - - } - - }, - "log4j_format":"%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n", - "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", - "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\(%{JAVAFILE:file}:%{JAVAMETHOD:method}\\(%{INT:line_number}\\)\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", - "post_map_values":{ - "logtime":{ - "map_date":{ - "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" - } - } - - } - - }, - { - "filter":"grok", - "conditions":{ - "fields":{ - "type":[ - "hdfs_audit" - ] - - } - - }, - "log4j_format":"%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n", - "multiline_pattern":"^(%{TIMESTAMP_ISO8601:evtTime})", - "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:evtTime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}:%{SPACE}%{GREEDYDATA:log_message}", - "post_map_values":{ - "evtTime":{ - "map_date":{ - "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" - } - - } - - } - - }, - { - "filter":"keyvalue", - "sort_order":1, - "conditions":{ - "fields":{ - "type":[ - "hdfs_audit" - ] - - } - - }, - "source_field":"log_message", - "value_split":"=", - "field_split":"\t", - "post_map_values":{ - "src":{ - "map_fieldname":{ - "new_fieldname":"resource" - } - - }, - "ip":{ - "map_fieldname":{ - "new_fieldname":"cliIP" - } - - }, - "allowed":[ - { - "map_fieldvalue":{ - "pre_value":"true", - "post_value":"1" - } - - }, - { - "map_fieldvalue":{ - "pre_value":"false", - "post_value":"0" - } - - }, - { - "map_fieldname":{ - "new_fieldname":"result" - } - - } - - ], - "cmd":{ - "map_fieldname":{ - "new_fieldname":"action" - } - - }, - "proto":{ - "map_fieldname":{ - "new_fieldname":"cliType" - } - - }, - "callerContext":{ - "map_fieldname":{ - "new_fieldname":"req_caller_id" - } - - } - - } - - }, - { - "filter":"grok", - "sort_order":2, - "source_field":"ugi", - "remove_source_field":"false", - "conditions":{ - "fields":{ - "type":[ - "hdfs_audit" - ] - - } - - }, - "message_pattern":"%{USERNAME:p_user}.+auth:%{USERNAME:p_authType}.+via %{USERNAME:k_user}.+auth:%{USERNAME:k_authType}|%{USERNAME:user}.+auth:%{USERNAME:authType}|%{USERNAME:x_user}", - "post_map_values":{ - "user":{ - "map_fieldname":{ - "new_fieldname":"reqUser" - } - - }, - "x_user":{ - "map_fieldname":{ - "new_fieldname":"reqUser" - } - - }, - "p_user":{ - "map_fieldname":{ - "new_fieldname":"reqUser" - } - - }, - "k_user":{ - "map_fieldname":{ - "new_fieldname":"proxyUsers" - } - - }, - "p_authType":{ - "map_fieldname":{ - "new_fieldname":"authType" - } - - }, - "k_authType":{ - "map_fieldname":{ - "new_fieldname":"proxyAuthType" - } - - } - - } - - } - - ] - -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/e734a7bf/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hive.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hive.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hive.json.j2 deleted file mode 100644 index f96b16f..0000000 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hive.json.j2 +++ /dev/null @@ -1,97 +0,0 @@ -{# - # Licensed to the Apache Software Foundation (ASF) under one - # or more contributor license agreements. See the NOTICE file - # distributed with this work for additional information - # regarding copyright ownership. The ASF licenses this file - # to you under the Apache License, Version 2.0 (the - # "License"); you may not use this file except in compliance - # with the License. You may obtain a copy of the License at - # - # http://www.apache.org/licenses/LICENSE-2.0 - # - # Unless required by applicable law or agreed to in writing, software - # distributed under the License is distributed on an "AS IS" BASIS, - # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - # See the License for the specific language governing permissions and - # limitations under the License. - #} -{ - "input":[ - { - "type":"hive_hiveserver2", - "rowtype":"service", - "path":"{{hive_log_dir}}/hiveserver2.log" - }, - { - "type":"hive_metastore", - "rowtype":"service", - "path":"{{hive_log_dir}}/hivemetastore.log" - }, - { - "type": "webhcat_server", - "rowntype":"service", - "path":"{{hcat_log_dir}}/webhcat.log" - } - ], - "filter":[ - { - "filter":"grok", - "conditions":{ - "fields":{ - "type":[ - "hive_hiveserver2", - "hive_metastore" - ] - - } - - }, - "log4j_format":"%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n", - "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", - "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]:%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\(%{JAVAFILE:file}:%{JAVAMETHOD:method}\\(%{INT:line_number}\\)\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", - "post_map_values":{ - "logtime":{ - "map_date":{ - "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" - } - - } - - } - - }, - { - "filter":"grok", - "conditions":{ - "fields":{ - "type":[ - "webhcat_server" - ] - - } - - }, - "log4j_format":" %-5p | %d{DATE} | %c | %m%n", - "multiline_pattern":"^(%{SPACE}%{LOGLEVEL:level}%{CUSTOM_SEPARATOR}%{CUSTOM_DATESTAMP:logtime})", - "message_pattern":"(?m)^%{SPACE}%{LOGLEVEL:level}%{CUSTOM_SEPARATOR}%{CUSTOM_DATESTAMP:logtime}%{CUSTOM_SEPARATOR}%{JAVACLASS:file}%{CUSTOM_SEPARATOR}%{GREEDYDATA:log_message}", - "post_map_values":{ - "logtime":{ - "map_date":{ - "target_date_pattern":"dd MMM yyyy HH:mm:ss,SSS" - } - - }, - "level":{ - "map_fieldvalue":{ - "pre_value":"WARNING", - "post_value":"WARN" - } - } - - } - - } - - ] - -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/e734a7bf/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-infra.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-infra.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-infra.json.j2 deleted file mode 100644 index 20fa49d..0000000 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-infra.json.j2 +++ /dev/null @@ -1,56 +0,0 @@ -{# - # Licensed to the Apache Software Foundation (ASF) under one - # or more contributor license agreements. See the NOTICE file - # distributed with this work for additional information - # regarding copyright ownership. The ASF licenses this file - # to you under the Apache License, Version 2.0 (the - # "License"); you may not use this file except in compliance - # with the License. You may obtain a copy of the License at - # - # http://www.apache.org/licenses/LICENSE-2.0 - # - # Unless required by applicable law or agreed to in writing, software - # distributed under the License is distributed on an "AS IS" BASIS, - # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - # See the License for the specific language governing permissions and - # limitations under the License. - #} -{ - "input":[ - { - "type":"infra_solr", - "rowtype":"service", - "path":"{{infra_solr_log_dir}}/solr.log" - } - - ], - "filter":[ - { - "filter":"grok", - "conditions":{ - "fields":{ - "type":[ - "infra_solr" - ] - - } - - }, - "log4j_format":"", - "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", - "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{LOGLEVEL:level}%{SPACE}%{GREEDYDATA:log_message}", - "post_map_values":{ - "logtime":{ - "map_date":{ - "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" - } - - } - - } - - } - - ] - -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/e734a7bf/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-kafka.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-kafka.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-kafka.json.j2 deleted file mode 100644 index d05d4ad..0000000 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-kafka.json.j2 +++ /dev/null @@ -1,105 +0,0 @@ -{# - # Licensed to the Apache Software Foundation (ASF) under one - # or more contributor license agreements. See the NOTICE file - # distributed with this work for additional information - # regarding copyright ownership. The ASF licenses this file - # to you under the Apache License, Version 2.0 (the - # "License"); you may not use this file except in compliance - # with the License. You may obtain a copy of the License at - # - # http://www.apache.org/licenses/LICENSE-2.0 - # - # Unless required by applicable law or agreed to in writing, software - # distributed under the License is distributed on an "AS IS" BASIS, - # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - # See the License for the specific language governing permissions and - # limitations under the License. - #} -{ - "input":[ - { - "type":"kafka_controller", - "rowtype":"service", - "path":"{{kafka_log_dir}}/controller.log" - }, - { - "type":"kafka_request", - "rowtype":"service", - "path":"{{kafka_log_dir}}/kafka-request.log" - }, - { - "type":"kafka_logcleaner", - "rowtype":"service", - "path":"{{kafka_log_dir}}/log-cleaner.log" - }, - { - "type":"kafka_server", - "rowtype":"service", - "path":"{{kafka_log_dir}}/server.log" - }, - { - "type":"kafka_statechange", - "rowtype":"service", - "path":"{{kafka_log_dir}}/state-change.log" - } - - ], - "filter":[ - { - "filter":"grok", - "conditions":{ - "fields":{ - "type":[ - "kafka_controller", - "kafka_request", - "kafka_logcleaner" - ] - - } - - }, - "log4j_format":"[%d] %p %m (%c)%n", - "multiline_pattern":"^(\\[%{TIMESTAMP_ISO8601:logtime}\\])", - "message_pattern":"(?m)^\\[%{TIMESTAMP_ISO8601:logtime}\\]%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{GREEDYDATA:log_message}", - "post_map_values":{ - "logtime":{ - "map_date":{ - "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" - } - - } - - } - - }, - { - "filter":"grok", - "comment":"Suppose to be same log4j pattern as other kafka processes, but some reason thread is not printed", - "conditions":{ - "fields":{ - "type":[ - "kafka_server", - "kafka_statechange" - ] - - } - - }, - "log4j_format":"[%d] %p %m (%c)%n", - "multiline_pattern":"^(\\[%{TIMESTAMP_ISO8601:logtime}\\])", - "message_pattern":"(?m)^\\[%{TIMESTAMP_ISO8601:logtime}\\]%{SPACE}%{LOGLEVEL:level}%{SPACE}%{GREEDYDATA:log_message}", - "post_map_values":{ - "logtime":{ - "map_date":{ - "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" - } - - } - - } - - } - - ] - -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/e734a7bf/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-knox.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-knox.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-knox.json.j2 deleted file mode 100644 index df98e25..0000000 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-knox.json.j2 +++ /dev/null @@ -1,68 +0,0 @@ -{# - # Licensed to the Apache Software Foundation (ASF) under one - # or more contributor license agreements. See the NOTICE file - # distributed with this work for additional information - # regarding copyright ownership. The ASF licenses this file - # to you under the Apache License, Version 2.0 (the - # "License"); you may not use this file except in compliance - # with the License. You may obtain a copy of the License at - # - # http://www.apache.org/licenses/LICENSE-2.0 - # - # Unless required by applicable law or agreed to in writing, software - # distributed under the License is distributed on an "AS IS" BASIS, - # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - # See the License for the specific language governing permissions and - # limitations under the License. - #} -{ - "input":[ - { - "type":"knox_gateway", - "rowtype":"service", - "path":"{{knox_log_dir}}/gateway.log" - }, - { - "type":"knox_cli", - "rowtype":"service", - "path":"{{knox_log_dir}}/knoxcli.log" - }, - { - "type":"knox_ldap", - "rowtype":"service", - "path":"{{knox_log_dir}}/ldap.log" - } - - ], - "filter":[ - { - "filter":"grok", - "conditions":{ - "fields":{ - "type":[ - "knox_gateway", - "knox_cli", - "knox_ldap" - ] - - } - - }, - "log4j_format":"%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n", - "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", - "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\(%{JAVAFILE:file}:%{JAVAMETHOD:method}\\(%{INT:line_number}\\)\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", - "post_map_values":{ - "logtime":{ - "map_date":{ - "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" - } - - } - - } - - } - - ] - -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/e734a7bf/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-oozie.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-oozie.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-oozie.json.j2 deleted file mode 100644 index ddb8198..0000000 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-oozie.json.j2 +++ /dev/null @@ -1,56 +0,0 @@ -{# - # Licensed to the Apache Software Foundation (ASF) under one - # or more contributor license agreements. See the NOTICE file - # distributed with this work for additional information - # regarding copyright ownership. The ASF licenses this file - # to you under the Apache License, Version 2.0 (the - # "License"); you may not use this file except in compliance - # with the License. You may obtain a copy of the License at - # - # http://www.apache.org/licenses/LICENSE-2.0 - # - # Unless required by applicable law or agreed to in writing, software - # distributed under the License is distributed on an "AS IS" BASIS, - # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - # See the License for the specific language governing permissions and - # limitations under the License. - #} -{ - "input":[ - { - "type":"oozie_app", - "rowtype":"service", - "path":"{{oozie_log_dir}}/oozie.log" - } - - ], - "filter":[ - { - "filter":"grok", - "conditions":{ - "fields":{ - "type":[ - "oozie_app" - ] - - } - - }, - "log4j_format":"%d{ISO8601} %5p %c{1}:%L - SERVER[${oozie.instance.id}] %m%n", - "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", - "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{DATA:logger_name}:%{INT:line_number}%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", - "post_map_values":{ - "logtime":{ - "map_date":{ - "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" - } - - } - - } - - } - - ] - -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/e734a7bf/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-ranger.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-ranger.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-ranger.json.j2 deleted file mode 100644 index 7b12869..0000000 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-ranger.json.j2 +++ /dev/null @@ -1,122 +0,0 @@ -{# - # Licensed to the Apache Software Foundation (ASF) under one - # or more contributor license agreements. See the NOTICE file - # distributed with this work for additional information - # regarding copyright ownership. The ASF licenses this file - # to you under the Apache License, Version 2.0 (the - # "License"); you may not use this file except in compliance - # with the License. You may obtain a copy of the License at - # - # http://www.apache.org/licenses/LICENSE-2.0 - # - # Unless required by applicable law or agreed to in writing, software - # distributed under the License is distributed on an "AS IS" BASIS, - # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - # See the License for the specific language governing permissions and - # limitations under the License. - #} -{ - "input":[ - { - "type":"ranger_admin", - "rowtype":"service", - "path":"{{ranger_admin_log_dir}}/xa_portal.log" - }, - { - "type":"ranger_dbpatch", - "is_enabled":"true", - "path":"{{ranger_admin_log_dir}}/ranger_db_patch.log" - }, - { - "type":"ranger_kms", - "rowtype":"service", - "path":"{{ranger_kms_log_dir}}/kms.log" - }, - { - "type":"ranger_usersync", - "rowtype":"service", - "path":"{{ranger_usersync_log_dir}}/usersync.log" - } - - ], - "filter":[ - { - "filter":"grok", - "conditions":{ - "fields":{ - "type":[ - "ranger_admin", - "ranger_dbpatch" - ] - - } - - }, - "log4j_format":"%d [%t] %-5p %C{6} (%F:%L) - %m%n", - "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", - "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\(%{JAVAFILE:file}:%{INT:line_number}\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", - "post_map_values":{ - "logtime":{ - "map_date":{ - "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" - } - - } - - } - - }, - { - "filter":"grok", - "conditions":{ - "fields":{ - "type":[ - "ranger_kms" - ] - - } - - }, - "log4j_format":"%d{ISO8601} %-5p %c{1} - %m%n", - "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", - "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", - "post_map_values":{ - "logtime":{ - "map_date":{ - "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" - } - - } - - } - - }, - { - "filter":"grok", - "conditions":{ - "fields":{ - "type":[ - "ranger_usersync" - ] - - } - - }, - "log4j_format":"%d{dd MMM yyyy HH:mm:ss} %5p %c{1} [%t] - %m%n", - "multiline_pattern":"^(%{USER_SYNC_DATE:logtime})", - "message_pattern":"(?m)^%{USER_SYNC_DATE:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", - "post_map_values":{ - "logtime":{ - "map_date":{ - "target_date_pattern":"dd MMM yyyy HH:mm:ss" - } - - } - - } - - } - - ] - -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/e734a7bf/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-spark.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-spark.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-spark.json.j2 deleted file mode 100644 index 4371276..0000000 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-spark.json.j2 +++ /dev/null @@ -1,72 +0,0 @@ -{# - # Licensed to the Apache Software Foundation (ASF) under one - # or more contributor license agreements. See the NOTICE file - # distributed with this work for additional information - # regarding copyright ownership. The ASF licenses this file - # to you under the Apache License, Version 2.0 (the - # "License"); you may not use this file except in compliance - # with the License. You may obtain a copy of the License at - # - # http://www.apache.org/licenses/LICENSE-2.0 - # - # Unless required by applicable law or agreed to in writing, software - # distributed under the License is distributed on an "AS IS" BASIS, - # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - # See the License for the specific language governing permissions and - # limitations under the License. - #} - { - "input":[ - - { - "type":"spark_jobhistory_server", - "rowtype":"service", - "path":"{{spark_log_dir}}/spark-*-org.apache.spark.deploy.history.HistoryServer*.out" - }, - { - "type":"spark_thriftserver", - "rowtype":"service", - "path":"{{spark_log_dir}}/spark-*-org.apache.spark.sql.hive.thriftserver.HiveThriftServer2*.out" - }, - { - "type":"livy_server", - "rowtype":"service", - "path":"{{livy_log_dir}}/livy-livy-server.out" - } - ], - "filter":[ - { - "filter":"grok", - "conditions":{ - "fields":{ - "type":[ - "spark_jobhistory_server", - "spark_thriftserver", - "livy_server" - ] - - } - }, - "log4j_format":"", - "multiline_pattern":"^(%{SPARK_DATESTAMP:logtime}%{SPACE}%{LOGLEVEL:level})", - "message_pattern":"(?m)^%{SPARK_DATESTAMP:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVAFILE:file}:%{SPACE}%{GREEDYDATA:log_message}", - "post_map_values":{ - "logtime":{ - "map_date":{ - "target_date_pattern":"yy/MM/dd HH:mm:ss" - } - - }, - "level":{ - "map_fieldvalue":{ - "pre_value":"WARNING", - "post_value":"WARN" - } - - } - - } - } - ] - - } http://git-wip-us.apache.org/repos/asf/ambari/blob/e734a7bf/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-spark2.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-spark2.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-spark2.json.j2 deleted file mode 100644 index 01aea36..0000000 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-spark2.json.j2 +++ /dev/null @@ -1,66 +0,0 @@ -{# - # Licensed to the Apache Software Foundation (ASF) under one - # or more contributor license agreements. See the NOTICE file - # distributed with this work for additional information - # regarding copyright ownership. The ASF licenses this file - # to you under the Apache License, Version 2.0 (the - # "License"); you may not use this file except in compliance - # with the License. You may obtain a copy of the License at - # - # http://www.apache.org/licenses/LICENSE-2.0 - # - # Unless required by applicable law or agreed to in writing, software - # distributed under the License is distributed on an "AS IS" BASIS, - # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - # See the License for the specific language governing permissions and - # limitations under the License. - #} - { - "input":[ - - { - "type":"spark2_jobhistory_server", - "rowtype":"service", - "path":"{{spark2_log_dir}}/spark-*-org.apache.spark.deploy.history.HistoryServer*.out" - }, - { - "type":"spark2_thriftserver", - "rowtype":"service", - "path":"{{spark2_log_dir}}/spark-*-org.apache.spark.sql.hive.thriftserver.HiveThriftServer2*.out" - } - ], - "filter":[ - { - "filter":"grok", - "conditions":{ - "fields":{ - "type":[ - "spark2_jobhistory_server", - "spark2_thriftserver" - ] - - } - }, - "log4j_format":"", - "multiline_pattern":"^(%{SPARK_DATESTAMP:logtime}%{SPACE}%{LOGLEVEL:level})", - "message_pattern":"(?m)^%{SPARK_DATESTAMP:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVAFILE:file}:%{SPACE}%{GREEDYDATA:log_message}", - "post_map_values":{ - "logtime":{ - "map_date":{ - "target_date_pattern":"yy/MM/dd HH:mm:ss" - } - - }, - "level":{ - "map_fieldvalue":{ - "pre_value":"WARNING", - "post_value":"WARN" - } - - } - - } - } - ] - - } http://git-wip-us.apache.org/repos/asf/ambari/blob/e734a7bf/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-storm.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-storm.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-storm.json.j2 deleted file mode 100644 index 07a4c2e..0000000 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-storm.json.j2 +++ /dev/null @@ -1,86 +0,0 @@ -{# - # Licensed to the Apache Software Foundation (ASF) under one - # or more contributor license agreements. See the NOTICE file - # distributed with this work for additional information - # regarding copyright ownership. The ASF licenses this file - # to you under the Apache License, Version 2.0 (the - # "License"); you may not use this file except in compliance - # with the License. You may obtain a copy of the License at - # - # http://www.apache.org/licenses/LICENSE-2.0 - # - # Unless required by applicable law or agreed to in writing, software - # distributed under the License is distributed on an "AS IS" BASIS, - # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - # See the License for the specific language governing permissions and - # limitations under the License. - #} -{ - "input":[ - { - "type":"storm_drpc", - "rowtype":"service", - "path":"{{storm_log_dir}}/drpc.log" - }, - { - "type":"storm_logviewer", - "rowtype":"service", - "path":"{{storm_log_dir}}/logviewer.log" - }, - { - "type":"storm_nimbus", - "rowtype":"service", - "path":"{{storm_log_dir}}/nimbus.log" - }, - { - "type":"storm_supervisor", - "rowtype":"service", - "path":"{{storm_log_dir}}/supervisor.log" - }, - { - "type":"storm_ui", - "rowtype":"service", - "path":"{{storm_log_dir}}/ui.log" - }, - { - "type":"storm_worker", - "rowtype":"service", - "path":"{{storm_log_dir}}/*worker*.log" - } - - ], - "filter":[ - { - "filter":"grok", - "conditions":{ - "fields":{ - "type":[ - "storm_drpc", - "storm_logviewer", - "storm_nimbus", - "storm_supervisor", - "storm_ui", - "storm_worker" - ] - - } - - }, - "log4j_format":"", - "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", - "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\[%{LOGLEVEL:level}\\]%{SPACE}%{SPACE}%{GREEDYDATA:log_message}", - "post_map_values":{ - "logtime":{ - "map_date":{ - "target_date_pattern":"yyyy-MM-dd HH:mm:ss.SSS" - } - - } - - } - - } - - ] - -} \ No newline at end of file
