http://git-wip-us.apache.org/repos/asf/ambari/blob/559fc2e4/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-yarn.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-yarn.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-yarn.json.j2 deleted file mode 100644 index f8e77ae..0000000 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-yarn.json.j2 +++ /dev/null @@ -1,86 +0,0 @@ -{# - # Licensed to the Apache Software Foundation (ASF) under one - # or more contributor license agreements. See the NOTICE file - # distributed with this work for additional information - # regarding copyright ownership. The ASF licenses this file - # to you under the Apache License, Version 2.0 (the - # "License"); you may not use this file except in compliance - # with the License. You may obtain a copy of the License at - # - # http://www.apache.org/licenses/LICENSE-2.0 - # - # Unless required by applicable law or agreed to in writing, software - # distributed under the License is distributed on an "AS IS" BASIS, - # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - # See the License for the specific language governing permissions and - # limitations under the License. - #} -{ - "input":[ - { - "type":"mapred_historyserver", - "rowtype":"service", - "path":"{{mapred_log_dir_prefix}}/{{mapred_user}}/mapred-{{mapred_user}}-historyserver*.log" - }, - { - "type":"yarn_nodemanager", - "rowtype":"service", - "path":"{{yarn_log_dir_prefix}}/{{yarn_user}}/yarn-{{yarn_user}}-nodemanager-*.log" - }, - { - "type":"yarn_resourcemanager", - "rowtype":"service", - "path":"{{yarn_log_dir_prefix}}/{{yarn_user}}/yarn-{{yarn_user}}-resourcemanager-*.log" - }, - { - "type":"yarn_timelineserver", - "rowtype":"service", - "path":"{{yarn_log_dir_prefix}}/{{yarn_user}}/yarn-{{yarn_user}}-timelineserver-*.log" - }, - { - "type":"yarn_historyserver", - "rowtype":"service", - "path":"{{yarn_log_dir_prefix}}/{{yarn_user}}/yarn-{{yarn_user}}-historyserver-*.log" - }, - { - "type":"yarn_jobsummary", - "rowtype":"service", - "path":"{{yarn_log_dir_prefix}}/{{yarn_user}}/hadoop-mapreduce.jobsummary.log" - } - - ], - "filter":[ - { - "filter":"grok", - "conditions":{ - "fields":{ - "type":[ - "mapred_historyserver", - "yarn_historyserver", - "yarn_jobsummary", - "yarn_nodemanager", - "yarn_resourcemanager", - "yarn_timelineserver" - ] - - } - - }, - "log4j_format":"%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n", - "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", - "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\(%{JAVAFILE:file}:%{JAVAMETHOD:method}\\(%{INT:line_number}\\)\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", - "post_map_values":{ - "logtime":{ - "map_date":{ - "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" - } - - } - - } - - } - - ] - -} \ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/559fc2e4/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-zeppelin.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-zeppelin.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-zeppelin.json.j2 deleted file mode 100644 index c0948b5..0000000 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-zeppelin.json.j2 +++ /dev/null @@ -1,56 +0,0 @@ -{# - # Licensed to the Apache Software Foundation (ASF) under one - # or more contributor license agreements. See the NOTICE file - # distributed with this work for additional information - # regarding copyright ownership. The ASF licenses this file - # to you under the Apache License, Version 2.0 (the - # "License"); you may not use this file except in compliance - # with the License. You may obtain a copy of the License at - # - # http://www.apache.org/licenses/LICENSE-2.0 - # - # Unless required by applicable law or agreed to in writing, software - # distributed under the License is distributed on an "AS IS" BASIS, - # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - # See the License for the specific language governing permissions and - # limitations under the License. - #} -{ - "input":[ - { - "type":"zeppelin", - "rowtype":"service", - "path":"{{zeppelin_log_dir}}/zeppelin-zeppelin-*.log" - } - - ], - "filter":[ - { - "filter":"grok", - "conditions":{ - "fields":{ - "type":[ - "zeppelin" - ] - - } - - }, - "log4j_format":"", - "multiline_pattern":"^(%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{TIMESTAMP_ISO8601:logtime}\\])", - "message_pattern":"(?m)^%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{TIMESTAMP_ISO8601:logtime}\\]%{SPACE}\\(\\{{"{"}}%{DATA:thread_name}\\{{"}"}}%{SPACE}%{JAVAFILE:file}\\[%{JAVAMETHOD:method}\\]:%{INT:line_number}\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", - "post_map_values":{ - "logtime":{ - "map_date":{ - "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" - } - - } - - } - - } - - ] - -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/559fc2e4/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-zookeeper.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-zookeeper.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-zookeeper.json.j2 deleted file mode 100644 index fcc5dc9..0000000 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-zookeeper.json.j2 +++ /dev/null @@ -1,56 +0,0 @@ -{# - # Licensed to the Apache Software Foundation (ASF) under one - # or more contributor license agreements. See the NOTICE file - # distributed with this work for additional information - # regarding copyright ownership. The ASF licenses this file - # to you under the Apache License, Version 2.0 (the - # "License"); you may not use this file except in compliance - # with the License. You may obtain a copy of the License at - # - # http://www.apache.org/licenses/LICENSE-2.0 - # - # Unless required by applicable law or agreed to in writing, software - # distributed under the License is distributed on an "AS IS" BASIS, - # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - # See the License for the specific language governing permissions and - # limitations under the License. - #} -{ - "input":[ - { - "type":"zookeeper", - "rowtype":"service", - "path":"{{zk_log_dir}}/zookeeper*.log" - } - - ], - "filter":[ - { - "filter":"grok", - "conditions":{ - "fields":{ - "type":[ - "zookeeper" - ] - - } - - }, - "log4j_format":"%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n", - "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", - "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}-%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\@%{INT:line_number}\\]%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", - "post_map_values":{ - "logtime":{ - "map_date":{ - "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" - } - - } - - } - - } - - ] - -} http://git-wip-us.apache.org/repos/asf/ambari/blob/559fc2e4/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/secure_log.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/secure_log.json b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/secure_log.json new file mode 100644 index 0000000..9d9585e --- /dev/null +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/secure_log.json @@ -0,0 +1,31 @@ +{ + "input": [ + { + "type": "secure_log", + "rowtype": "service", + "path": "/var/log/secure" + } + ], + "filter": [ + { + "filter": "grok", + "conditions": { + "fields": { + "type": [ + "secure_log" + ] + } + }, + "multiline_pattern": "^(%{SYSLOGTIMESTAMP:logtime})", + "message_pattern": "(?m)^%{SYSLOGTIMESTAMP:logtime}%{SPACE}%{SYSLOGHOST:host}%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values": { + "logtime": { + "map_date": { + "target_date_pattern": "yyyy-MM-dd HH:mm:ss,SSS", + "src_date_pattern" :"MMM dd HH:mm:ss" + } + } + } + } + ] +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/559fc2e4/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/system_messages.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/system_messages.json b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/system_messages.json new file mode 100644 index 0000000..e2ea3eb --- /dev/null +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/system_messages.json @@ -0,0 +1,31 @@ +{ + "input": [ + { + "type": "system_message", + "rowtype": "service", + "path": "/var/log/messages" + } + ], + "filter": [ + { + "filter": "grok", + "conditions": { + "fields": { + "type": [ + "system_message" + ] + } + }, + "multiline_pattern": "^(%{SYSLOGTIMESTAMP:logtime})", + "message_pattern": "(?m)^%{SYSLOGTIMESTAMP:logtime}%{SPACE}%{SYSLOGHOST:host}%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values": { + "logtime": { + "map_date": { + "target_date_pattern": "yyyy-MM-dd HH:mm:ss,SSS", + "src_date_pattern" :"MMM dd HH:mm:ss" + } + } + } + } + ] +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/559fc2e4/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/configuration/oozie-logsearch-conf.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/configuration/oozie-logsearch-conf.xml b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/configuration/oozie-logsearch-conf.xml new file mode 100644 index 0000000..2f13d3f --- /dev/null +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/configuration/oozie-logsearch-conf.xml @@ -0,0 +1,80 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration supports_final="false" supports_adding_forbidden="true"> + <property> + <name>service_name</name> + <display-name>Service name</display-name> + <description>Service name for Logsearch Portal (label)</description> + <value>Oozie</value> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>component_mappings</name> + <display-name>Component mapping</display-name> + <description>Logsearch component logid mapping list (e.g.: COMPONENT1:logid1,logid2;COMPONENT2:logid3)</description> + <value>OOZIE_SERVER:oozie_app</value> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>content</name> + <display-name>Logfeeder Config</display-name> + <description>Metadata jinja template for Logfeeder which contains grok patterns for reading service specific logs.</description> + <value> +{ + "input":[ + { + "type":"oozie_app", + "rowtype":"service", + "path":"{{default('/configurations/oozie-env/oozie_log_dir', '/var/log/oozie')}}/oozie.log" + } + ], + "filter":[ + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "oozie_app" + ] + } + }, + "log4j_format":"%d{ISO8601} %5p %c{1}:%L - SERVER[${oozie.instance.id}] %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{DATA:logger_name}:%{INT:line_number}%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + } + } + } + ] + } + </value> + <value-attributes> + <type>content</type> + <show-property-name>false</show-property-name> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/559fc2e4/ambari-server/src/main/resources/common-services/RANGER/0.4.0/configuration/ranger-logsearch-conf.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/configuration/ranger-logsearch-conf.xml b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/configuration/ranger-logsearch-conf.xml new file mode 100644 index 0000000..fc26336 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/configuration/ranger-logsearch-conf.xml @@ -0,0 +1,111 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration supports_final="false" supports_adding_forbidden="true"> + <property> + <name>service_name</name> + <display-name>Service name</display-name> + <description>Service name for Logsearch Portal (label)</description> + <value>Ranger</value> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>component_mappings</name> + <display-name>Component mapping</display-name> + <description>Logsearch component logid mapping list (e.g.: COMPONENT1:logid1,logid2;COMPONENT2:logid3)</description> + <value>RANGER_SERVER:ranger_admin,ranger_dbpatch;RANGER_USERSYNC:ranger_usersync;</value> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>content</name> + <display-name>Logfeeder Config</display-name> + <description>Metadata jinja template for Logfeeder which contains grok patterns for reading service specific logs.</description> + <value> +{ + "input":[ + { + "type":"ranger_admin", + "rowtype":"service", + "path":"{{default('/configurations/ranger-env/ranger_admin_log_dir', '/var/log/ranger/admin')}}/xa_portal.log" + }, + { + "type":"ranger_dbpatch", + "is_enabled":"true", + "path":"{{default('/configurations/ranger-env/ranger_admin_log_dir', '/var/log/ranger/admin')}}/ranger_db_patch.log" + }, + { + "type":"ranger_usersync", + "rowtype":"service", + "path":"{{default('/configurations/ranger-env/ranger_usersync_log_dir', '/var/log/ranger/usersync')}}/usersync.log" + } + ], + "filter":[ + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "ranger_admin", + "ranger_dbpatch" + ] + } + }, + "log4j_format":"%d [%t] %-5p %C{6} (%F:%L) - %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\(%{JAVAFILE:file}:%{INT:line_number}\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + } + } + }, + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "ranger_usersync" + ] + } + }, + "log4j_format":"%d{dd MMM yyyy HH:mm:ss} %5p %c{1} [%t] - %m%n", + "multiline_pattern":"^(%{USER_SYNC_DATE:logtime})", + "message_pattern":"(?m)^%{USER_SYNC_DATE:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "target_date_pattern":"dd MMM yyyy HH:mm:ss" + } + } + } + } + ] + } + </value> + <value-attributes> + <type>content</type> + <show-property-name>false</show-property-name> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/559fc2e4/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/configuration/ranger-kms-logsearch-conf.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/configuration/ranger-kms-logsearch-conf.xml b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/configuration/ranger-kms-logsearch-conf.xml new file mode 100644 index 0000000..226d2b0 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/configuration/ranger-kms-logsearch-conf.xml @@ -0,0 +1,80 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration supports_final="false" supports_adding_forbidden="true"> + <property> + <name>service_name</name> + <display-name>Service name</display-name> + <description>Service name for Logsearch Portal (label)</description> + <value>Ranger KMS</value> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>component_mappings</name> + <display-name>Component mapping</display-name> + <description>Logsearch component logid mapping list (e.g.: COMPONENT1:logid1,logid2;COMPONENT2:logid3)</description> + <value>RANGER_KMS:ranger_kms</value> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>content</name> + <display-name>Logfeeder Config</display-name> + <description>Metadata jinja template for Logfeeder which contains grok patterns for reading service specific logs.</description> + <value> +{ + "input":[ + { + "type":"ranger_kms", + "rowtype":"service", + "path":"{{default('/configurations/kms-env/kms_log_dir', '/var/log/ranger/kms')}}/kms.log" + } + ], + "filter":[ + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "ranger_kms" + ] + } + }, + "log4j_format":"%d{ISO8601} %-5p %c{1} - %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + } + } + } + ] + } + </value> + <value-attributes> + <type>content</type> + <show-property-name>false</show-property-name> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/559fc2e4/ambari-server/src/main/resources/common-services/SPARK/1.2.1/configuration/spark-logsearch-conf.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.1/configuration/spark-logsearch-conf.xml b/ambari-server/src/main/resources/common-services/SPARK/1.2.1/configuration/spark-logsearch-conf.xml new file mode 100644 index 0000000..63201ef --- /dev/null +++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.1/configuration/spark-logsearch-conf.xml @@ -0,0 +1,98 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration supports_final="false" supports_adding_forbidden="true"> + <property> + <name>service_name</name> + <display-name>Service name</display-name> + <description>Service name for Logsearch Portal (label)</description> + <value>Spark</value> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>component_mappings</name> + <display-name>Component mapping</display-name> + <description>Logsearch component logid mapping list (e.g.: COMPONENT1:logid1,logid2;COMPONENT2:logid3)</description> + <value>SPARK_JOBHISTORYSERVER:spark_jobhistory_server;SPARK_THRIFTSERVER:spark_thriftserver;LIVY_SERVER:livy_server</value> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>content</name> + <display-name>Logfeeder Config</display-name> + <description>Metadata jinja template for Logfeeder which contains grok patterns for reading service specific logs.</description> + <value> +{ + "input":[ + { + "type":"spark_jobhistory_server", + "rowtype":"service", + "path":"{{default('/configurations/spark-env/spark_log_dir', '/var/log/spark')}}/spark-*-org.apache.spark.deploy.history.HistoryServer*.out" + }, + { + "type":"spark_thriftserver", + "rowtype":"service", + "path":"{{default('/configurations/spark-env/spark_log_dir', '/var/log/spark')}}/spark-*-org.apache.spark.sql.hive.thriftserver.HiveThriftServer2*.out" + }, + { + "type":"livy_server", + "rowtype":"service", + "path":"{{default('/configurations/livy-env/livy_log_dir', '/var/log/livy')}}/livy-livy-server.out" + } + ], + "filter":[ + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "spark_jobhistory_server", + "spark_thriftserver", + "livy_server" + ] + } + }, + "log4j_format":"", + "multiline_pattern":"^(%{SPARK_DATESTAMP:logtime}%{SPACE}%{LOGLEVEL:level})", + "message_pattern":"(?m)^%{SPARK_DATESTAMP:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVAFILE:file}:%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "target_date_pattern":"yy/MM/dd HH:mm:ss" + } + }, + "level":{ + "map_fieldvalue":{ + "pre_value":"WARNING", + "post_value":"WARN" + } + } + } + } + ] +} + </value> + <value-attributes> + <type>content</type> + <show-property-name>false</show-property-name> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/559fc2e4/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/configuration/spark2-logsearch-conf.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/configuration/spark2-logsearch-conf.xml b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/configuration/spark2-logsearch-conf.xml new file mode 100644 index 0000000..3b00d51 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/configuration/spark2-logsearch-conf.xml @@ -0,0 +1,92 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration supports_final="false" supports_adding_forbidden="true"> + <property> + <name>service_name</name> + <display-name>Service name</display-name> + <description>Service name for Logsearch Portal (label)</description> + <value>Spark2</value> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>component_mappings</name> + <display-name>Component mapping</display-name> + <description>Logsearch component logid mapping list (e.g.: COMPONENT1:logid1,logid2;COMPONENT2:logid3)</description> + <value>SPARK2_JOBHISTORYSERVER:spark2_jobhistory_server;SPARK2_THRIFTSERVER:spark2_thriftserver</value> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>content</name> + <display-name>Logfeeder Config</display-name> + <description>Metadata jinja template for Logfeeder which contains grok patterns for reading service specific logs.</description> + <value> +{ + "input":[ + { + "type":"spark2_jobhistory_server", + "rowtype":"service", + "path":"{{default('/configurations/spark2-env/spark_log_dir', '/var/log/spark2')}}/spark-*-org.apache.spark.deploy.history.HistoryServer*.out" + }, + { + "type":"spark2_thriftserver", + "rowtype":"service", + "path":"{{default('/configurations/spark2-env/spark_log_dir', '/var/log/spark2')}}/spark-*-org.apache.spark.sql.hive.thriftserver.HiveThriftServer2*.out" + } + ], + "filter":[ + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "spark2_jobhistory_server", + "spark2_thriftserver" + ] + } + }, + "log4j_format":"", + "multiline_pattern":"^(%{SPARK_DATESTAMP:logtime}%{SPACE}%{LOGLEVEL:level})", + "message_pattern":"(?m)^%{SPARK_DATESTAMP:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVAFILE:file}:%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "target_date_pattern":"yy/MM/dd HH:mm:ss" + } + }, + "level":{ + "map_fieldvalue":{ + "pre_value":"WARNING", + "post_value":"WARN" + } + } + } + } + ] +} + </value> + <value-attributes> + <type>content</type> + <show-property-name>false</show-property-name> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/559fc2e4/ambari-server/src/main/resources/common-services/STORM/0.9.1/configuration/storm-logsearch-conf.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1/configuration/storm-logsearch-conf.xml b/ambari-server/src/main/resources/common-services/STORM/0.9.1/configuration/storm-logsearch-conf.xml new file mode 100644 index 0000000..d485017 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1/configuration/storm-logsearch-conf.xml @@ -0,0 +1,110 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration supports_final="false" supports_adding_forbidden="true"> + <property> + <name>service_name</name> + <display-name>Service name</display-name> + <description>Service name for Logsearch Portal (label)</description> + <value>Storm</value> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>component_mappings</name> + <display-name>Component mapping</display-name> + <description>Logsearch component logid mapping list (e.g.: COMPONENT1:logid1,logid2;COMPONENT2:logid3)</description> + <value>NIMBUS:storm_nimbus;SUPERVISOR:storm_supervisor,storm_worker,storm_logviewer;STORM_UI_SERVER:storm_ui;DRPC_SERVER:storm_drpc</value> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>content</name> + <display-name>Logfeeder Config</display-name> + <description>Metadata jinja template for Logfeeder which contains grok patterns for reading service specific logs.</description> + <value> +{ + "input":[ + { + "type":"storm_drpc", + "rowtype":"service", + "path":"{{default('/configurations/storm-env/storm_log_dir', '/var/log/storm')}}/drpc.log" + }, + { + "type":"storm_logviewer", + "rowtype":"service", + "path":"{{default('/configurations/storm-env/storm_log_dir', '/var/log/storm')}}/logviewer.log" + }, + { + "type":"storm_nimbus", + "rowtype":"service", + "path":"{{default('/configurations/storm-env/storm_log_dir', '/var/log/storm')}}/nimbus.log" + }, + { + "type":"storm_supervisor", + "rowtype":"service", + "path":"{{default('/configurations/storm-env/storm_log_dir', '/var/log/storm')}}/supervisor.log" + }, + { + "type":"storm_ui", + "rowtype":"service", + "path":"{{default('/configurations/storm-env/storm_log_dir', '/var/log/storm')}}/ui.log" + }, + { + "type":"storm_worker", + "rowtype":"service", + "path":"{{default('/configurations/storm-env/storm_log_dir', '/var/log/storm')}}/*worker*.log" + } + ], + "filter":[ + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "storm_drpc", + "storm_logviewer", + "storm_nimbus", + "storm_supervisor", + "storm_ui", + "storm_worker" + ] + } + }, + "log4j_format":"", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\[%{LOGLEVEL:level}\\]%{SPACE}%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "target_date_pattern":"yyyy-MM-dd HH:mm:ss.SSS" + } + } + } + } + ] + } + </value> + <value-attributes> + <type>content</type> + <show-property-name>false</show-property-name> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/559fc2e4/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration-mapred/mapred-logsearch-conf.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration-mapred/mapred-logsearch-conf.xml b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration-mapred/mapred-logsearch-conf.xml new file mode 100644 index 0000000..3c0abbf --- /dev/null +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration-mapred/mapred-logsearch-conf.xml @@ -0,0 +1,80 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration supports_final="false" supports_adding_forbidden="true"> + <property> + <name>service_name</name> + <display-name>Service name</display-name> + <description>Service name for Logsearch Portal (label)</description> + <value>MapReduce</value> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>component_mappings</name> + <display-name>Component mapping</display-name> + <description>Logsearch component logid mapping list (e.g.: COMPONENT1:logid1,logid2;COMPONENT2:logid3)</description> + <value>HISTORYSERVER:mapred_historyserver</value> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>content</name> + <display-name>Logfeeder Config</display-name> + <description>Metadata jinja template for Logfeeder which contains grok patterns for reading service specific logs.</description> + <value> +{ + "input":[ + { + "type":"mapred_historyserver", + "rowtype":"service", + "path":"{{default('/configurations/mapred-env/mapred_log_dir_prefix', '/var/log/hadoop')}}/{{default('configurations/mapred-env/mapred_user', 'mapred')}}/mapred-{{default('configurations/mapred-env/mapred_user', 'mapred')}}-historyserver*.log" + } + ], + "filter":[ + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "mapred_historyserver" + ] + } + }, + "log4j_format":"%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\(%{JAVAFILE:file}:%{JAVAMETHOD:method}\\(%{INT:line_number}\\)\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + } + } + } + ] + } + </value> + <value-attributes> + <type>content</type> + <show-property-name>false</show-property-name> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/559fc2e4/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-logsearch-conf.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-logsearch-conf.xml b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-logsearch-conf.xml new file mode 100644 index 0000000..95cf0c9 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-logsearch-conf.xml @@ -0,0 +1,104 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration supports_final="false" supports_adding_forbidden="true"> + <property> + <name>service_name</name> + <display-name>Service name</display-name> + <description>Service name for Logsearch Portal (label)</description> + <value>YARN</value> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>component_mappings</name> + <display-name>Component mapping</display-name> + <description>Logsearch component logid mapping list (e.g.: COMPONENT1:logid1,logid2;COMPONENT2:logid3)</description> + <value>RESOURCEMANAGER:yarn_resourcemanager,yarn_historyserver,yarn_jobsummary;NODEMANAGER:yarn_nodemanager;APP_TIMELINE_SERVER:yarn_timelineserver</value> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>content</name> + <display-name>Logfeeder Config</display-name> + <description>Metadata jinja template for Logfeeder which contains grok patterns for reading service specific logs.</description> + <value> +{ + "input":[ + { + "type":"yarn_nodemanager", + "rowtype":"service", + "path":"{{default('/configurations/yarn-env/yarn_log_dir_prefix', '/var/log/hadoop')}}/{{default('configurations/yarn-env/yarn_user', 'yarn')}}/yarn-{{default('configurations/yarn-env/yarn_user', 'yarn')}}-nodemanager-*.log" + }, + { + "type":"yarn_resourcemanager", + "rowtype":"service", + "path":"{{default('/configurations/yarn-env/yarn_log_dir_prefix', '/var/log/hadoop')}}/{{default('configurations/yarn-env/yarn_user', 'yarn')}}/yarn-{{default('configurations/yarn-env/yarn_user', 'yarn')}}-resourcemanager-*.log" + }, + { + "type":"yarn_timelineserver", + "rowtype":"service", + "path":"{{default('/configurations/yarn-env/yarn_log_dir_prefix', '/var/log/hadoop')}}/{{default('configurations/yarn-env/yarn_user', 'yarn')}}/yarn-{{default('configurations/yarn-env/yarn_user', 'yarn')}}-timelineserver-*.log" + }, + { + "type":"yarn_historyserver", + "rowtype":"service", + "path":"{{default('/configurations/yarn-env/yarn_log_dir_prefix', '/var/log/hadoop')}}/{{default('configurations/yarn-env/yarn_user', 'yarn')}}/yarn-{{default('configurations/yarn-env/yarn_user', 'yarn')}}-historyserver-*.log" + }, + { + "type":"yarn_jobsummary", + "rowtype":"service", + "path":"{{default('/configurations/yarn-env/yarn_log_dir_prefix', '/var/log/hadoop')}}/{{default('configurations/yarn-env/yarn_user', 'yarn')}}/hadoop-mapreduce.jobsummary.log" + } + ], + "filter":[ + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "yarn_historyserver", + "yarn_jobsummary", + "yarn_nodemanager", + "yarn_resourcemanager", + "yarn_timelineserver" + ] + } + }, + "log4j_format":"%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\(%{JAVAFILE:file}:%{JAVAMETHOD:method}\\(%{INT:line_number}\\)\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + } + } + } + ] +} + </value> + <value-attributes> + <type>content</type> + <show-property-name>false</show-property-name> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/559fc2e4/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/configuration/zeppelin-logsearch-conf.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/configuration/zeppelin-logsearch-conf.xml b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/configuration/zeppelin-logsearch-conf.xml new file mode 100644 index 0000000..0b8ab7a --- /dev/null +++ b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/configuration/zeppelin-logsearch-conf.xml @@ -0,0 +1,80 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration supports_final="false" supports_adding_forbidden="true"> + <property> + <name>service_name</name> + <display-name>Service name</display-name> + <description>Service name for Logsearch Portal (label)</description> + <value>Zeppelin</value> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>component_mappings</name> + <display-name>Component mapping</display-name> + <description>Logsearch component logid mapping list (e.g.: COMPONENT1:logid1,logid2;COMPONENT2:logid3)</description> + <value>ZEPPELIN_MASTER:zeppelin</value> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>content</name> + <display-name>Logfeeder Config</display-name> + <description>Metadata jinja template for Logfeeder which contains grok patterns for reading service specific logs.</description> + <value> +{ + "input":[ + { + "type":"zeppelin", + "rowtype":"service", + "path":"{{default('/configurations/zeppelin-env/zeppelin_log_dir', '/var/log/zeppelin')}}/zeppelin-zeppelin-*.log" + } + ], + "filter":[ + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "zeppelin" + ] + } + }, + "log4j_format":"", + "multiline_pattern":"^(%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{TIMESTAMP_ISO8601:logtime}\\])", + "message_pattern":"(?m)^%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{TIMESTAMP_ISO8601:logtime}\\]%{SPACE}\\(\\{{"{"}}%{DATA:thread_name}\\{{"}"}}%{SPACE}%{JAVAFILE:file}\\[%{JAVAMETHOD:method}\\]:%{INT:line_number}\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + } + } + } + ] + } + </value> + <value-attributes> + <type>content</type> + <show-property-name>false</show-property-name> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/559fc2e4/ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5/configuration/zookeeper-logsearch-conf.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5/configuration/zookeeper-logsearch-conf.xml b/ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5/configuration/zookeeper-logsearch-conf.xml new file mode 100644 index 0000000..325af14 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5/configuration/zookeeper-logsearch-conf.xml @@ -0,0 +1,76 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration supports_final="false" supports_adding_forbidden="true"> + <property> + <name>service_name</name> + <display-name>Service name</display-name> + <description>Service name for Logsearch Portal (label)</description> + <value>Zookeeper</value> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>component_mappings</name> + <display-name>Component mapping</display-name> + <description>Logsearch component logid mapping list (e.g.: COMPONENT1:logid1,logid2;COMPONENT2:logid3)</description> + <value>ZOOKEEPER_SERVER:zookeeper</value> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>content</name> + <display-name>Logfeeder Config</display-name> + <description>Metadata jinja template for Logfeeder which contains grok patterns for reading service specific logs.</description> + <value> +{ + "input":[ + { + "type":"zookeeper", + "rowtype":"service", + "path":"{{default('/configurations/zookeeper-env/zk_log_dir', '/var/log/zookeeper')}}/zookeeper*.log" + } + ], + "filter":[ + { + "filter":"grok", + "conditions":{ + "fields":{"type":["zookeeper"]} + }, + "log4j_format":"%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}-%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\@%{INT:line_number}\\]%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values": { + "logtime": { + "map_date":{ + "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + } + } + } + ] +} + </value> + <value-attributes> + <type>content</type> + <show-property-name>false</show-property-name> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> +</configuration> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/559fc2e4/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py index ba070fe..7a13637 100644 --- a/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py +++ b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py @@ -62,7 +62,7 @@ class TestLogFeeder(RMFTestCase): ) self.assertResourceCalled('PropertiesFile', '/etc/ambari-logsearch-logfeeder/conf/logfeeder.properties', properties={'logfeeder.checkpoint.folder': '/etc/ambari-logsearch-logfeeder/conf/checkpoints', - 'logfeeder.config.files': 'global.config.json,output.config.json,input.config-accumulo.json,input.config-ambari.json,input.config-ams.json,input.config-atlas.json,input.config-falcon.json,input.config-flume.json,input.config-hbase.json,input.config-hdfs.json,input.config-hive.json,input.config-hst.json,input.config-infra.json,input.config-kafka.json,input.config-knox.json,input.config-logsearch.json,input.config-nifi.json,input.config-oozie.json,input.config-ranger.json,input.config-spark.json,input.config-spark2.json,input.config-storm.json,input.config-yarn.json,input.config-zeppelin.json,input.config-zookeeper.json', + 'logfeeder.config.files': 'global.config.json,output.config.json,input.config-ambari.json,input.config-logsearch.json,input.config-zookeeper.json', 'logfeeder.metrics.collector.hosts': '', 'logfeeder.solr.core.config.name': 'history', 'logfeeder.solr.zk_connect_string': 'c6401.ambari.apache.org:2181/infra-solr' @@ -80,9 +80,7 @@ class TestLogFeeder(RMFTestCase): encoding='utf-8' ) - logfeeder_supported_services = ['accumulo', 'ambari', 'ams', 'atlas', 'falcon', 'flume', 'hbase', 'hdfs', 'hive', 'hst', - 'infra', 'kafka', 'knox', 'logsearch', 'nifi', 'oozie', 'ranger', 'spark', 'spark2', 'storm', - 'yarn', 'zeppelin', 'zookeeper'] + logfeeder_supported_services = ['ambari','logsearch'] logfeeder_config_file_names = ['global.config.json', 'output.config.json'] + \ ['input.config-%s.json' % (tag) for tag in logfeeder_supported_services] @@ -91,6 +89,9 @@ class TestLogFeeder(RMFTestCase): self.assertResourceCalled('File', '/etc/ambari-logsearch-logfeeder/conf/' + file_name, content=Template(file_name + ".j2") ) + self.assertResourceCalled('File', '/etc/ambari-logsearch-logfeeder/conf/input.config-zookeeper.json', + content=InlineTemplate("pattern content") + ) def test_configure_default(self): self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/logfeeder.py", http://git-wip-us.apache.org/repos/asf/ambari/blob/559fc2e4/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logsearch.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logsearch.py b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logsearch.py index 8c0ed16..6b4fab8 100644 --- a/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logsearch.py +++ b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logsearch.py @@ -173,7 +173,6 @@ class TestLogSearch(RMFTestCase): stack_version = self.STACK_VERSION, target = RMFTestCase.TARGET_COMMON_SERVICES ) - self.configureResourcesCalled() self.assertNoMoreResources() http://git-wip-us.apache.org/repos/asf/ambari/blob/559fc2e4/ambari-server/src/test/python/stacks/2.4/configs/default.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.4/configs/default.json b/ambari-server/src/test/python/stacks/2.4/configs/default.json index a2c4791..c3eba53 100644 --- a/ambari-server/src/test/python/stacks/2.4/configs/default.json +++ b/ambari-server/src/test/python/stacks/2.4/configs/default.json @@ -229,6 +229,11 @@ "zoo.cfg": { "clientPort": "2181" }, + "zookeeper-logsearch-conf" : { + "service_name": "Zookeeper", + "component_mappings" : "ZOOKEEPER_SERVER:zookeeper", + "content" : "pattern content" + }, "logsearch-admin-json" : { "logsearch_admin_username" : "admin", "logsearch_admin_password" : "admin", @@ -314,9 +319,6 @@ }, "logfeeder-log4j": { "content": "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n<!--\n Licensed to the Apache Software Foundation (ASF) under one or more\n contributor license agreements. See the NOTICE file distributed with\n this work for additional information regarding copyright ownership.\n The ASF licenses this file to You under the Apache License, Version 2.0\n (the \"License\"); you may not use this file except in compliance with\n the License. You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n-->\n<!DOCTYPE log4j:configuration SYSTEM \"log4j.dtd\">\n<log4j:configuration xmlns:log4j=\"http://jakarta. apache.org/log4j/\">\n <appender name=\"console\" class=\"org.apache.log4j.ConsoleAppender\">\n <param name=\"Target\" value=\"System.out\" />\n <layout class=\"org.apache.log4j.PatternLayout\">\n <param name=\"ConversionPattern\" value=\"%d [%t] %-5p %C{6} (%F:%L) - %m%n\" />\n </layout>\n </appender>\n\n <appender name=\"rolling_file\" class=\"org.apache.log4j.RollingFileAppender\"> \n <param name=\"file\" value=\"{{logfeeder_log_dir}}/logfeeder.log\" />\n <param name=\"append\" value=\"true\" /> \n <param name=\"maxFileSize\" value=\"10MB\" /> \n <param name=\"maxBackupIndex\" value=\"10\" /> \n <layout class=\"org.apache.log4j.PatternLayout\"> \n <param name=\"ConversionPattern\" value=\"%d [%t] %-5p %C{6} (%F:%L) - %m%n\"/> \n </layout> \n </appender> \n\n <category name=\"org.apache.ambari.logfeeder\" additivity=\"false\">\n <priori ty value=\"info\" />\n <appender-ref ref=\"rolling_file\" />\n </category>\n\n <root>\n <priority value=\"warn\" />\n <appender-ref ref=\"rolling_file\" />\n </root>\n</log4j:configuration>" - }, - "logfeeder-input-configs": { - "content": "{\n \"global\":{\n \"add_fields\":{\n \"cluster\":\"{{cluster_name}}\"\n },\n \"source\":\"file\",\n \"tail\":\"true\",\n \"gen_event_md5\":\"true\",\n \"start_position\":\"beginning\"\n },\n \"input\":[\n {\n \"type\":\"accumulo_gc\",\n \"rowtype\":\"service\",\n \"path\":\"{{accumulo_log_dir}}/gc_*.log\"\n },\n {\n \"type\":\"accumulo_master\",\n \"rowtype\":\"service\",\n \"path\":\"{{accumulo_log_dir}}/master_*.log\"\n },\n {\n \"type\":\"accumulo_monitor\",\n \"rowtype\":\"service\",\n \"path\":\"{{accumulo_log_dir}}/monitor_*.log\"\n },\n {\n \"type\":\"accumulo_tracer\",\n \"rowtype\":\"service\",\n \"path\":\"{{accumulo_log_dir}}/tracer_*.log\"\n },\n {\n \"type\":\"accumulo_tserver\",\n \"rowtype\":\"service\",\n \"path\":\"{{accumulo_log_dir}}/tserver_*.log\"\n },\n {\n \"type\":\"atlas_app\",\n \"rowtype\":\"se rvice\",\n \"path\":\"{{atlas_log_dir}}/application.log\"\n },\n {\n \"type\":\"ambari_agent\",\n \"rowtype\":\"service\",\n \"path\":\"{{ambari_agent_log_dir}}/ambari-agent.log\"\n },\n {\n \"type\":\"ambari_server\",\n \"rowtype\":\"service\",\n \"path\":\"{{ambari_server_log_dir}}/ambari-server.log\"\n },\n {\n \"type\":\"ams_hbase_master\",\n \"rowtype\":\"service\",\n \"path\":\"{{metrics_collector_log_dir}}/hbase-ams-master-*.log\"\n },\n {\n \"type\":\"ams_hbase_regionserver\",\n \"rowtype\":\"service\",\n \"path\":\"{{metrics_collector_log_dir}}/hbase-ams-regionserver-*.log\"\n },\n {\n \"type\":\"ams_collector\",\n \"rowtype\":\"service\",\n \"path\":\"{{metrics_collector_log_dir}}/ambari-metrics-collector.log\"\n },\n {\n \"type\":\"falcon_app\",\n \"rowtype\":\"service\",\n \"path\":\"{{falcon_log_dir}}/falcon.application.log\"\n },\n {\ n \"type\":\"hbase_master\",\n \"rowtype\":\"service\",\n \"path\":\"{{hbase_log_dir}}/hbase-hbase-master-*.log\"\n },\n {\n \"type\":\"hbase_regionserver\",\n \"rowtype\":\"service\",\n \"path\":\"{{hbase_log_dir}}/hbase-hbase-regionserver-*.log\"\n },\n {\n \"type\":\"hdfs_datanode\",\n \"rowtype\":\"service\",\n \"path\":\"{{hdfs_log_dir_prefix}}/hdfs/hadoop-hdfs-datanode-*.log\"\n },\n {\n \"type\":\"hdfs_namenode\",\n \"rowtype\":\"service\",\n \"path\":\"{{hdfs_log_dir_prefix}}/hdfs/hadoop-hdfs-namenode-*.log\"\n },\n {\n \"type\":\"hdfs_journalnode\",\n \"rowtype\":\"service\",\n \"path\":\"{{hdfs_log_dir_prefix}}/hdfs/hadoop-hdfs-journalnode-*.log\"\n },\n {\n \"type\":\"hdfs_secondarynamenode\",\n \"rowtype\":\"service\",\n \"path\":\"{{hdfs_log_dir_prefix}}/hdfs/hadoop-hdfs-secondarynamenode-*.log\"\n },\n {\n \"type\":\"hdfs_zkfc\",\n \"r owtype\":\"service\",\n \"path\":\"{{hdfs_log_dir_prefix}}/hdfs/hadoop-hdfs-zkfc-*.log\"\n },\n {\n \"type\":\"hive_hiveserver2\",\n \"rowtype\":\"service\",\n \"path\":\"{{hive_log_dir}}/hiveserver2.log\"\n },\n {\n \"type\":\"hive_metastore\",\n \"rowtype\":\"service\",\n \"path\":\"{{hive_log_dir}}/hivemetastore.log\"\n },\n {\n \"type\":\"kafka_controller\",\n \"rowtype\":\"service\",\n \"path\":\"{{kafka_log_dir}}/controller.log\"\n },\n {\n \"type\":\"kafka_request\",\n \"rowtype\":\"service\",\n \"path\":\"{{kafka_log_dir}}/kafka-request.log\"\n },\n {\n \"type\":\"kafka_logcleaner\",\n \"rowtype\":\"service\",\n \"path\":\"{{kafka_log_dir}}/log-cleaner.log\"\n },\n {\n \"type\":\"kafka_server\",\n \"rowtype\":\"service\",\n \"path\":\"{{kafka_log_dir}}/server.log\"\n },\n {\n \"type\":\"kafka_statechange\",\n \"rowtype\":\"service \",\n \"path\":\"{{kafka_log_dir}}/state-change.log\"\n },\n {\n \"type\":\"knox_gateway\",\n \"rowtype\":\"service\",\n \"path\":\"{{knox_log_dir}}/gateway.log\"\n },\n {\n \"type\":\"knox_cli\",\n \"rowtype\":\"service\",\n \"path\":\"{{knox_log_dir}}/knoxcli.log\"\n },\n {\n \"type\":\"knox_ldap\",\n \"rowtype\":\"service\",\n \"path\":\"{{knox_log_dir}}/ldap.log\"\n },\n {\n \"type\":\"mapred_historyserver\",\n \"rowtype\":\"service\",\n \"path\":\"{{mapred_log_dir_prefix}}/mapred/mapred-mapred-historyserver*.log\"\n },\n {\n \"type\":\"logsearch_app\",\n \"rowtype\":\"service\",\n \"path\":\"{{logsearch_log_dir}}/logsearch.log\"\n },\n {\n \"type\":\"logsearch_feeder\",\n \"rowtype\":\"service\",\n \"path\":\"{{logfeeder_log_dir}}/logfeeder.log\"\n },\n {\n \"type\":\"logsearch_perf\",\n \"rowtype\":\"service\",\n \"path\":\"{{l ogsearch_log_dir}}/logsearch-performance.log\"\n },\n {\n \"type\":\"ranger_admin\",\n \"rowtype\":\"service\",\n \"path\":\"{{ranger_admin_log_dir}}/xa_portal.log\"\n },\n {\n \"type\":\"ranger_dbpatch\",\n \"is_enabled\":\"true\",\n \"path\":\"{{ranger_admin_log_dir}}/ranger_db_patch.log\"\n },\n {\n \"type\":\"ranger_kms\",\n \"rowtype\":\"service\",\n \"path\":\"{{ranger_kms_log_dir}}/kms.log\"\n },\n {\n \"type\":\"ranger_usersync\",\n \"rowtype\":\"service\",\n \"path\":\"{{ranger_usersync_log_dir}}/usersync.log\"\n },\n {\n \"type\":\"oozie_app\",\n \"rowtype\":\"service\",\n \"path\":\"{{oozie_log_dir}}/oozie.log\"\n },\n {\n \"type\":\"yarn_nodemanager\",\n \"rowtype\":\"service\",\n \"path\":\"{{yarn_log_dir_prefix}}/yarn/yarn-yarn-nodemanager-*.log\"\n },\n {\n \"type\":\"yarn_resourcemanager\",\n \"rowtype\":\"service\",\n \"p ath\":\"{{yarn_log_dir_prefix}}/yarn/yarn-yarn-resourcemanager-*.log\"\n },\n {\n \"type\":\"yarn_timelineserver\",\n \"rowtype\":\"service\",\n \"path\":\"{{yarn_log_dir_prefix}}/yarn/yarn-yarn-timelineserver-*.log\"\n },\n {\n \"type\":\"yarn_historyserver\",\n \"rowtype\":\"service\",\n \"path\":\"{{yarn_log_dir_prefix}}/yarn/yarn-yarn-historyserver-*.log\"\n },\n {\n \"type\":\"yarn_jobsummary\",\n \"rowtype\":\"service\",\n \"path\":\"{{yarn_log_dir_prefix}}/yarn/hadoop-mapreduce.jobsummary.log\"\n },\n {\n \"type\":\"storm_drpc\",\n \"rowtype\":\"service\",\n \"path\":\"{{storm_log_dir}}/drpc.log\"\n },\n {\n \"type\":\"storm_logviewer\",\n \"rowtype\":\"service\",\n \"path\":\"{{storm_log_dir}}/logviewer.log\"\n },\n {\n \"type\":\"storm_nimbus\",\n \"rowtype\":\"service\",\n \"path\":\"{{storm_log_dir}}/nimbus.log\"\n },\n {\n \"type\":\"st orm_supervisor\",\n \"rowtype\":\"service\",\n \"path\":\"{{storm_log_dir}}/supervisor.log\"\n },\n {\n \"type\":\"storm_ui\",\n \"rowtype\":\"service\",\n \"path\":\"{{storm_log_dir}}/ui.log\"\n },\n {\n \"type\":\"storm_worker\",\n \"rowtype\":\"service\",\n \"path\":\"{{storm_log_dir}}/*worker*.log\"\n },\n {\n \"type\":\"zookeeper\",\n \"rowtype\":\"service\",\n \"path\":\"{{zk_log_dir}}/zookeeper/zookeeper*.out\"\n },\n {\n \"type\":\"hdfs_audit\",\n \"rowtype\":\"audit\",\n \"is_enabled\":\"true\",\n \"add_fields\":{\n \"logType\":\"HDFSAudit\",\n \"enforcer\":\"hadoop-acl\",\n \"repoType\":\"1\",\n \"repo\":\"hdfs\"\n },\n \"path\":\"{{hdfs_log_dir_prefix}}/hdfs/hdfs-audit.log\"\n }\n \n ],\n \"filter\":[\n {\n \"filter\":\"grok\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"accumulo_mas ter\"\n ]\n \n }\n \n },\n \"log4j_format\":\"%d{ISO8601} [%-8c{2}] %-5p: %m%n\",\n \"multiline_pattern\":\"^(%{TIMESTAMP_ISO8601:logtime})\",\n \"message_pattern\":\"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}\\[%{JAVACLASS:logger_name}\\]%{SPACE}%{LOGLEVEL:level}:%{SPACE}%{GREEDYDATA:log_message}\",\n \"post_map_values\":{\n \"logtime\":{\n \"map_date\":{\n \"date_pattern\":\"yyyy-MM-dd HH:mm:ss,SSS\"\n }\n \n }\n \n }\n \n },\n {\n \"filter\":\"grok\",\n \"comment\":\"This one has one extra space after LEVEL\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"accumulo_gc\",\n \"accumulo_monitor\",\n \"accumulo_tracer\",\n \"accumulo_tserver\"\n ]\n \n }\n \n },\n \"log4j_format\":\"%d{ISO8601} [%-8c{2}] %-5p: %X{application} %m%n\",\n \"multiline_pattern\":\"^(%{TIMESTAMP_ISO8601:logtime})\",\n \"message_pattern\":\"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}\\[%{JAVACLASS:logger_name}\\]%{SPACE}%{LOGLEVEL:level}%{SPACE}:%{SPACE}%{GREEDYDATA:log_message}\",\n \"post_map_values\":{\n \"logtime\":{\n \"map_date\":{\n \"date_pattern\":\"yyyy-MM-dd HH:mm:ss,SSS\"\n }\n \n }\n \n }\n \n },\n {\n \"filter\":\"grok\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"atlas_app\",\n \"falcon_app\"\n ]\n \n }\n \n },\n \"log4j_format\":\"%d %-5p - [%t:%x] ~ %m (%c{1}:%L)%n\",\n \"multiline_pattern\":\"^(%{TIMESTAMP_ISO8601:logtime})\",\n \"message_pattern\":\"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{SPACE}-%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}~%{SPACE}%{GREEDYDATA:log_message}\",\n \"post_map_values\" :{\n \"logtime\":{\n \"map_date\":{\n \"date_pattern\":\"yyyy-MM-dd HH:mm:ss,SSS\"\n }\n \n }\n \n }\n \n },\n {\n \"filter\":\"grok\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"ams_collector\"\n ]\n \n }\n \n },\n \"log4j_format\":\"%d{ISO8601} %p %c: %m%n\",\n \"multiline_pattern\":\"^(%{TIMESTAMP_ISO8601:logtime})\",\n \"message_pattern\":\"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}:%{SPACE}%{GREEDYDATA:log_message}\",\n \"post_map_values\":{\n \"logtime\":{\n \"map_date\":{\n \"date_pattern\":\"yyyy-MM-dd HH:mm:ss,SSS\"\n }\n \n }\n \n }\n \n },\n {\n \"filter\":\"grok\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"ams_hbase_master\",\n \"ams_hbase_regionserver\",\n \"hbase_master\",\n \"hbase_regionserver\"\n ]\n \n }\n \n },\n \"log4j_format\":\"%d{ISO8601} %-5p [%t] %c{2}: %m%n\",\n \"multiline_pattern\":\"^(%{TIMESTAMP_ISO8601:logtime})\",\n \"message_pattern\":\"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{JAVACLASS:logger_name}:%{SPACE}%{GREEDYDATA:log_message}\",\n \"post_map_values\":{\n \"logtime\":{\n \"map_date\":{\n \"date_pattern\":\"yyyy-MM-dd HH:mm:ss,SSS\"\n }\n \n }\n \n }\n \n },\n {\n \"filter\":\"grok\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"ambari_agent\"\n ]\n \n }\n \n },\n \"log4j_format\":\"\",\n \"multiline_pattern\":\"^(%{LOGLEVEL:level} %{TIMESTAMP_ISO8601:logtime})\",\n \" message_pattern\":\"(?m)^%{LOGLEVEL:level} %{TIMESTAMP_ISO8601:logtime} %{JAVAFILE:file}:%{INT:line_number} - %{GREEDYDATA:log_message}\",\n \"post_map_values\":{\n \"logtime\":{\n \"map_date\":{\n \"date_pattern\":\"yyyy-MM-dd HH:mm:ss,SSS\"\n }\n \n },\n \"level\":{\n \"map_fieldvalue\":{\n \"pre_value\":\"WARNING\",\n \"post_value\":\"WARN\"\n }\n \n }\n \n }\n \n },\n {\n \"filter\":\"grok\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"ambari_server\"\n ]\n \n }\n \n },\n \"log4j_format\":\"%d{DATE} %5p [%t] %c{1}:%L - %m%n\",\n \"multiline_pattern\":\"^(%{USER_SYNC_DATE:logtime})\",\n \"message_pattern\":\"(?m)^%{USER_SYNC_DATE:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{JAVACLASS:logger_name}:%{INT:line_number} %{SPACE}-%{SPACE}%{GREEDYDATA:log_message}\",\n \"post_map_values\":{\n \"logtime\":{\n \"map_date\":{\n \"date_pattern\":\"dd MMM yyyy HH:mm:ss\"\n }\n \n }\n \n }\n \n },\n {\n \"filter\":\"grok\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"hdfs_datanode\",\n \"hdfs_journalnode\",\n \"hdfs_secondarynamenode\",\n \"hdfs_namenode\",\n \"hdfs_zkfc\",\n \"knox_gateway\",\n \"knox_cli\",\n \"knox_ldap\",\n \"mapred_historyserver\",\n \"yarn_historyserver\",\n \"yarn_jobsummary\",\n \"yarn_nodemanager\",\n \"yarn_resourcemanager\",\n \"yarn_timelineserver\"\n ]\n \n }\n \n },\n \"log4j_format\":\"%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n\",\n \"multiline_pattern\":\"^(%{TIMESTA MP_ISO8601:logtime})\",\n \"message_pattern\":\"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\(%{JAVAFILE:file}:%{JAVAMETHOD:method}\\(%{INT:line_number}\\)\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}\",\n \"post_map_values\":{\n \"logtime\":{\n \"map_date\":{\n \"date_pattern\":\"yyyy-MM-dd HH:mm:ss,SSS\"\n }\n \n }\n \n }\n \n },\n {\n \"filter\":\"grok\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"hive_hiveserver2\",\n \"hive_metastore\"\n ]\n \n }\n \n },\n \"log4j_format\":\"%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n\",\n \"multiline_pattern\":\"^(%{TIMESTAMP_ISO8601:logtime})\",\n \"message_pattern\":\"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]:%{SPACE}%{JAVACLASS:logger_name}%{SP ACE}\\(%{JAVAFILE:file}:%{JAVAMETHOD:method}\\(%{INT:line_number}\\)\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}\",\n \"post_map_values\":{\n \"logtime\":{\n \"map_date\":{\n \"date_pattern\":\"yyyy-MM-dd HH:mm:ss,SSS\"\n }\n \n }\n \n }\n \n },\n {\n \"filter\":\"grok\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"kafka_controller\",\n \"kafka_request\",\n \"kafka_logcleaner\"\n ]\n \n }\n \n },\n \"log4j_format\":\"[%d] %p %m (%c)%n\",\n \"multiline_pattern\":\"^(\\[%{TIMESTAMP_ISO8601:logtime}\\])\",\n \"message_pattern\":\"(?m)^\\[%{TIMESTAMP_ISO8601:logtime}\\]%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{GREEDYDATA:log_message}\",\n \"post_map_values\":{\n \"logtime\":{\n \"map_date\":{\n \"date_pattern\":\"yyyy-MM-dd HH:mm:ss,S SS\"\n }\n \n }\n \n }\n \n },\n {\n \"filter\":\"grok\",\n \"comment\":\"Suppose to be same log4j pattern as other kafka processes, but some reason thread is not printed\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"kafka_server\",\n \"kafka_statechange\"\n ]\n \n }\n \n },\n \"log4j_format\":\"[%d] %p %m (%c)%n\",\n \"multiline_pattern\":\"^(\\[%{TIMESTAMP_ISO8601:logtime}\\])\",\n \"message_pattern\":\"(?m)^\\[%{TIMESTAMP_ISO8601:logtime}\\]%{SPACE}%{LOGLEVEL:level}%{SPACE}%{GREEDYDATA:log_message}\",\n \"post_map_values\":{\n \"logtime\":{\n \"map_date\":{\n \"date_pattern\":\"yyyy-MM-dd HH:mm:ss,SSS\"\n }\n \n }\n \n }\n \n },\n {\n \"filter\":\"grok\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"o ozie_app\"\n ]\n \n }\n \n },\n \"log4j_format\":\"%d{ISO8601} %5p %c{1}:%L - SERVER[${oozie.instance.id}] %m%n\",\n \"multiline_pattern\":\"^(%{TIMESTAMP_ISO8601:logtime})\",\n \"message_pattern\":\"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{DATA:logger_name}:%{INT:line_number}%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}\",\n \"post_map_values\":{\n \"logtime\":{\n \"map_date\":{\n \"date_pattern\":\"yyyy-MM-dd HH:mm:ss,SSS\"\n }\n \n }\n \n }\n \n },\n {\n \"filter\":\"grok\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"logsearch_app\",\n \"logsearch_feeder\",\n \"logsearch_perf\",\n \"ranger_admin\",\n \"ranger_dbpatch\"\n ]\n \n }\n \n },\n \"log4j_format\":\"%d [%t] %-5p %C{6} (%F:%L) - %m%n\",\n \"multiline_pattern\":\"^(%{TIMESTAMP_ISO8601:logtime})\",\n \"message_pattern\":\"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\(%{JAVAFILE:file}:%{INT:line_number}\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}\",\n \"post_map_values\":{\n \"logtime\":{\n \"map_date\":{\n \"date_pattern\":\"yyyy-MM-dd HH:mm:ss,SSS\"\n }\n \n }\n \n }\n \n },\n {\n \"filter\":\"grok\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"ranger_kms\"\n ]\n \n }\n \n },\n \"log4j_format\":\"%d{ISO8601} %-5p %c{1} - %m%n\",\n \"multiline_pattern\":\"^(%{TIMESTAMP_ISO8601:logtime})\",\n \"message_pattern\":\"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}\",\n \"post_map_values\":{\n \"logtime\":{\n \"map_date\":{\n \"date_pattern\":\"yyyy-MM-dd HH:mm:ss,SSS\"\n }\n \n }\n \n }\n \n },\n {\n \"filter\":\"grok\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"ranger_usersync\"\n ]\n \n }\n \n },\n \"log4j_format\":\"%d{dd MMM yyyy HH:mm:ss} %5p %c{1} [%t] - %m%n\",\n \"multiline_pattern\":\"^(%{USER_SYNC_DATE:logtime})\",\n \"message_pattern\":\"(?m)^%{USER_SYNC_DATE:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}\",\n \"post_map_values\":{\n \"logtime\":{\n \"map_date\":{\n \"date_pattern\":\"dd MMM yyyy HH:mm:ss\"\n }\n \n }\n \n }\n \n },\n {\n \"filter\":\"grok\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"storm_drpc\",\n \"storm_logviewer\",\n \"storm_nimbus\",\n \"storm_supervisor\",\n \"storm_ui\",\n \"storm_worker\"\n ]\n \n }\n \n },\n \"log4j_format\":\"\",\n \"multiline_pattern\":\"^(%{TIMESTAMP_ISO8601:logtime})\",\n \"message_pattern\":\"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\[%{LOGLEVEL:level}\\]%{SPACE}%{SPACE}%{GREEDYDATA:log_message}\",\n \"post_map_values\":{\n \"logtime\":{\n \"map_date\":{\n \"date_pattern\":\"yyyy-MM-dd HH:mm:ss.SSS\"\n }\n \n }\n \n }\n \n },\n {\n \"filter\":\"grok\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"zookeeper\"\n ]\n \n }\n \n },\n \"log4j_format\":\"%d{ISO8601} - %-5p [%t:%C{1}@% L] - %m%n\",\n \"multiline_pattern\":\"^(%{TIMESTAMP_ISO8601:logtime})\",\n \"message_pattern\":\"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}-%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\@%{INT:line_number}\\]%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}\",\n \"post_map_values\":{\n \"logtime\":{\n \"map_date\":{\n \"date_pattern\":\"yyyy-MM-dd HH:mm:ss,SSS\"\n }\n \n }\n \n }\n \n },\n {\n \"filter\":\"grok\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"hdfs_audit\"\n ]\n \n }\n \n },\n \"log4j_format\":\"%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n\",\n \"multiline_pattern\":\"^(%{TIMESTAMP_ISO8601:evtTime})\",\n \"message_pattern\":\"(?m)^%{TIMESTAMP_ISO8601:evtTime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}:%{SPACE}%{GREEDYDATA:log_message}\",\n \"post_map_values\":{\n \"evtTime\":{\n \"map_date\":{\n \"date_pattern\":\"yyyy-MM-dd HH:mm:ss,SSS\"\n }\n \n }\n \n }\n \n },\n {\n \"filter\":\"keyvalue\",\n \"sort_order\":1,\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"hdfs_audit\"\n ]\n \n }\n \n },\n \"source_field\":\"log_message\",\n \"value_split\":\"=\",\n \"field_split\":\"\t\",\n \"post_map_values\":{\n \"src\":{\n \"map_fieldname\":{\n \"new_fieldname\":\"resource\"\n }\n \n },\n \"ip\":{\n \"map_fieldname\":{\n \"new_fieldname\":\"cliIP\"\n }\n \n },\n \"allowed\":[\n {\n \"map_fieldvalue\":{\n \"pre_value\":\"true\",\n \"post_value\":\"1\"\n }\n \n },\n {\n \"map _fieldvalue\":{\n \"pre_value\":\"false\",\n \"post_value\":\"0\"\n }\n \n },\n {\n \"map_fieldname\":{\n \"new_fieldname\":\"result\"\n }\n \n }\n \n ],\n \"cmd\":{\n \"map_fieldname\":{\n \"new_fieldname\":\"action\"\n }\n \n },\n \"proto\":{\n \"map_fieldname\":{\n \"new_fieldname\":\"cliType\"\n }\n \n },\n \"callerContext\":{\n \"map_fieldname\":{\n \"new_fieldname\":\"req_caller_id\"\n }\n \n }\n \n }\n \n },\n {\n \"filter\":\"grok\",\n \"sort_order\":2,\n \"source_field\":\"ugi\",\n \"remove_source_field\":\"false\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"hdfs_audit\"\n ]\n \n }\n \n },\n \"message_pattern\":\"%{USERNAME:p_user}.+auth:%{USERNAME:p_authType}.+via %{USERNAME:k_user}.+auth:%{USERNAME:k_authType}|%{USERNAME:user}.+auth:%{USERNAME:authType}|%{USERNAME:x_user}\",\n \"post_map_values\":{\n \"user\":{\n \"map_fieldname\":{\n \"new_fieldname\":\"reqUser\"\n }\n \n },\n \"x_user\":{\n \"map_fieldname\":{\n \"new_fieldname\":\"reqUser\"\n }\n \n },\n \"p_user\":{\n \"map_fieldname\":{\n \"new_fieldname\":\"reqUser\"\n }\n \n },\n \"k_user\":{\n \"map_fieldname\":{\n \"new_fieldname\":\"proxyUsers\"\n }\n \n },\n \"p_authType\":{\n \"map_fieldname\":{\n \"new_fieldname\":\"authType\"\n }\n \n },\n \"k_authType\":{\n \"map_fieldname\":{\n \"new_fie ldname\":\"proxyAuthType\"\n }\n \n }\n \n }\n \n }\n \n ],\n \"output\":[\n {\n \"is_enabled\":\"{{solr_service_logs_enable}}\",\n \"comment\":\"Output to solr for service logs\",\n \"destination\":\"solr\",\n \"zk_connect_string\":\"{{zookeeper_quorum}}{{solr_znode}}\",\n \"collection\":\"{{solr_collection_service_logs}}\",\n \"number_of_shards\": \"{{logsearch_collection_service_logs_numshards}}\",\n \"splits_interval_mins\": \"{{logsearch_service_logs_split_interval_mins}}\",\n \"conditions\":{\n \"fields\":{\n \"rowtype\":[\n \"service\"\n ]\n \n }\n \n }\n \n },\n {\n \"comment\":\"Output to solr for audit records\",\n \"is_enabled\":\"{{solr_audit_logs_enable}}\",\n \"destination\":\"solr\",\n \"zk_connect_string\":\"{{zookeeper_quorum}}{{solr_znode}}\",\n \"collection\":\"{{solr_collec tion_audit_logs}}\",\n \"number_of_shards\": \"{{logsearch_collection_audit_logs_numshards}}\",\n \"splits_interval_mins\": \"{{logsearch_audit_logs_split_interval_mins}}\",\n \"conditions\":{\n \"fields\":{\n \"rowtype\":[\n \"audit\"\n ]\n \n }\n \n }\n \n },\n {\n \"is_enabled\":\"{{kafka_service_logs_enable}}\",\n \"destination\":\"kafka\",\n \"broker_list\":\"{{kafka_broker_list}}\",\n \"topic\":\"{{kafka_topic_service_logs}}\",\n \"kafka.security.protocol\":\"{{kafka_security_protocol}}\",\n \"kafka.sasl.kerberos.service.name\":\"{{kafka_kerberos_service_name}}\",\n \"conditions\":{\n \"fields\":{\n \"rowtype\":[\n \"service\"\n ]\n \n }\n \n }\n \n },\n {\n \"is_enabled\":\"{{kafka_audit_logs_enable}}\",\n \"destination\":\"kafka\",\n \"broker_list\":\"{{kafka_brok er_list}}\",\n \"topic\":\"{{kafka_topic_audit_logs}}\",\n \"kafka.security.protocol\":\"{{kafka_security_protocol}}\",\n \"kafka.sasl.kerberos.service.name\":\"{{kafka_kerberos_service_name}}\",\n \"conditions\":{\n \"fields\":{\n \"rowtype\":[\n \"audit\"\n ]\n \n }\n \n }\n \n }\n \n ]\n \n}" } }, "configuration_attributes": {
