Repository: ambari Updated Branches: refs/heads/branch-2.4 f65ce6cd7 -> 9471646c7
http://git-wip-us.apache.org/repos/asf/ambari/blob/9471646c/ambari-server/src/test/python/stacks/2.4/configs/default.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.4/configs/default.json b/ambari-server/src/test/python/stacks/2.4/configs/default.json index ff548e0..848be40 100644 --- a/ambari-server/src/test/python/stacks/2.4/configs/default.json +++ b/ambari-server/src/test/python/stacks/2.4/configs/default.json @@ -302,7 +302,7 @@ "content": "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n<!--\n Licensed to the Apache Software Foundation (ASF) under one or more\n contributor license agreements. See the NOTICE file distributed with\n this work for additional information regarding copyright ownership.\n The ASF licenses this file to You under the Apache License, Version 2.0\n (the \"License\"); you may not use this file except in compliance with\n the License. You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n-->\n<!DOCTYPE log4j:configuration SYSTEM \"log4j.dtd\">\n<log4j:configuration xmlns:log4j=\"http://jakarta. apache.org/log4j/\">\n <appender name=\"console\" class=\"org.apache.log4j.ConsoleAppender\">\n <param name=\"Target\" value=\"System.out\" />\n <layout class=\"org.apache.log4j.PatternLayout\">\n <param name=\"ConversionPattern\" value=\"%d [%t] %-5p %C{6} (%F:%L) - %m%n\" />\n </layout>\n </appender>\n\n <appender name=\"rolling_file\" class=\"org.apache.log4j.RollingFileAppender\"> \n <param name=\"file\" value=\"{{logfeeder_log_dir}}/logfeeder.log\" />\n <param name=\"append\" value=\"true\" /> \n <param name=\"maxFileSize\" value=\"10MB\" /> \n <param name=\"maxBackupIndex\" value=\"10\" /> \n <layout class=\"org.apache.log4j.PatternLayout\"> \n <param name=\"ConversionPattern\" value=\"%d [%t] %-5p %C{6} (%F:%L) - %m%n\"/> \n </layout> \n </appender> \n\n <category name=\"org.apache.ambari.logfeeder\" additivity=\"false\">\n <priori ty value=\"info\" />\n <appender-ref ref=\"rolling_file\" />\n </category>\n\n <root>\n <priority value=\"warn\" />\n <appender-ref ref=\"rolling_file\" />\n </root>\n</log4j:configuration>" }, "logfeeder-input-configs": { - "content": "{\n \"global\":{\n \"add_fields\":{\n \"cluster\":\"{{cluster_name}}\"\n },\n \"source\":\"file\",\n \"tail\":\"true\",\n \"gen_event_md5\":\"true\",\n \"start_position\":\"beginning\"\n },\n \"input\":[\n {\n \"type\":\"accumulo_gc\",\n \"rowtype\":\"service\",\n \"path\":\"{{accumulo_log_dir}}/gc_*.log\"\n },\n {\n \"type\":\"accumulo_master\",\n \"rowtype\":\"service\",\n \"path\":\"{{accumulo_log_dir}}/master_*.log\"\n },\n {\n \"type\":\"accumulo_monitor\",\n \"rowtype\":\"service\",\n \"path\":\"{{accumulo_log_dir}}/monitor_*.log\"\n },\n {\n \"type\":\"accumulo_tracer\",\n \"rowtype\":\"service\",\n \"path\":\"{{accumulo_log_dir}}/tracer_*.log\"\n },\n {\n \"type\":\"accumulo_tserver\",\n \"rowtype\":\"service\",\n \"path\":\"{{accumulo_log_dir}}/tserver_*.log\"\n },\n {\n \"type\":\"atlas_app\",\n \"rowtype\":\"se rvice\",\n \"path\":\"{{atlas_log_dir}}/application.log\"\n },\n {\n \"type\":\"ambari_agent\",\n \"rowtype\":\"service\",\n \"path\":\"{{ambari_agent_log_dir}}/ambari-agent.log\"\n },\n {\n \"type\":\"ambari_server\",\n \"rowtype\":\"service\",\n \"path\":\"{{ambari_server_log_dir}}/ambari-server.log\"\n },\n {\n \"type\":\"ams_hbase_master\",\n \"rowtype\":\"service\",\n \"path\":\"{{metrics_collector_log_dir}}/hbase-ams-master-*.log\"\n },\n {\n \"type\":\"ams_hbase_regionserver\",\n \"rowtype\":\"service\",\n \"path\":\"{{metrics_collector_log_dir}}/hbase-ams-regionserver-*.log\"\n },\n {\n \"type\":\"ams_collector\",\n \"rowtype\":\"service\",\n \"path\":\"{{metrics_collector_log_dir}}/ambari-metrics-collector.log\"\n },\n {\n \"type\":\"falcon_app\",\n \"rowtype\":\"service\",\n \"path\":\"{{falcon_log_dir}}/falcon.application.log\"\n },\n {\ n \"type\":\"hbase_master\",\n \"rowtype\":\"service\",\n \"path\":\"{{hbase_log_dir}}/hbase-hbase-master-*.log\"\n },\n {\n \"type\":\"hbase_regionserver\",\n \"rowtype\":\"service\",\n \"path\":\"{{hbase_log_dir}}/hbase-hbase-regionserver-*.log\"\n },\n {\n \"type\":\"hdfs_datanode\",\n \"rowtype\":\"service\",\n \"path\":\"{{hdfs_log_dir_prefix}}/hdfs/hadoop-hdfs-datanode-*.log\"\n },\n {\n \"type\":\"hdfs_namenode\",\n \"rowtype\":\"service\",\n \"path\":\"{{hdfs_log_dir_prefix}}/hdfs/hadoop-hdfs-namenode-*.log\"\n },\n {\n \"type\":\"hdfs_journalnode\",\n \"rowtype\":\"service\",\n \"path\":\"{{hdfs_log_dir_prefix}}/hdfs/hadoop-hdfs-journalnode-*.log\"\n },\n {\n \"type\":\"hdfs_secondarynamenode\",\n \"rowtype\":\"service\",\n \"path\":\"{{hdfs_log_dir_prefix}}/hdfs/hadoop-hdfs-secondarynamenode-*.log\"\n },\n {\n \"type\":\"hdfs_zkfc\",\n \"r owtype\":\"service\",\n \"path\":\"{{hdfs_log_dir_prefix}}/hdfs/hadoop-hdfs-zkfc-*.log\"\n },\n {\n \"type\":\"hive_hiveserver2\",\n \"rowtype\":\"service\",\n \"path\":\"{{hive_log_dir}}/hiveserver2.log\"\n },\n {\n \"type\":\"hive_metastore\",\n \"rowtype\":\"service\",\n \"path\":\"{{hive_log_dir}}/hivemetastore.log\"\n },\n {\n \"type\":\"kafka_controller\",\n \"rowtype\":\"service\",\n \"path\":\"{{kafka_log_dir}}/controller.log\"\n },\n {\n \"type\":\"kafka_request\",\n \"rowtype\":\"service\",\n \"path\":\"{{kafka_log_dir}}/kafka-request.log\"\n },\n {\n \"type\":\"kafka_logcleaner\",\n \"rowtype\":\"service\",\n \"path\":\"{{kafka_log_dir}}/log-cleaner.log\"\n },\n {\n \"type\":\"kafka_server\",\n \"rowtype\":\"service\",\n \"path\":\"{{kafka_log_dir}}/server.log\"\n },\n {\n \"type\":\"kafka_statechange\",\n \"rowtype\":\"service \",\n \"path\":\"{{kafka_log_dir}}/state-change.log\"\n },\n {\n \"type\":\"knox_gateway\",\n \"rowtype\":\"service\",\n \"path\":\"{{knox_log_dir}}/gateway.log\"\n },\n {\n \"type\":\"knox_cli\",\n \"rowtype\":\"service\",\n \"path\":\"{{knox_log_dir}}/knoxcli.log\"\n },\n {\n \"type\":\"knox_ldap\",\n \"rowtype\":\"service\",\n \"path\":\"{{knox_log_dir}}/ldap.log\"\n },\n {\n \"type\":\"mapred_historyserver\",\n \"rowtype\":\"service\",\n \"path\":\"{{mapred_log_dir_prefix}}/mapred/mapred-mapred-historyserver*.log\"\n },\n {\n \"type\":\"logsearch_app\",\n \"rowtype\":\"service\",\n \"path\":\"{{logsearch_log_dir}}/logsearch.log\"\n },\n {\n \"type\":\"logsearch_feeder\",\n \"rowtype\":\"service\",\n \"path\":\"{{logfeeder_log_dir}}/logfeeder.log\"\n },\n {\n \"type\":\"logsearch_perf\",\n \"rowtype\":\"service\",\n \"path\":\"{{l ogsearch_log_dir}}/logsearch-performance.log\"\n },\n {\n \"type\":\"ranger_admin\",\n \"rowtype\":\"service\",\n \"path\":\"{{ranger_admin_log_dir}}/xa_portal.log\"\n },\n {\n \"type\":\"ranger_dbpatch\",\n \"is_enabled\":\"true\",\n \"path\":\"{{ranger_admin_log_dir}}/ranger_db_patch.log\"\n },\n {\n \"type\":\"ranger_kms\",\n \"rowtype\":\"service\",\n \"path\":\"{{ranger_kms_log_dir}}/kms.log\"\n },\n {\n \"type\":\"ranger_usersync\",\n \"rowtype\":\"service\",\n \"path\":\"{{ranger_usersync_log_dir}}/usersync.log\"\n },\n {\n \"type\":\"oozie_app\",\n \"rowtype\":\"service\",\n \"path\":\"{{oozie_log_dir}}/oozie.log\"\n },\n {\n \"type\":\"yarn_nodemanager\",\n \"rowtype\":\"service\",\n \"path\":\"{{yarn_log_dir_prefix}}/yarn/yarn-yarn-nodemanager-*.log\"\n },\n {\n \"type\":\"yarn_resourcemanager\",\n \"rowtype\":\"service\",\n \"p ath\":\"{{yarn_log_dir_prefix}}/yarn/yarn-yarn-resourcemanager-*.log\"\n },\n {\n \"type\":\"yarn_timelineserver\",\n \"rowtype\":\"service\",\n \"path\":\"{{yarn_log_dir_prefix}}/yarn/yarn-yarn-timelineserver-*.log\"\n },\n {\n \"type\":\"yarn_historyserver\",\n \"rowtype\":\"service\",\n \"path\":\"{{yarn_log_dir_prefix}}/yarn/yarn-yarn-historyserver-*.log\"\n },\n {\n \"type\":\"yarn_jobsummary\",\n \"rowtype\":\"service\",\n \"path\":\"{{yarn_log_dir_prefix}}/yarn/hadoop-mapreduce.jobsummary.log\"\n },\n {\n \"type\":\"storm_drpc\",\n \"rowtype\":\"service\",\n \"path\":\"{{storm_log_dir}}/drpc.log\"\n },\n {\n \"type\":\"storm_logviewer\",\n \"rowtype\":\"service\",\n \"path\":\"{{storm_log_dir}}/logviewer.log\"\n },\n {\n \"type\":\"storm_nimbus\",\n \"rowtype\":\"service\",\n \"path\":\"{{storm_log_dir}}/nimbus.log\"\n },\n {\n \"type\":\"st orm_supervisor\",\n \"rowtype\":\"service\",\n \"path\":\"{{storm_log_dir}}/supervisor.log\"\n },\n {\n \"type\":\"storm_ui\",\n \"rowtype\":\"service\",\n \"path\":\"{{storm_log_dir}}/ui.log\"\n },\n {\n \"type\":\"storm_worker\",\n \"rowtype\":\"service\",\n \"path\":\"{{storm_log_dir}}/*worker*.log\"\n },\n {\n \"type\":\"zookeeper\",\n \"rowtype\":\"service\",\n \"path\":\"{{zk_log_dir}}/zookeeper/zookeeper*.out\"\n },\n {\n \"type\":\"hdfs_audit\",\n \"rowtype\":\"audit\",\n \"is_enabled\":\"true\",\n \"add_fields\":{\n \"logType\":\"HDFSAudit\",\n \"enforcer\":\"hadoop-acl\",\n \"repoType\":\"1\",\n \"repo\":\"hdfs\"\n },\n \"path\":\"{{hdfs_log_dir_prefix}}/hdfs/hdfs-audit.log\"\n }\n \n ],\n \"filter\":[\n {\n \"filter\":\"grok\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"accumulo_mas ter\"\n ]\n \n }\n \n },\n \"log4j_format\":\"%d{ISO8601} [%-8c{2}] %-5p: %m%n\",\n \"multiline_pattern\":\"^(%{TIMESTAMP_ISO8601:logtime})\",\n \"message_pattern\":\"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}\\[%{JAVACLASS:logger_name}\\]%{SPACE}%{LOGLEVEL:level}:%{SPACE}%{GREEDYDATA:log_message}\",\n \"post_map_values\":{\n \"logtime\":{\n \"map_date\":{\n \"date_pattern\":\"yyyy-MM-dd HH:mm:ss,SSS\"\n }\n \n }\n \n }\n \n },\n {\n \"filter\":\"grok\",\n \"comment\":\"This one has one extra space after LEVEL\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"accumulo_gc\",\n \"accumulo_monitor\",\n \"accumulo_tracer\",\n \"accumulo_tserver\"\n ]\n \n }\n \n },\n \"log4j_format\":\"%d{ISO8601} [%-8c{2}] %-5p: %X{application} %m%n\",\n \"multiline_pattern\":\"^(%{TIMESTAMP_ISO8601:logtime})\",\n \"message_pattern\":\"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}\\[%{JAVACLASS:logger_name}\\]%{SPACE}%{LOGLEVEL:level}%{SPACE}:%{SPACE}%{GREEDYDATA:log_message}\",\n \"post_map_values\":{\n \"logtime\":{\n \"map_date\":{\n \"date_pattern\":\"yyyy-MM-dd HH:mm:ss,SSS\"\n }\n \n }\n \n }\n \n },\n {\n \"filter\":\"grok\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"atlas_app\",\n \"falcon_app\"\n ]\n \n }\n \n },\n \"log4j_format\":\"%d %-5p - [%t:%x] ~ %m (%c{1}:%L)%n\",\n \"multiline_pattern\":\"^(%{TIMESTAMP_ISO8601:logtime})\",\n \"message_pattern\":\"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{SPACE}-%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}~%{SPACE}%{GREEDYDATA:log_message}\",\n \"post_map_values\" :{\n \"logtime\":{\n \"map_date\":{\n \"date_pattern\":\"yyyy-MM-dd HH:mm:ss,SSS\"\n }\n \n }\n \n }\n \n },\n {\n \"filter\":\"grok\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"ams_collector\"\n ]\n \n }\n \n },\n \"log4j_format\":\"%d{ISO8601} %p %c: %m%n\",\n \"multiline_pattern\":\"^(%{TIMESTAMP_ISO8601:logtime})\",\n \"message_pattern\":\"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}:%{SPACE}%{GREEDYDATA:log_message}\",\n \"post_map_values\":{\n \"logtime\":{\n \"map_date\":{\n \"date_pattern\":\"yyyy-MM-dd HH:mm:ss,SSS\"\n }\n \n }\n \n }\n \n },\n {\n \"filter\":\"grok\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"ams_hbase_master\",\n \"ams_hbase_regionserver\",\n \"hbase_master\",\n \"hbase_regionserver\"\n ]\n \n }\n \n },\n \"log4j_format\":\"%d{ISO8601} %-5p [%t] %c{2}: %m%n\",\n \"multiline_pattern\":\"^(%{TIMESTAMP_ISO8601:logtime})\",\n \"message_pattern\":\"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{JAVACLASS:logger_name}:%{SPACE}%{GREEDYDATA:log_message}\",\n \"post_map_values\":{\n \"logtime\":{\n \"map_date\":{\n \"date_pattern\":\"yyyy-MM-dd HH:mm:ss,SSS\"\n }\n \n }\n \n }\n \n },\n {\n \"filter\":\"grok\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"ambari_agent\"\n ]\n \n }\n \n },\n \"log4j_format\":\"\",\n \"multiline_pattern\":\"^(%{LOGLEVEL:level} %{TIMESTAMP_ISO8601:logtime})\",\n \" message_pattern\":\"(?m)^%{LOGLEVEL:level} %{TIMESTAMP_ISO8601:logtime} %{JAVAFILE:file}:%{INT:line_number} - %{GREEDYDATA:log_message}\",\n \"post_map_values\":{\n \"logtime\":{\n \"map_date\":{\n \"date_pattern\":\"yyyy-MM-dd HH:mm:ss,SSS\"\n }\n \n },\n \"level\":{\n \"map_fieldvalue\":{\n \"pre_value\":\"WARNING\",\n \"post_value\":\"WARN\"\n }\n \n }\n \n }\n \n },\n {\n \"filter\":\"grok\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"ambari_server\"\n ]\n \n }\n \n },\n \"log4j_format\":\"%d{DATE} %5p [%t] %c{1}:%L - %m%n\",\n \"multiline_pattern\":\"^(%{USER_SYNC_DATE:logtime})\",\n \"message_pattern\":\"(?m)^%{USER_SYNC_DATE:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{JAVACLASS:logger_name}:%{INT:line_number} %{SPACE}-%{SPACE}%{GREEDYDATA:log_message}\",\n \"post_map_values\":{\n \"logtime\":{\n \"map_date\":{\n \"date_pattern\":\"dd MMM yyyy HH:mm:ss\"\n }\n \n }\n \n }\n \n },\n {\n \"filter\":\"grok\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"hdfs_datanode\",\n \"hdfs_journalnode\",\n \"hdfs_secondarynamenode\",\n \"hdfs_namenode\",\n \"hdfs_zkfc\",\n \"knox_gateway\",\n \"knox_cli\",\n \"knox_ldap\",\n \"mapred_historyserver\",\n \"yarn_historyserver\",\n \"yarn_jobsummary\",\n \"yarn_nodemanager\",\n \"yarn_resourcemanager\",\n \"yarn_timelineserver\"\n ]\n \n }\n \n },\n \"log4j_format\":\"%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n\",\n \"multiline_pattern\":\"^(%{TIMESTA MP_ISO8601:logtime})\",\n \"message_pattern\":\"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\(%{JAVAFILE:file}:%{JAVAMETHOD:method}\\(%{INT:line_number}\\)\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}\",\n \"post_map_values\":{\n \"logtime\":{\n \"map_date\":{\n \"date_pattern\":\"yyyy-MM-dd HH:mm:ss,SSS\"\n }\n \n }\n \n }\n \n },\n {\n \"filter\":\"grok\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"hive_hiveserver2\",\n \"hive_metastore\"\n ]\n \n }\n \n },\n \"log4j_format\":\"%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n\",\n \"multiline_pattern\":\"^(%{TIMESTAMP_ISO8601:logtime})\",\n \"message_pattern\":\"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]:%{SPACE}%{JAVACLASS:logger_name}%{SP ACE}\\(%{JAVAFILE:file}:%{JAVAMETHOD:method}\\(%{INT:line_number}\\)\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}\",\n \"post_map_values\":{\n \"logtime\":{\n \"map_date\":{\n \"date_pattern\":\"yyyy-MM-dd HH:mm:ss,SSS\"\n }\n \n }\n \n }\n \n },\n {\n \"filter\":\"grok\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"kafka_controller\",\n \"kafka_request\",\n \"kafka_logcleaner\"\n ]\n \n }\n \n },\n \"log4j_format\":\"[%d] %p %m (%c)%n\",\n \"multiline_pattern\":\"^(\\[%{TIMESTAMP_ISO8601:logtime}\\])\",\n \"message_pattern\":\"(?m)^\\[%{TIMESTAMP_ISO8601:logtime}\\]%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{GREEDYDATA:log_message}\",\n \"post_map_values\":{\n \"logtime\":{\n \"map_date\":{\n \"date_pattern\":\"yyyy-MM-dd HH:mm:ss,S SS\"\n }\n \n }\n \n }\n \n },\n {\n \"filter\":\"grok\",\n \"comment\":\"Suppose to be same log4j pattern as other kafka processes, but some reason thread is not printed\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"kafka_server\",\n \"kafka_statechange\"\n ]\n \n }\n \n },\n \"log4j_format\":\"[%d] %p %m (%c)%n\",\n \"multiline_pattern\":\"^(\\[%{TIMESTAMP_ISO8601:logtime}\\])\",\n \"message_pattern\":\"(?m)^\\[%{TIMESTAMP_ISO8601:logtime}\\]%{SPACE}%{LOGLEVEL:level}%{SPACE}%{GREEDYDATA:log_message}\",\n \"post_map_values\":{\n \"logtime\":{\n \"map_date\":{\n \"date_pattern\":\"yyyy-MM-dd HH:mm:ss,SSS\"\n }\n \n }\n \n }\n \n },\n {\n \"filter\":\"grok\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"o ozie_app\"\n ]\n \n }\n \n },\n \"log4j_format\":\"%d{ISO8601} %5p %c{1}:%L - SERVER[${oozie.instance.id}] %m%n\",\n \"multiline_pattern\":\"^(%{TIMESTAMP_ISO8601:logtime})\",\n \"message_pattern\":\"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{DATA:logger_name}:%{INT:line_number}%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}\",\n \"post_map_values\":{\n \"logtime\":{\n \"map_date\":{\n \"date_pattern\":\"yyyy-MM-dd HH:mm:ss,SSS\"\n }\n \n }\n \n }\n \n },\n {\n \"filter\":\"grok\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"logsearch_app\",\n \"logsearch_feeder\",\n \"logsearch_perf\",\n \"ranger_admin\",\n \"ranger_dbpatch\"\n ]\n \n }\n \n },\n \"log4j_format\":\"%d [%t] %-5p %C{6} (%F:%L) - %m%n\",\n \"multiline_pattern\":\"^(%{TIMESTAMP_ISO8601:logtime})\",\n \"message_pattern\":\"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\(%{JAVAFILE:file}:%{INT:line_number}\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}\",\n \"post_map_values\":{\n \"logtime\":{\n \"map_date\":{\n \"date_pattern\":\"yyyy-MM-dd HH:mm:ss,SSS\"\n }\n \n }\n \n }\n \n },\n {\n \"filter\":\"grok\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"ranger_kms\"\n ]\n \n }\n \n },\n \"log4j_format\":\"%d{ISO8601} %-5p %c{1} - %m%n\",\n \"multiline_pattern\":\"^(%{TIMESTAMP_ISO8601:logtime})\",\n \"message_pattern\":\"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}\",\n \"post_map_values\":{\n \"logtime\":{\n \"map_date\":{\n \"date_pattern\":\"yyyy-MM-dd HH:mm:ss,SSS\"\n }\n \n }\n \n }\n \n },\n {\n \"filter\":\"grok\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"ranger_usersync\"\n ]\n \n }\n \n },\n \"log4j_format\":\"%d{dd MMM yyyy HH:mm:ss} %5p %c{1} [%t] - %m%n\",\n \"multiline_pattern\":\"^(%{USER_SYNC_DATE:logtime})\",\n \"message_pattern\":\"(?m)^%{USER_SYNC_DATE:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}\",\n \"post_map_values\":{\n \"logtime\":{\n \"map_date\":{\n \"date_pattern\":\"dd MMM yyyy HH:mm:ss\"\n }\n \n }\n \n }\n \n },\n {\n \"filter\":\"grok\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"storm_drpc\",\n \"storm_logviewer\",\n \"storm_nimbus\",\n \"storm_supervisor\",\n \"storm_ui\",\n \"storm_worker\"\n ]\n \n }\n \n },\n \"log4j_format\":\"\",\n \"multiline_pattern\":\"^(%{TIMESTAMP_ISO8601:logtime})\",\n \"message_pattern\":\"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\[%{LOGLEVEL:level}\\]%{SPACE}%{SPACE}%{GREEDYDATA:log_message}\",\n \"post_map_values\":{\n \"logtime\":{\n \"map_date\":{\n \"date_pattern\":\"yyyy-MM-dd HH:mm:ss.SSS\"\n }\n \n }\n \n }\n \n },\n {\n \"filter\":\"grok\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"zookeeper\"\n ]\n \n }\n \n },\n \"log4j_format\":\"%d{ISO8601} - %-5p [%t:%C{1}@% L] - %m%n\",\n \"multiline_pattern\":\"^(%{TIMESTAMP_ISO8601:logtime})\",\n \"message_pattern\":\"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}-%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\@%{INT:line_number}\\]%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}\",\n \"post_map_values\":{\n \"logtime\":{\n \"map_date\":{\n \"date_pattern\":\"yyyy-MM-dd HH:mm:ss,SSS\"\n }\n \n }\n \n }\n \n },\n {\n \"filter\":\"grok\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"hdfs_audit\"\n ]\n \n }\n \n },\n \"log4j_format\":\"%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n\",\n \"multiline_pattern\":\"^(%{TIMESTAMP_ISO8601:evtTime})\",\n \"message_pattern\":\"(?m)^%{TIMESTAMP_ISO8601:evtTime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}:%{SPACE}%{GREEDYDATA:log_message}\",\n \"post_map_values\":{\n \"evtTime\":{\n \"map_date\":{\n \"date_pattern\":\"yyyy-MM-dd HH:mm:ss,SSS\"\n }\n \n }\n \n }\n \n },\n {\n \"filter\":\"keyvalue\",\n \"sort_order\":1,\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"hdfs_audit\"\n ]\n \n }\n \n },\n \"source_field\":\"log_message\",\n \"value_split\":\"=\",\n \"field_split\":\"\t\",\n \"post_map_values\":{\n \"src\":{\n \"map_fieldname\":{\n \"new_fieldname\":\"resource\"\n }\n \n },\n \"ip\":{\n \"map_fieldname\":{\n \"new_fieldname\":\"cliIP\"\n }\n \n },\n \"allowed\":[\n {\n \"map_fieldvalue\":{\n \"pre_value\":\"true\",\n \"post_value\":\"1\"\n }\n \n },\n {\n \"map _fieldvalue\":{\n \"pre_value\":\"false\",\n \"post_value\":\"0\"\n }\n \n },\n {\n \"map_fieldname\":{\n \"new_fieldname\":\"result\"\n }\n \n }\n \n ],\n \"cmd\":{\n \"map_fieldname\":{\n \"new_fieldname\":\"action\"\n }\n \n },\n \"proto\":{\n \"map_fieldname\":{\n \"new_fieldname\":\"cliType\"\n }\n \n },\n \"callerContext\":{\n \"map_fieldname\":{\n \"new_fieldname\":\"req_caller_id\"\n }\n \n }\n \n }\n \n },\n {\n \"filter\":\"grok\",\n \"sort_order\":2,\n \"source_field\":\"ugi\",\n \"remove_source_field\":\"false\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"hdfs_audit\"\n ]\n \n }\n \n },\n \"message_pattern\":\"%{USERNAME:p_user}.+auth:%{USERNAME:p_authType}.+via %{USERNAME:k_user}.+auth:%{USERNAME:k_authType}|%{USERNAME:user}.+auth:%{USERNAME:authType}|%{USERNAME:x_user}\",\n \"post_map_values\":{\n \"user\":{\n \"map_fieldname\":{\n \"new_fieldname\":\"reqUser\"\n }\n \n },\n \"x_user\":{\n \"map_fieldname\":{\n \"new_fieldname\":\"reqUser\"\n }\n \n },\n \"p_user\":{\n \"map_fieldname\":{\n \"new_fieldname\":\"reqUser\"\n }\n \n },\n \"k_user\":{\n \"map_fieldname\":{\n \"new_fieldname\":\"proxyUsers\"\n }\n \n },\n \"p_authType\":{\n \"map_fieldname\":{\n \"new_fieldname\":\"authType\"\n }\n \n },\n \"k_authType\":{\n \"map_fieldname\":{\n \"new_fie ldname\":\"proxyAuthType\"\n }\n \n }\n \n }\n \n }\n \n ],\n \"output\":[\n {\n \"is_enabled\":\"{{solr_service_logs_enable}}\",\n \"comment\":\"Output to solr for service logs\",\n \"destination\":\"solr\",\n \"zk_hosts\":\"{{zookeeper_quorum}}{{solr_znode}}\",\n \"collection\":\"{{solr_collection_service_logs}}\",\n \"number_of_shards\": \"{{logsearch_collection_service_logs_numshards}}\",\n \"splits_interval_mins\": \"{{logsearch_service_logs_split_interval_mins}}\",\n \"conditions\":{\n \"fields\":{\n \"rowtype\":[\n \"service\"\n ]\n \n }\n \n }\n \n },\n {\n \"comment\":\"Output to solr for audit records\",\n \"is_enabled\":\"{{solr_audit_logs_enable}}\",\n \"destination\":\"solr\",\n \"zk_hosts\":\"{{zookeeper_quorum}}{{solr_znode}}\",\n \"collection\":\"{{solr_collection_audit_logs}}\ ",\n \"number_of_shards\": \"{{logsearch_collection_audit_logs_numshards}}\",\n \"splits_interval_mins\": \"{{logsearch_audit_logs_split_interval_mins}}\",\n \"conditions\":{\n \"fields\":{\n \"rowtype\":[\n \"audit\"\n ]\n \n }\n \n }\n \n },\n {\n \"is_enabled\":\"{{kafka_service_logs_enable}}\",\n \"destination\":\"kafka\",\n \"broker_list\":\"{{kafka_broker_list}}\",\n \"topic\":\"{{kafka_topic_service_logs}}\",\n \"kafka.security.protocol\":\"{{kafka_security_protocol}}\",\n \"kafka.sasl.kerberos.service.name\":\"{{kafka_kerberos_service_name}}\",\n \"conditions\":{\n \"fields\":{\n \"rowtype\":[\n \"service\"\n ]\n \n }\n \n }\n \n },\n {\n \"is_enabled\":\"{{kafka_audit_logs_enable}}\",\n \"destination\":\"kafka\",\n \"broker_list\":\"{{kafka_broker_list}}\",\n \"topic\":\"{{kafka_topic_audit_logs}}\",\n \"kafka.security.protocol\":\"{{kafka_security_protocol}}\",\n \"kafka.sasl.kerberos.service.name\":\"{{kafka_kerberos_service_name}}\",\n \"conditions\":{\n \"fields\":{\n \"rowtype\":[\n \"audit\"\n ]\n \n }\n \n }\n \n }\n \n ]\n \n}" + "content": "{\n \"global\":{\n \"add_fields\":{\n \"cluster\":\"{{cluster_name}}\"\n },\n \"source\":\"file\",\n \"tail\":\"true\",\n \"gen_event_md5\":\"true\",\n \"start_position\":\"beginning\"\n },\n \"input\":[\n {\n \"type\":\"accumulo_gc\",\n \"rowtype\":\"service\",\n \"path\":\"{{accumulo_log_dir}}/gc_*.log\"\n },\n {\n \"type\":\"accumulo_master\",\n \"rowtype\":\"service\",\n \"path\":\"{{accumulo_log_dir}}/master_*.log\"\n },\n {\n \"type\":\"accumulo_monitor\",\n \"rowtype\":\"service\",\n \"path\":\"{{accumulo_log_dir}}/monitor_*.log\"\n },\n {\n \"type\":\"accumulo_tracer\",\n \"rowtype\":\"service\",\n \"path\":\"{{accumulo_log_dir}}/tracer_*.log\"\n },\n {\n \"type\":\"accumulo_tserver\",\n \"rowtype\":\"service\",\n \"path\":\"{{accumulo_log_dir}}/tserver_*.log\"\n },\n {\n \"type\":\"atlas_app\",\n \"rowtype\":\"se rvice\",\n \"path\":\"{{atlas_log_dir}}/application.log\"\n },\n {\n \"type\":\"ambari_agent\",\n \"rowtype\":\"service\",\n \"path\":\"{{ambari_agent_log_dir}}/ambari-agent.log\"\n },\n {\n \"type\":\"ambari_server\",\n \"rowtype\":\"service\",\n \"path\":\"{{ambari_server_log_dir}}/ambari-server.log\"\n },\n {\n \"type\":\"ams_hbase_master\",\n \"rowtype\":\"service\",\n \"path\":\"{{metrics_collector_log_dir}}/hbase-ams-master-*.log\"\n },\n {\n \"type\":\"ams_hbase_regionserver\",\n \"rowtype\":\"service\",\n \"path\":\"{{metrics_collector_log_dir}}/hbase-ams-regionserver-*.log\"\n },\n {\n \"type\":\"ams_collector\",\n \"rowtype\":\"service\",\n \"path\":\"{{metrics_collector_log_dir}}/ambari-metrics-collector.log\"\n },\n {\n \"type\":\"falcon_app\",\n \"rowtype\":\"service\",\n \"path\":\"{{falcon_log_dir}}/falcon.application.log\"\n },\n {\ n \"type\":\"hbase_master\",\n \"rowtype\":\"service\",\n \"path\":\"{{hbase_log_dir}}/hbase-hbase-master-*.log\"\n },\n {\n \"type\":\"hbase_regionserver\",\n \"rowtype\":\"service\",\n \"path\":\"{{hbase_log_dir}}/hbase-hbase-regionserver-*.log\"\n },\n {\n \"type\":\"hdfs_datanode\",\n \"rowtype\":\"service\",\n \"path\":\"{{hdfs_log_dir_prefix}}/hdfs/hadoop-hdfs-datanode-*.log\"\n },\n {\n \"type\":\"hdfs_namenode\",\n \"rowtype\":\"service\",\n \"path\":\"{{hdfs_log_dir_prefix}}/hdfs/hadoop-hdfs-namenode-*.log\"\n },\n {\n \"type\":\"hdfs_journalnode\",\n \"rowtype\":\"service\",\n \"path\":\"{{hdfs_log_dir_prefix}}/hdfs/hadoop-hdfs-journalnode-*.log\"\n },\n {\n \"type\":\"hdfs_secondarynamenode\",\n \"rowtype\":\"service\",\n \"path\":\"{{hdfs_log_dir_prefix}}/hdfs/hadoop-hdfs-secondarynamenode-*.log\"\n },\n {\n \"type\":\"hdfs_zkfc\",\n \"r owtype\":\"service\",\n \"path\":\"{{hdfs_log_dir_prefix}}/hdfs/hadoop-hdfs-zkfc-*.log\"\n },\n {\n \"type\":\"hive_hiveserver2\",\n \"rowtype\":\"service\",\n \"path\":\"{{hive_log_dir}}/hiveserver2.log\"\n },\n {\n \"type\":\"hive_metastore\",\n \"rowtype\":\"service\",\n \"path\":\"{{hive_log_dir}}/hivemetastore.log\"\n },\n {\n \"type\":\"kafka_controller\",\n \"rowtype\":\"service\",\n \"path\":\"{{kafka_log_dir}}/controller.log\"\n },\n {\n \"type\":\"kafka_request\",\n \"rowtype\":\"service\",\n \"path\":\"{{kafka_log_dir}}/kafka-request.log\"\n },\n {\n \"type\":\"kafka_logcleaner\",\n \"rowtype\":\"service\",\n \"path\":\"{{kafka_log_dir}}/log-cleaner.log\"\n },\n {\n \"type\":\"kafka_server\",\n \"rowtype\":\"service\",\n \"path\":\"{{kafka_log_dir}}/server.log\"\n },\n {\n \"type\":\"kafka_statechange\",\n \"rowtype\":\"service \",\n \"path\":\"{{kafka_log_dir}}/state-change.log\"\n },\n {\n \"type\":\"knox_gateway\",\n \"rowtype\":\"service\",\n \"path\":\"{{knox_log_dir}}/gateway.log\"\n },\n {\n \"type\":\"knox_cli\",\n \"rowtype\":\"service\",\n \"path\":\"{{knox_log_dir}}/knoxcli.log\"\n },\n {\n \"type\":\"knox_ldap\",\n \"rowtype\":\"service\",\n \"path\":\"{{knox_log_dir}}/ldap.log\"\n },\n {\n \"type\":\"mapred_historyserver\",\n \"rowtype\":\"service\",\n \"path\":\"{{mapred_log_dir_prefix}}/mapred/mapred-mapred-historyserver*.log\"\n },\n {\n \"type\":\"logsearch_app\",\n \"rowtype\":\"service\",\n \"path\":\"{{logsearch_log_dir}}/logsearch.log\"\n },\n {\n \"type\":\"logsearch_feeder\",\n \"rowtype\":\"service\",\n \"path\":\"{{logfeeder_log_dir}}/logfeeder.log\"\n },\n {\n \"type\":\"logsearch_perf\",\n \"rowtype\":\"service\",\n \"path\":\"{{l ogsearch_log_dir}}/logsearch-performance.log\"\n },\n {\n \"type\":\"ranger_admin\",\n \"rowtype\":\"service\",\n \"path\":\"{{ranger_admin_log_dir}}/xa_portal.log\"\n },\n {\n \"type\":\"ranger_dbpatch\",\n \"is_enabled\":\"true\",\n \"path\":\"{{ranger_admin_log_dir}}/ranger_db_patch.log\"\n },\n {\n \"type\":\"ranger_kms\",\n \"rowtype\":\"service\",\n \"path\":\"{{ranger_kms_log_dir}}/kms.log\"\n },\n {\n \"type\":\"ranger_usersync\",\n \"rowtype\":\"service\",\n \"path\":\"{{ranger_usersync_log_dir}}/usersync.log\"\n },\n {\n \"type\":\"oozie_app\",\n \"rowtype\":\"service\",\n \"path\":\"{{oozie_log_dir}}/oozie.log\"\n },\n {\n \"type\":\"yarn_nodemanager\",\n \"rowtype\":\"service\",\n \"path\":\"{{yarn_log_dir_prefix}}/yarn/yarn-yarn-nodemanager-*.log\"\n },\n {\n \"type\":\"yarn_resourcemanager\",\n \"rowtype\":\"service\",\n \"p ath\":\"{{yarn_log_dir_prefix}}/yarn/yarn-yarn-resourcemanager-*.log\"\n },\n {\n \"type\":\"yarn_timelineserver\",\n \"rowtype\":\"service\",\n \"path\":\"{{yarn_log_dir_prefix}}/yarn/yarn-yarn-timelineserver-*.log\"\n },\n {\n \"type\":\"yarn_historyserver\",\n \"rowtype\":\"service\",\n \"path\":\"{{yarn_log_dir_prefix}}/yarn/yarn-yarn-historyserver-*.log\"\n },\n {\n \"type\":\"yarn_jobsummary\",\n \"rowtype\":\"service\",\n \"path\":\"{{yarn_log_dir_prefix}}/yarn/hadoop-mapreduce.jobsummary.log\"\n },\n {\n \"type\":\"storm_drpc\",\n \"rowtype\":\"service\",\n \"path\":\"{{storm_log_dir}}/drpc.log\"\n },\n {\n \"type\":\"storm_logviewer\",\n \"rowtype\":\"service\",\n \"path\":\"{{storm_log_dir}}/logviewer.log\"\n },\n {\n \"type\":\"storm_nimbus\",\n \"rowtype\":\"service\",\n \"path\":\"{{storm_log_dir}}/nimbus.log\"\n },\n {\n \"type\":\"st orm_supervisor\",\n \"rowtype\":\"service\",\n \"path\":\"{{storm_log_dir}}/supervisor.log\"\n },\n {\n \"type\":\"storm_ui\",\n \"rowtype\":\"service\",\n \"path\":\"{{storm_log_dir}}/ui.log\"\n },\n {\n \"type\":\"storm_worker\",\n \"rowtype\":\"service\",\n \"path\":\"{{storm_log_dir}}/*worker*.log\"\n },\n {\n \"type\":\"zookeeper\",\n \"rowtype\":\"service\",\n \"path\":\"{{zk_log_dir}}/zookeeper/zookeeper*.out\"\n },\n {\n \"type\":\"hdfs_audit\",\n \"rowtype\":\"audit\",\n \"is_enabled\":\"true\",\n \"add_fields\":{\n \"logType\":\"HDFSAudit\",\n \"enforcer\":\"hadoop-acl\",\n \"repoType\":\"1\",\n \"repo\":\"hdfs\"\n },\n \"path\":\"{{hdfs_log_dir_prefix}}/hdfs/hdfs-audit.log\"\n }\n \n ],\n \"filter\":[\n {\n \"filter\":\"grok\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"accumulo_mas ter\"\n ]\n \n }\n \n },\n \"log4j_format\":\"%d{ISO8601} [%-8c{2}] %-5p: %m%n\",\n \"multiline_pattern\":\"^(%{TIMESTAMP_ISO8601:logtime})\",\n \"message_pattern\":\"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}\\[%{JAVACLASS:logger_name}\\]%{SPACE}%{LOGLEVEL:level}:%{SPACE}%{GREEDYDATA:log_message}\",\n \"post_map_values\":{\n \"logtime\":{\n \"map_date\":{\n \"date_pattern\":\"yyyy-MM-dd HH:mm:ss,SSS\"\n }\n \n }\n \n }\n \n },\n {\n \"filter\":\"grok\",\n \"comment\":\"This one has one extra space after LEVEL\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"accumulo_gc\",\n \"accumulo_monitor\",\n \"accumulo_tracer\",\n \"accumulo_tserver\"\n ]\n \n }\n \n },\n \"log4j_format\":\"%d{ISO8601} [%-8c{2}] %-5p: %X{application} %m%n\",\n \"multiline_pattern\":\"^(%{TIMESTAMP_ISO8601:logtime})\",\n \"message_pattern\":\"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}\\[%{JAVACLASS:logger_name}\\]%{SPACE}%{LOGLEVEL:level}%{SPACE}:%{SPACE}%{GREEDYDATA:log_message}\",\n \"post_map_values\":{\n \"logtime\":{\n \"map_date\":{\n \"date_pattern\":\"yyyy-MM-dd HH:mm:ss,SSS\"\n }\n \n }\n \n }\n \n },\n {\n \"filter\":\"grok\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"atlas_app\",\n \"falcon_app\"\n ]\n \n }\n \n },\n \"log4j_format\":\"%d %-5p - [%t:%x] ~ %m (%c{1}:%L)%n\",\n \"multiline_pattern\":\"^(%{TIMESTAMP_ISO8601:logtime})\",\n \"message_pattern\":\"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{SPACE}-%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}~%{SPACE}%{GREEDYDATA:log_message}\",\n \"post_map_values\" :{\n \"logtime\":{\n \"map_date\":{\n \"date_pattern\":\"yyyy-MM-dd HH:mm:ss,SSS\"\n }\n \n }\n \n }\n \n },\n {\n \"filter\":\"grok\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"ams_collector\"\n ]\n \n }\n \n },\n \"log4j_format\":\"%d{ISO8601} %p %c: %m%n\",\n \"multiline_pattern\":\"^(%{TIMESTAMP_ISO8601:logtime})\",\n \"message_pattern\":\"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}:%{SPACE}%{GREEDYDATA:log_message}\",\n \"post_map_values\":{\n \"logtime\":{\n \"map_date\":{\n \"date_pattern\":\"yyyy-MM-dd HH:mm:ss,SSS\"\n }\n \n }\n \n }\n \n },\n {\n \"filter\":\"grok\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"ams_hbase_master\",\n \"ams_hbase_regionserver\",\n \"hbase_master\",\n \"hbase_regionserver\"\n ]\n \n }\n \n },\n \"log4j_format\":\"%d{ISO8601} %-5p [%t] %c{2}: %m%n\",\n \"multiline_pattern\":\"^(%{TIMESTAMP_ISO8601:logtime})\",\n \"message_pattern\":\"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{JAVACLASS:logger_name}:%{SPACE}%{GREEDYDATA:log_message}\",\n \"post_map_values\":{\n \"logtime\":{\n \"map_date\":{\n \"date_pattern\":\"yyyy-MM-dd HH:mm:ss,SSS\"\n }\n \n }\n \n }\n \n },\n {\n \"filter\":\"grok\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"ambari_agent\"\n ]\n \n }\n \n },\n \"log4j_format\":\"\",\n \"multiline_pattern\":\"^(%{LOGLEVEL:level} %{TIMESTAMP_ISO8601:logtime})\",\n \" message_pattern\":\"(?m)^%{LOGLEVEL:level} %{TIMESTAMP_ISO8601:logtime} %{JAVAFILE:file}:%{INT:line_number} - %{GREEDYDATA:log_message}\",\n \"post_map_values\":{\n \"logtime\":{\n \"map_date\":{\n \"date_pattern\":\"yyyy-MM-dd HH:mm:ss,SSS\"\n }\n \n },\n \"level\":{\n \"map_fieldvalue\":{\n \"pre_value\":\"WARNING\",\n \"post_value\":\"WARN\"\n }\n \n }\n \n }\n \n },\n {\n \"filter\":\"grok\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"ambari_server\"\n ]\n \n }\n \n },\n \"log4j_format\":\"%d{DATE} %5p [%t] %c{1}:%L - %m%n\",\n \"multiline_pattern\":\"^(%{USER_SYNC_DATE:logtime})\",\n \"message_pattern\":\"(?m)^%{USER_SYNC_DATE:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{JAVACLASS:logger_name}:%{INT:line_number} %{SPACE}-%{SPACE}%{GREEDYDATA:log_message}\",\n \"post_map_values\":{\n \"logtime\":{\n \"map_date\":{\n \"date_pattern\":\"dd MMM yyyy HH:mm:ss\"\n }\n \n }\n \n }\n \n },\n {\n \"filter\":\"grok\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"hdfs_datanode\",\n \"hdfs_journalnode\",\n \"hdfs_secondarynamenode\",\n \"hdfs_namenode\",\n \"hdfs_zkfc\",\n \"knox_gateway\",\n \"knox_cli\",\n \"knox_ldap\",\n \"mapred_historyserver\",\n \"yarn_historyserver\",\n \"yarn_jobsummary\",\n \"yarn_nodemanager\",\n \"yarn_resourcemanager\",\n \"yarn_timelineserver\"\n ]\n \n }\n \n },\n \"log4j_format\":\"%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n\",\n \"multiline_pattern\":\"^(%{TIMESTA MP_ISO8601:logtime})\",\n \"message_pattern\":\"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\(%{JAVAFILE:file}:%{JAVAMETHOD:method}\\(%{INT:line_number}\\)\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}\",\n \"post_map_values\":{\n \"logtime\":{\n \"map_date\":{\n \"date_pattern\":\"yyyy-MM-dd HH:mm:ss,SSS\"\n }\n \n }\n \n }\n \n },\n {\n \"filter\":\"grok\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"hive_hiveserver2\",\n \"hive_metastore\"\n ]\n \n }\n \n },\n \"log4j_format\":\"%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n\",\n \"multiline_pattern\":\"^(%{TIMESTAMP_ISO8601:logtime})\",\n \"message_pattern\":\"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]:%{SPACE}%{JAVACLASS:logger_name}%{SP ACE}\\(%{JAVAFILE:file}:%{JAVAMETHOD:method}\\(%{INT:line_number}\\)\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}\",\n \"post_map_values\":{\n \"logtime\":{\n \"map_date\":{\n \"date_pattern\":\"yyyy-MM-dd HH:mm:ss,SSS\"\n }\n \n }\n \n }\n \n },\n {\n \"filter\":\"grok\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"kafka_controller\",\n \"kafka_request\",\n \"kafka_logcleaner\"\n ]\n \n }\n \n },\n \"log4j_format\":\"[%d] %p %m (%c)%n\",\n \"multiline_pattern\":\"^(\\[%{TIMESTAMP_ISO8601:logtime}\\])\",\n \"message_pattern\":\"(?m)^\\[%{TIMESTAMP_ISO8601:logtime}\\]%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{GREEDYDATA:log_message}\",\n \"post_map_values\":{\n \"logtime\":{\n \"map_date\":{\n \"date_pattern\":\"yyyy-MM-dd HH:mm:ss,S SS\"\n }\n \n }\n \n }\n \n },\n {\n \"filter\":\"grok\",\n \"comment\":\"Suppose to be same log4j pattern as other kafka processes, but some reason thread is not printed\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"kafka_server\",\n \"kafka_statechange\"\n ]\n \n }\n \n },\n \"log4j_format\":\"[%d] %p %m (%c)%n\",\n \"multiline_pattern\":\"^(\\[%{TIMESTAMP_ISO8601:logtime}\\])\",\n \"message_pattern\":\"(?m)^\\[%{TIMESTAMP_ISO8601:logtime}\\]%{SPACE}%{LOGLEVEL:level}%{SPACE}%{GREEDYDATA:log_message}\",\n \"post_map_values\":{\n \"logtime\":{\n \"map_date\":{\n \"date_pattern\":\"yyyy-MM-dd HH:mm:ss,SSS\"\n }\n \n }\n \n }\n \n },\n {\n \"filter\":\"grok\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"o ozie_app\"\n ]\n \n }\n \n },\n \"log4j_format\":\"%d{ISO8601} %5p %c{1}:%L - SERVER[${oozie.instance.id}] %m%n\",\n \"multiline_pattern\":\"^(%{TIMESTAMP_ISO8601:logtime})\",\n \"message_pattern\":\"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{DATA:logger_name}:%{INT:line_number}%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}\",\n \"post_map_values\":{\n \"logtime\":{\n \"map_date\":{\n \"date_pattern\":\"yyyy-MM-dd HH:mm:ss,SSS\"\n }\n \n }\n \n }\n \n },\n {\n \"filter\":\"grok\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"logsearch_app\",\n \"logsearch_feeder\",\n \"logsearch_perf\",\n \"ranger_admin\",\n \"ranger_dbpatch\"\n ]\n \n }\n \n },\n \"log4j_format\":\"%d [%t] %-5p %C{6} (%F:%L) - %m%n\",\n \"multiline_pattern\":\"^(%{TIMESTAMP_ISO8601:logtime})\",\n \"message_pattern\":\"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\(%{JAVAFILE:file}:%{INT:line_number}\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}\",\n \"post_map_values\":{\n \"logtime\":{\n \"map_date\":{\n \"date_pattern\":\"yyyy-MM-dd HH:mm:ss,SSS\"\n }\n \n }\n \n }\n \n },\n {\n \"filter\":\"grok\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"ranger_kms\"\n ]\n \n }\n \n },\n \"log4j_format\":\"%d{ISO8601} %-5p %c{1} - %m%n\",\n \"multiline_pattern\":\"^(%{TIMESTAMP_ISO8601:logtime})\",\n \"message_pattern\":\"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}\",\n \"post_map_values\":{\n \"logtime\":{\n \"map_date\":{\n \"date_pattern\":\"yyyy-MM-dd HH:mm:ss,SSS\"\n }\n \n }\n \n }\n \n },\n {\n \"filter\":\"grok\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"ranger_usersync\"\n ]\n \n }\n \n },\n \"log4j_format\":\"%d{dd MMM yyyy HH:mm:ss} %5p %c{1} [%t] - %m%n\",\n \"multiline_pattern\":\"^(%{USER_SYNC_DATE:logtime})\",\n \"message_pattern\":\"(?m)^%{USER_SYNC_DATE:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}\",\n \"post_map_values\":{\n \"logtime\":{\n \"map_date\":{\n \"date_pattern\":\"dd MMM yyyy HH:mm:ss\"\n }\n \n }\n \n }\n \n },\n {\n \"filter\":\"grok\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"storm_drpc\",\n \"storm_logviewer\",\n \"storm_nimbus\",\n \"storm_supervisor\",\n \"storm_ui\",\n \"storm_worker\"\n ]\n \n }\n \n },\n \"log4j_format\":\"\",\n \"multiline_pattern\":\"^(%{TIMESTAMP_ISO8601:logtime})\",\n \"message_pattern\":\"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\[%{LOGLEVEL:level}\\]%{SPACE}%{SPACE}%{GREEDYDATA:log_message}\",\n \"post_map_values\":{\n \"logtime\":{\n \"map_date\":{\n \"date_pattern\":\"yyyy-MM-dd HH:mm:ss.SSS\"\n }\n \n }\n \n }\n \n },\n {\n \"filter\":\"grok\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"zookeeper\"\n ]\n \n }\n \n },\n \"log4j_format\":\"%d{ISO8601} - %-5p [%t:%C{1}@% L] - %m%n\",\n \"multiline_pattern\":\"^(%{TIMESTAMP_ISO8601:logtime})\",\n \"message_pattern\":\"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}-%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\@%{INT:line_number}\\]%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}\",\n \"post_map_values\":{\n \"logtime\":{\n \"map_date\":{\n \"date_pattern\":\"yyyy-MM-dd HH:mm:ss,SSS\"\n }\n \n }\n \n }\n \n },\n {\n \"filter\":\"grok\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"hdfs_audit\"\n ]\n \n }\n \n },\n \"log4j_format\":\"%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n\",\n \"multiline_pattern\":\"^(%{TIMESTAMP_ISO8601:evtTime})\",\n \"message_pattern\":\"(?m)^%{TIMESTAMP_ISO8601:evtTime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}:%{SPACE}%{GREEDYDATA:log_message}\",\n \"post_map_values\":{\n \"evtTime\":{\n \"map_date\":{\n \"date_pattern\":\"yyyy-MM-dd HH:mm:ss,SSS\"\n }\n \n }\n \n }\n \n },\n {\n \"filter\":\"keyvalue\",\n \"sort_order\":1,\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"hdfs_audit\"\n ]\n \n }\n \n },\n \"source_field\":\"log_message\",\n \"value_split\":\"=\",\n \"field_split\":\"\t\",\n \"post_map_values\":{\n \"src\":{\n \"map_fieldname\":{\n \"new_fieldname\":\"resource\"\n }\n \n },\n \"ip\":{\n \"map_fieldname\":{\n \"new_fieldname\":\"cliIP\"\n }\n \n },\n \"allowed\":[\n {\n \"map_fieldvalue\":{\n \"pre_value\":\"true\",\n \"post_value\":\"1\"\n }\n \n },\n {\n \"map _fieldvalue\":{\n \"pre_value\":\"false\",\n \"post_value\":\"0\"\n }\n \n },\n {\n \"map_fieldname\":{\n \"new_fieldname\":\"result\"\n }\n \n }\n \n ],\n \"cmd\":{\n \"map_fieldname\":{\n \"new_fieldname\":\"action\"\n }\n \n },\n \"proto\":{\n \"map_fieldname\":{\n \"new_fieldname\":\"cliType\"\n }\n \n },\n \"callerContext\":{\n \"map_fieldname\":{\n \"new_fieldname\":\"req_caller_id\"\n }\n \n }\n \n }\n \n },\n {\n \"filter\":\"grok\",\n \"sort_order\":2,\n \"source_field\":\"ugi\",\n \"remove_source_field\":\"false\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"hdfs_audit\"\n ]\n \n }\n \n },\n \"message_pattern\":\"%{USERNAME:p_user}.+auth:%{USERNAME:p_authType}.+via %{USERNAME:k_user}.+auth:%{USERNAME:k_authType}|%{USERNAME:user}.+auth:%{USERNAME:authType}|%{USERNAME:x_user}\",\n \"post_map_values\":{\n \"user\":{\n \"map_fieldname\":{\n \"new_fieldname\":\"reqUser\"\n }\n \n },\n \"x_user\":{\n \"map_fieldname\":{\n \"new_fieldname\":\"reqUser\"\n }\n \n },\n \"p_user\":{\n \"map_fieldname\":{\n \"new_fieldname\":\"reqUser\"\n }\n \n },\n \"k_user\":{\n \"map_fieldname\":{\n \"new_fieldname\":\"proxyUsers\"\n }\n \n },\n \"p_authType\":{\n \"map_fieldname\":{\n \"new_fieldname\":\"authType\"\n }\n \n },\n \"k_authType\":{\n \"map_fieldname\":{\n \"new_fie ldname\":\"proxyAuthType\"\n }\n \n }\n \n }\n \n }\n \n ],\n \"output\":[\n {\n \"is_enabled\":\"{{solr_service_logs_enable}}\",\n \"comment\":\"Output to solr for service logs\",\n \"destination\":\"solr\",\n \"zk_connect_string\":\"{{zookeeper_quorum}}{{solr_znode}}\",\n \"collection\":\"{{solr_collection_service_logs}}\",\n \"number_of_shards\": \"{{logsearch_collection_service_logs_numshards}}\",\n \"splits_interval_mins\": \"{{logsearch_service_logs_split_interval_mins}}\",\n \"conditions\":{\n \"fields\":{\n \"rowtype\":[\n \"service\"\n ]\n \n }\n \n }\n \n },\n {\n \"comment\":\"Output to solr for audit records\",\n \"is_enabled\":\"{{solr_audit_logs_enable}}\",\n \"destination\":\"solr\",\n \"zk_connect_string\":\"{{zookeeper_quorum}}{{solr_znode}}\",\n \"collection\":\"{{solr_collec tion_audit_logs}}\",\n \"number_of_shards\": \"{{logsearch_collection_audit_logs_numshards}}\",\n \"splits_interval_mins\": \"{{logsearch_audit_logs_split_interval_mins}}\",\n \"conditions\":{\n \"fields\":{\n \"rowtype\":[\n \"audit\"\n ]\n \n }\n \n }\n \n },\n {\n \"is_enabled\":\"{{kafka_service_logs_enable}}\",\n \"destination\":\"kafka\",\n \"broker_list\":\"{{kafka_broker_list}}\",\n \"topic\":\"{{kafka_topic_service_logs}}\",\n \"kafka.security.protocol\":\"{{kafka_security_protocol}}\",\n \"kafka.sasl.kerberos.service.name\":\"{{kafka_kerberos_service_name}}\",\n \"conditions\":{\n \"fields\":{\n \"rowtype\":[\n \"service\"\n ]\n \n }\n \n }\n \n },\n {\n \"is_enabled\":\"{{kafka_audit_logs_enable}}\",\n \"destination\":\"kafka\",\n \"broker_list\":\"{{kafka_brok er_list}}\",\n \"topic\":\"{{kafka_topic_audit_logs}}\",\n \"kafka.security.protocol\":\"{{kafka_security_protocol}}\",\n \"kafka.sasl.kerberos.service.name\":\"{{kafka_kerberos_service_name}}\",\n \"conditions\":{\n \"fields\":{\n \"rowtype\":[\n \"audit\"\n ]\n \n }\n \n }\n \n }\n \n ]\n \n}" } }, "configuration_attributes": {
