AMBARI-19001. Support configurable grok filters (output + ambari) (oleewere)
Change-Id: Ibf02fc6ca6759435ac14dc59de9f71782d7ea709 Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/c295941b Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/c295941b Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/c295941b Branch: refs/heads/branch-feature-AMBARI-18901 Commit: c295941b7259b0deb1e452522ceeca2d3e9d7d4a Parents: 1fca37d Author: oleewere <oleew...@gmail.com> Authored: Tue Nov 29 13:46:32 2016 +0100 Committer: oleewere <oleew...@gmail.com> Committed: Tue Nov 29 15:54:26 2016 +0100 ---------------------------------------------------------------------- .../configuration/logfeeder-ambari-config.xml | 37 ++ .../configuration/logfeeder-output-config.xml | 37 ++ .../LOGSEARCH/0.5.0/metainfo.xml | 5 + .../LOGSEARCH/0.5.0/package/scripts/params.py | 17 +- .../0.5.0/package/scripts/setup_logfeeder.py | 10 + .../templates/input.config-ambari.json.j2 | 602 ------------------- .../package/templates/output.config.json.j2 | 61 -- .../properties/input.config-ambari.json.j2 | 602 +++++++++++++++++++ .../0.5.0/properties/output.config.json.j2 | 61 ++ .../stacks/2.4/LOGSEARCH/test_logfeeder.py | 14 +- .../test/python/stacks/2.4/configs/default.json | 6 + 11 files changed, 780 insertions(+), 672 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/c295941b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-ambari-config.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-ambari-config.xml b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-ambari-config.xml new file mode 100644 index 0000000..64d7946 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-ambari-config.xml @@ -0,0 +1,37 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration supports_final="false" supports_adding_forbidden="true"> + <property> + <name>content</name> + <display-name>Ambari LogFeeder config</display-name> + <description>Grok filters and input file locations for ambari related log files</description> + <value/> + <property-type>VALUE_FROM_PROPERTY_FILE</property-type> + <value-attributes> + <type>content</type> + <show-property-name>false</show-property-name> + <property-file-name>input.config-ambari.json.j2</property-file-name> + <property-file-type>text</property-file-type> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> +</configuration> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/c295941b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-output-config.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-output-config.xml b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-output-config.xml new file mode 100644 index 0000000..85654c8 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-output-config.xml @@ -0,0 +1,37 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration supports_final="false" supports_adding_forbidden="true"> + <property> + <name>content</name> + <display-name>LogFeeder output config</display-name> + <description>Output configuration for LogFeeder (data shipping)</description> + <value/> + <property-type>VALUE_FROM_PROPERTY_FILE</property-type> + <value-attributes> + <type>content</type> + <show-property-name>false</show-property-name> + <property-file-name>output.config.json.j2</property-file-name> + <property-file-type>text</property-file-type> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> +</configuration> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/c295941b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml index 6d1cf38..f69bd3c 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml @@ -72,6 +72,7 @@ <config-type>logsearch-admin-json</config-type> <config-type>logsearch-service_logs-solrconfig.xml</config-type> <config-type>logsearch-audit_logs-solrconfig.xml</config-type> + <config-type>logfeeder-custom-logsearch-config</config-type> </configuration-dependencies> </component> @@ -111,6 +112,10 @@ <config-type>logfeeder-env</config-type> <config-type>logfeeder-grok</config-type> <config-type>logfeeder-log4j</config-type> + <config-type>logfeeder-system_log-env</config-type> + <config-type>logfeeder-ambari-config</config-type> + <config-type>logfeeder-output-config</config-type> + <config-type>logfeeder-custom-logsearch-config</config-type> </configuration-dependencies> </component> http://git-wip-us.apache.org/repos/asf/ambari/blob/c295941b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py index bdb0c10..d086f9f 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py @@ -251,16 +251,19 @@ logfeeder_truststore_location = config['configurations']['logfeeder-env']['logfe logfeeder_truststore_password = config['configurations']['logfeeder-env']['logfeeder_truststore_password'] logfeeder_truststore_type = config['configurations']['logfeeder-env']['logfeeder_truststore_type'] -logfeeder_default_services = ['ambari', 'logsearch'] -logfeeder_default_config_file_names = ['global.config.json', 'output.config.json'] + ['input.config-%s.json' % (tag) for tag in logfeeder_default_services] +logfeeder_ambari_config_content = config['configurations']['logfeeder-ambari-config']['content'] +logfeeder_output_config_content = config['configurations']['logfeeder-output-config']['content'] + +logfeeder_default_services = ['logsearch'] +logfeeder_default_config_file_names = ['global.config.json'] + ['input.config-%s.json' % (tag) for tag in logfeeder_default_services] logfeeder_custom_config_file_names = ['input.config-%s.json' % (tag.replace('-logsearch-conf', '')) for tag, content in logfeeder_metadata.iteritems() if any(logfeeder_metadata)] if logfeeder_system_log_enabled: - default_config_files = ','.join(logfeeder_default_config_file_names + logfeeder_custom_config_file_names + default_config_files = ','.join(['output.json','input.config-ambari.json'] + logfeeder_default_config_file_names + logfeeder_custom_config_file_names + ['input.config-system_messages.json', 'input.config-secure_log.json']) else: - default_config_files = ','.join(logfeeder_default_config_file_names + logfeeder_custom_config_file_names) + default_config_files = ','.join(['output.config.json','input.config-ambari.json'] + logfeeder_default_config_file_names + logfeeder_custom_config_file_names) logfeeder_grok_patterns = config['configurations']['logfeeder-grok']['default_grok_patterns'] @@ -293,8 +296,10 @@ logfeeder_properties['logfeeder.config.files'] = format(logfeeder_properties['lo logfeeder_properties['logfeeder.solr.zk_connect_string'] = zookeeper_quorum + infra_solr_znode if security_enabled: - logfeeder_properties['logfeeder.solr.kerberos.enable'] = 'true' - logfeeder_properties['logfeeder.solr.jaas.file'] = logfeeder_jaas_file + if 'logfeeder.solr.kerberos.enable' not in logfeeder_properties: + logfeeder_properties['logfeeder.solr.kerberos.enable'] = 'true' + if 'logfeeder.solr.jaas.file' not in logfeeder_properties: + logfeeder_properties['logfeeder.solr.jaas.file'] = logfeeder_jaas_file logfeeder_checkpoint_folder = logfeeder_properties['logfeeder.checkpoint.folder'] http://git-wip-us.apache.org/repos/asf/ambari/blob/c295941b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py index 021c167..9582334 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py @@ -62,6 +62,16 @@ def setup_logfeeder(): encoding="utf-8" ) + File(format("{logsearch_logfeeder_conf}/input.config-ambari.json"), + content=InlineTemplate(params.logfeeder_ambari_config_content), + encoding="utf-8" + ) + + File(format("{logsearch_logfeeder_conf}/output.config.json"), + content=InlineTemplate(params.logfeeder_output_config_content), + encoding="utf-8" + ) + for file_name in params.logfeeder_default_config_file_names: File(format("{logsearch_logfeeder_conf}/" + file_name), content=Template(file_name + ".j2") http://git-wip-us.apache.org/repos/asf/ambari/blob/c295941b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-ambari.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-ambari.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-ambari.json.j2 deleted file mode 100644 index ad4adb2..0000000 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-ambari.json.j2 +++ /dev/null @@ -1,602 +0,0 @@ -{# - # Licensed to the Apache Software Foundation (ASF) under one - # or more contributor license agreements. See the NOTICE file - # distributed with this work for additional information - # regarding copyright ownership. The ASF licenses this file - # to you under the Apache License, Version 2.0 (the - # "License"); you may not use this file except in compliance - # with the License. You may obtain a copy of the License at - # - # http://www.apache.org/licenses/LICENSE-2.0 - # - # Unless required by applicable law or agreed to in writing, software - # distributed under the License is distributed on an "AS IS" BASIS, - # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - # See the License for the specific language governing permissions and - # limitations under the License. - #} -{ - "input":[ - { - "type":"ambari_agent", - "rowtype":"service", - "path":"{{ambari_agent_log_dir}}/ambari-agent.log" - }, - { - "type":"ambari_server", - "rowtype":"service", - "path":"{{ambari_server_log_dir}}/ambari-server.log" - }, - { - "type":"ambari_alerts", - "rowtype":"service", - "add_fields":{ - "level":"UNKNOWN" - }, - "path":"{{ambari_server_log_dir}}/ambari-alerts.log" - }, - { - "type":"ambari_config_changes", - "rowtype":"service", - "path":"{{ambari_server_log_dir}}/ambari-config-changes.log" - }, - { - "type":"ambari_eclipselink", - "rowtype":"service", - "path":"{{ambari_server_log_dir}}/ambari-eclipselink.log" - }, - { - "type":"ambari_server_check_database", - "rowtype":"service", - "path":"{{ambari_server_log_dir}}/ambari-server-check-database.log" - }, - { - "type":"ambari_audit", - "rowtype":"audit", - "add_fields":{ - "logType":"AmbariAudit", - "enforcer":"ambari-acl", - "repoType":"1", - "repo":"ambari", - "level":"INFO" - }, - "path":"{{ambari_server_log_dir}}/ambari-audit.log" - } - - ], - "filter":[ - { - "filter":"grok", - "conditions":{ - "fields":{ - "type":[ - "ambari_agent" - ] - - } - - }, - "log4j_format":"", - "multiline_pattern":"^(%{LOGLEVEL:level} %{TIMESTAMP_ISO8601:logtime})", - "message_pattern":"(?m)^%{LOGLEVEL:level} %{TIMESTAMP_ISO8601:logtime} %{JAVAFILE:file}:%{INT:line_number} - %{GREEDYDATA:log_message}", - "post_map_values":{ - "logtime":{ - "map_date":{ - "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" - } - - }, - "level":{ - "map_fieldvalue":{ - "pre_value":"WARNING", - "post_value":"WARN" - } - - } - - } - - }, - { - "filter":"grok", - "conditions":{ - "fields":{ - "type":[ - "ambari_server" - ] - - } - - }, - "log4j_format":"%d{DATE} %5p [%t] %c{1}:%L - %m%n", - "multiline_pattern":"^(%{USER_SYNC_DATE:logtime})", - "message_pattern":"(?m)^%{USER_SYNC_DATE:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{JAVACLASS:logger_name}:%{INT:line_number}%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", - "post_map_values":{ - "logtime":{ - "map_date":{ - "target_date_pattern":"dd MMM yyyy HH:mm:ss" - } - - } - - } - - }, - { - "filter":"grok", - "conditions":{ - "fields":{ - "type":[ - "ambari_alerts" - ] - - } - - }, - "log4j_format":"%d{DATE} %5p [%t] %c{1}:%L - %m%n", - "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", - "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{GREEDYDATA:log_message}", - "post_map_values":{ - "logtime":{ - "map_date":{ - "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" - } - - } - - } - - }, - { - "filter":"grok", - "conditions":{ - "fields":{ - "type":[ - "ambari_config_changes" - ] - - } - - }, - "log4j_format":"%d{DATE} %5p [%t] %c{1}:%L - %m%n", - "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", - "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", - "post_map_values":{ - "logtime":{ - "map_date":{ - "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" - } - - } - - } - - }, - { - "filter":"grok", - "conditions":{ - "fields":{ - "type":[ - "ambari_eclipselink" - ] - - } - - }, - "log4j_format":"%d{DATE} %5p [%t] %c{1}:%L - %m%n", - "multiline_pattern":"^(\\[EL%{SPACE}%{LOGLEVEL:level}\\])", - "message_pattern":"(?m)^\\[EL%{SPACE}%{LOGLEVEL:level}\\]:%{SPACE}%{TIMESTAMP_ISO8601:logtime}%{GREEDYDATA:log_message}", - "post_map_values":{ - "logtime":{ - "map_date":{ - "target_date_pattern":"yyyy-MM-dd HH:mm:ss.SSS" - } - - }, - "level":{ - "map_fieldvalue":{ - "pre_value":"Warning", - "post_value":"Warn" - } - - } - - } - - }, - { - "filter":"grok", - "conditions":{ - "fields":{ - "type":[ - "ambari_server_check_database" - ] - - } - - }, - "log4j_format":"%d{DATE} %5p [%t] %c{1}:%L - %m%n", - "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", - "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", - "post_map_values":{ - "logtime":{ - "map_date":{ - "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" - } - - } - - } - - }, - { - "filter":"grok", - "conditions":{ - "fields":{ - "type":[ - "ambari_audit" - ] - - } - - }, - "log4j_format":"%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n", - "multiline_pattern":"^(%{TIMESTAMP_ISO8601:evtTime})", - "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:evtTime},%{SPACE}%{GREEDYDATA:log_message}", - "post_map_values":{ - "evtTime":{ - "map_date":{ - "target_date_pattern":"yyyy-MM-dd'T'HH:mm:ss.SSSXX" - } - - } - - } - - }, - { - "filter":"keyvalue", - "sort_order":1, - "conditions":{ - "fields":{ - "type":[ - "ambari_audit" - ] - - } - - }, - "source_field":"log_message", - "field_split":", ", - "value_borders":"()", - "post_map_values":{ - "User":{ - "map_fieldvalue":{ - "pre_value":"null", - "post_value":"unknown" - }, - "map_fieldname":{ - "new_fieldname":"reqUser" - } - }, - "Hostname":{ - "map_fieldname":{ - "new_fieldname":"host" - } - }, - "Host name":{ - "map_fieldname":{ - "new_fieldname":"host" - } - }, - "RemoteIp":{ - "map_fieldname":{ - "new_fieldname":"cliIP" - } - }, - "RequestType":{ - "map_fieldname":{ - "new_fieldname":"cliType" - } - }, - "RequestId":{ - "map_fieldname":{ - "new_fieldname":"request_id" - } - }, - "TaskId":{ - "map_fieldname":{ - "new_fieldname":"task_id" - } - }, - "Operation":{ - "map_fieldname":{ - "new_fieldname":"action" - } - }, - "url":{ - "map_fieldname":{ - "new_fieldname":"resource" - } - }, - "ResourcePath":{ - "map_fieldname":{ - "new_fieldname":"resource" - } - }, - "Cluster name":{ - "map_fieldname":{ - "new_fieldname":"cluster" - } - }, - "Reason":{ - "map_fieldname":{ - "new_fieldname":"reason" - } - }, - "Base URL":{ - "map_fieldname":{ - "new_fieldname":"ws_base_url" - } - }, - "Command":{ - "map_fieldvalue":{ - "pre_value":"null", - "post_value":"unknown" - }, - "map_fieldname":{ - "new_fieldname":"ws_command" - } - }, - "Component":{ - "map_fieldname":{ - "new_fieldname":"ws_component" - } - }, - "Details":{ - "map_fieldname":{ - "new_fieldname":"ws_details" - } - }, - "Display name":{ - "map_fieldvalue":{ - "pre_value":"null", - "post_value":"unknown" - }, - "map_fieldname":{ - "new_fieldname":"ws_display_name" - } - }, - "OS":{ - "map_fieldname":{ - "new_fieldname":"ws_os" - } - }, - "Repo id":{ - "map_fieldname":{ - "new_fieldname":"ws_repo_id" - } - }, - "Repo version":{ - "map_fieldvalue":{ - "pre_value":"null", - "post_value":"unknown" - }, - "map_fieldname":{ - "new_fieldname":"ws_repo_version" - } - }, - "Repositories":{ - "map_fieldname":{ - "new_fieldname":"ws_repositories" - } - }, - "Roles":{ - "map_fieldname":{ - "new_fieldname":"ws_roles" - } - }, - "Stack":{ - "map_fieldname":{ - "new_fieldname":"ws_stack" - } - }, - "Stack version":{ - "map_fieldname":{ - "new_fieldname":"ws_stack_version" - } - }, - "VersionNote":{ - "map_fieldvalue":{ - "pre_value":"null", - "post_value":"unknown" - }, - "map_fieldname":{ - "new_fieldname":"ws_version_note" - } - }, - "VersionNumber":{ - "map_fieldvalue":{ - "pre_value":"Vnull", - "post_value":"unknown" - }, - "map_fieldname":{ - "new_fieldname":"ws_version_number" - } - }, - "Status":[ - { - "map_fieldcopy":{ - "copy_name": "ws_status" - } - }, - { - "map_fieldvalue":{ - "pre_value":"Success", - "post_value":"1" - } - }, - { - "map_fieldvalue":{ - "pre_value":"Successfully queued", - "post_value":"1" - } - }, - { - "map_fieldvalue":{ - "pre_value":"QUEUED", - "post_value":"1" - } - }, - { - "map_fieldvalue":{ - "pre_value":"PENDING", - "post_value":"1" - } - }, - { - "map_fieldvalue":{ - "pre_value":"COMPLETED", - "post_value":"1" - } - }, - { - "map_fieldvalue":{ - "pre_value":"IN_PROGRESS", - "post_value":"1" - } - }, - { - "map_fieldvalue":{ - "pre_value":"Failed", - "post_value":"0" - } - }, - { - "map_fieldvalue":{ - "pre_value":"Failed to queue", - "post_value":"0" - } - }, - { - "map_fieldvalue":{ - "pre_value":"HOLDING", - "post_value":"0" - } - }, - { - "map_fieldvalue":{ - "pre_value":"HOLDING_FAILED", - "post_value":"0" - } - }, - { - "map_fieldvalue":{ - "pre_value":"HOLDING_TIMEDOUT", - "post_value":"0" - } - }, - { - "map_fieldvalue":{ - "pre_value":"FAILED", - "post_value":"0" - } - }, - { - "map_fieldvalue":{ - "pre_value":"TIMEDOUT", - "post_value":"0" - } - }, - { - "map_fieldvalue":{ - "pre_value":"ABORTED", - "post_value":"0" - } - }, - { - "map_fieldvalue":{ - "pre_value":"SKIPPED_FAILED", - "post_value":"0" - } - }, - { - "map_fieldname":{ - "new_fieldname":"result" - } - } - ], - "ResultStatus":[ - { - "map_fieldcopy":{ - "copy_name": "ws_result_status" - } - }, - { - "map_fieldvalue":{ - "pre_value":"200 OK", - "post_value":"1" - } - }, - { - "map_fieldvalue":{ - "pre_value":"201 Created", - "post_value":"1" - } - }, - { - "map_fieldvalue":{ - "pre_value":"202 Accepted", - "post_value":"1" - } - }, - { - "map_fieldvalue":{ - "pre_value":"400 Bad Request", - "post_value":"0" - } - }, - { - "map_fieldvalue":{ - "pre_value":"401 Unauthorized", - "post_value":"0" - } - }, - { - "map_fieldvalue":{ - "pre_value":"403 Forbidden", - "post_value":"0" - } - }, - { - "map_fieldvalue":{ - "pre_value":"404 Not Found", - "post_value":"0" - } - }, - { - "map_fieldvalue":{ - "pre_value":"409 Resource Conflict", - "post_value":"0" - } - }, - { - "map_fieldvalue":{ - "pre_value":"500 Internal Server Error", - "post_value":"0" - } - }, - { - "map_fieldname":{ - "new_fieldname":"result" - } - } - ] - - } - - } - - ] - -} http://git-wip-us.apache.org/repos/asf/ambari/blob/c295941b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/output.config.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/output.config.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/output.config.json.j2 deleted file mode 100644 index ff43323..0000000 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/output.config.json.j2 +++ /dev/null @@ -1,61 +0,0 @@ -{# - # Licensed to the Apache Software Foundation (ASF) under one - # or more contributor license agreements. See the NOTICE file - # distributed with this work for additional information - # regarding copyright ownership. The ASF licenses this file - # to you under the Apache License, Version 2.0 (the - # "License"); you may not use this file except in compliance - # with the License. You may obtain a copy of the License at - # - # http://www.apache.org/licenses/LICENSE-2.0 - # - # Unless required by applicable law or agreed to in writing, software - # distributed under the License is distributed on an "AS IS" BASIS, - # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - # See the License for the specific language governing permissions and - # limitations under the License. - #} -{ - "output":[ - { - "comment":"Output to solr for service logs", - "is_enabled":"{{solr_service_logs_enable}}", - "destination":"solr", - "zk_connect_string":"{{zookeeper_quorum}}{{infra_solr_znode}}", - "collection":"{{logsearch_solr_collection_service_logs}}", - "number_of_shards": "{{logsearch_collection_service_logs_numshards}}", - "splits_interval_mins": "{{logsearch_service_logs_split_interval_mins}}", - "conditions":{ - "fields":{ - "rowtype":[ - "service" - ] - - } - - } - - }, - { - "comment":"Output to solr for audit records", - "is_enabled":"{{solr_audit_logs_enable}}", - "destination":"solr", - "zk_connect_string":"{{zookeeper_quorum}}{{infra_solr_znode}}", - "collection":"{{logsearch_solr_collection_audit_logs}}", - "number_of_shards": "{{logsearch_collection_audit_logs_numshards}}", - "splits_interval_mins": "{{logsearch_audit_logs_split_interval_mins}}", - "conditions":{ - "fields":{ - "rowtype":[ - "audit" - ] - - } - - } - - } - - ] - -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/c295941b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/input.config-ambari.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/input.config-ambari.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/input.config-ambari.json.j2 new file mode 100644 index 0000000..ad4adb2 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/input.config-ambari.json.j2 @@ -0,0 +1,602 @@ +{# + # Licensed to the Apache Software Foundation (ASF) under one + # or more contributor license agreements. See the NOTICE file + # distributed with this work for additional information + # regarding copyright ownership. The ASF licenses this file + # to you under the Apache License, Version 2.0 (the + # "License"); you may not use this file except in compliance + # with the License. You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + #} +{ + "input":[ + { + "type":"ambari_agent", + "rowtype":"service", + "path":"{{ambari_agent_log_dir}}/ambari-agent.log" + }, + { + "type":"ambari_server", + "rowtype":"service", + "path":"{{ambari_server_log_dir}}/ambari-server.log" + }, + { + "type":"ambari_alerts", + "rowtype":"service", + "add_fields":{ + "level":"UNKNOWN" + }, + "path":"{{ambari_server_log_dir}}/ambari-alerts.log" + }, + { + "type":"ambari_config_changes", + "rowtype":"service", + "path":"{{ambari_server_log_dir}}/ambari-config-changes.log" + }, + { + "type":"ambari_eclipselink", + "rowtype":"service", + "path":"{{ambari_server_log_dir}}/ambari-eclipselink.log" + }, + { + "type":"ambari_server_check_database", + "rowtype":"service", + "path":"{{ambari_server_log_dir}}/ambari-server-check-database.log" + }, + { + "type":"ambari_audit", + "rowtype":"audit", + "add_fields":{ + "logType":"AmbariAudit", + "enforcer":"ambari-acl", + "repoType":"1", + "repo":"ambari", + "level":"INFO" + }, + "path":"{{ambari_server_log_dir}}/ambari-audit.log" + } + + ], + "filter":[ + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "ambari_agent" + ] + + } + + }, + "log4j_format":"", + "multiline_pattern":"^(%{LOGLEVEL:level} %{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{LOGLEVEL:level} %{TIMESTAMP_ISO8601:logtime} %{JAVAFILE:file}:%{INT:line_number} - %{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + + }, + "level":{ + "map_fieldvalue":{ + "pre_value":"WARNING", + "post_value":"WARN" + } + + } + + } + + }, + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "ambari_server" + ] + + } + + }, + "log4j_format":"%d{DATE} %5p [%t] %c{1}:%L - %m%n", + "multiline_pattern":"^(%{USER_SYNC_DATE:logtime})", + "message_pattern":"(?m)^%{USER_SYNC_DATE:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{JAVACLASS:logger_name}:%{INT:line_number}%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "target_date_pattern":"dd MMM yyyy HH:mm:ss" + } + + } + + } + + }, + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "ambari_alerts" + ] + + } + + }, + "log4j_format":"%d{DATE} %5p [%t] %c{1}:%L - %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + + } + + } + + }, + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "ambari_config_changes" + ] + + } + + }, + "log4j_format":"%d{DATE} %5p [%t] %c{1}:%L - %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + + } + + } + + }, + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "ambari_eclipselink" + ] + + } + + }, + "log4j_format":"%d{DATE} %5p [%t] %c{1}:%L - %m%n", + "multiline_pattern":"^(\\[EL%{SPACE}%{LOGLEVEL:level}\\])", + "message_pattern":"(?m)^\\[EL%{SPACE}%{LOGLEVEL:level}\\]:%{SPACE}%{TIMESTAMP_ISO8601:logtime}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "target_date_pattern":"yyyy-MM-dd HH:mm:ss.SSS" + } + + }, + "level":{ + "map_fieldvalue":{ + "pre_value":"Warning", + "post_value":"Warn" + } + + } + + } + + }, + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "ambari_server_check_database" + ] + + } + + }, + "log4j_format":"%d{DATE} %5p [%t] %c{1}:%L - %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + + } + + } + + }, + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "ambari_audit" + ] + + } + + }, + "log4j_format":"%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:evtTime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:evtTime},%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "evtTime":{ + "map_date":{ + "target_date_pattern":"yyyy-MM-dd'T'HH:mm:ss.SSSXX" + } + + } + + } + + }, + { + "filter":"keyvalue", + "sort_order":1, + "conditions":{ + "fields":{ + "type":[ + "ambari_audit" + ] + + } + + }, + "source_field":"log_message", + "field_split":", ", + "value_borders":"()", + "post_map_values":{ + "User":{ + "map_fieldvalue":{ + "pre_value":"null", + "post_value":"unknown" + }, + "map_fieldname":{ + "new_fieldname":"reqUser" + } + }, + "Hostname":{ + "map_fieldname":{ + "new_fieldname":"host" + } + }, + "Host name":{ + "map_fieldname":{ + "new_fieldname":"host" + } + }, + "RemoteIp":{ + "map_fieldname":{ + "new_fieldname":"cliIP" + } + }, + "RequestType":{ + "map_fieldname":{ + "new_fieldname":"cliType" + } + }, + "RequestId":{ + "map_fieldname":{ + "new_fieldname":"request_id" + } + }, + "TaskId":{ + "map_fieldname":{ + "new_fieldname":"task_id" + } + }, + "Operation":{ + "map_fieldname":{ + "new_fieldname":"action" + } + }, + "url":{ + "map_fieldname":{ + "new_fieldname":"resource" + } + }, + "ResourcePath":{ + "map_fieldname":{ + "new_fieldname":"resource" + } + }, + "Cluster name":{ + "map_fieldname":{ + "new_fieldname":"cluster" + } + }, + "Reason":{ + "map_fieldname":{ + "new_fieldname":"reason" + } + }, + "Base URL":{ + "map_fieldname":{ + "new_fieldname":"ws_base_url" + } + }, + "Command":{ + "map_fieldvalue":{ + "pre_value":"null", + "post_value":"unknown" + }, + "map_fieldname":{ + "new_fieldname":"ws_command" + } + }, + "Component":{ + "map_fieldname":{ + "new_fieldname":"ws_component" + } + }, + "Details":{ + "map_fieldname":{ + "new_fieldname":"ws_details" + } + }, + "Display name":{ + "map_fieldvalue":{ + "pre_value":"null", + "post_value":"unknown" + }, + "map_fieldname":{ + "new_fieldname":"ws_display_name" + } + }, + "OS":{ + "map_fieldname":{ + "new_fieldname":"ws_os" + } + }, + "Repo id":{ + "map_fieldname":{ + "new_fieldname":"ws_repo_id" + } + }, + "Repo version":{ + "map_fieldvalue":{ + "pre_value":"null", + "post_value":"unknown" + }, + "map_fieldname":{ + "new_fieldname":"ws_repo_version" + } + }, + "Repositories":{ + "map_fieldname":{ + "new_fieldname":"ws_repositories" + } + }, + "Roles":{ + "map_fieldname":{ + "new_fieldname":"ws_roles" + } + }, + "Stack":{ + "map_fieldname":{ + "new_fieldname":"ws_stack" + } + }, + "Stack version":{ + "map_fieldname":{ + "new_fieldname":"ws_stack_version" + } + }, + "VersionNote":{ + "map_fieldvalue":{ + "pre_value":"null", + "post_value":"unknown" + }, + "map_fieldname":{ + "new_fieldname":"ws_version_note" + } + }, + "VersionNumber":{ + "map_fieldvalue":{ + "pre_value":"Vnull", + "post_value":"unknown" + }, + "map_fieldname":{ + "new_fieldname":"ws_version_number" + } + }, + "Status":[ + { + "map_fieldcopy":{ + "copy_name": "ws_status" + } + }, + { + "map_fieldvalue":{ + "pre_value":"Success", + "post_value":"1" + } + }, + { + "map_fieldvalue":{ + "pre_value":"Successfully queued", + "post_value":"1" + } + }, + { + "map_fieldvalue":{ + "pre_value":"QUEUED", + "post_value":"1" + } + }, + { + "map_fieldvalue":{ + "pre_value":"PENDING", + "post_value":"1" + } + }, + { + "map_fieldvalue":{ + "pre_value":"COMPLETED", + "post_value":"1" + } + }, + { + "map_fieldvalue":{ + "pre_value":"IN_PROGRESS", + "post_value":"1" + } + }, + { + "map_fieldvalue":{ + "pre_value":"Failed", + "post_value":"0" + } + }, + { + "map_fieldvalue":{ + "pre_value":"Failed to queue", + "post_value":"0" + } + }, + { + "map_fieldvalue":{ + "pre_value":"HOLDING", + "post_value":"0" + } + }, + { + "map_fieldvalue":{ + "pre_value":"HOLDING_FAILED", + "post_value":"0" + } + }, + { + "map_fieldvalue":{ + "pre_value":"HOLDING_TIMEDOUT", + "post_value":"0" + } + }, + { + "map_fieldvalue":{ + "pre_value":"FAILED", + "post_value":"0" + } + }, + { + "map_fieldvalue":{ + "pre_value":"TIMEDOUT", + "post_value":"0" + } + }, + { + "map_fieldvalue":{ + "pre_value":"ABORTED", + "post_value":"0" + } + }, + { + "map_fieldvalue":{ + "pre_value":"SKIPPED_FAILED", + "post_value":"0" + } + }, + { + "map_fieldname":{ + "new_fieldname":"result" + } + } + ], + "ResultStatus":[ + { + "map_fieldcopy":{ + "copy_name": "ws_result_status" + } + }, + { + "map_fieldvalue":{ + "pre_value":"200 OK", + "post_value":"1" + } + }, + { + "map_fieldvalue":{ + "pre_value":"201 Created", + "post_value":"1" + } + }, + { + "map_fieldvalue":{ + "pre_value":"202 Accepted", + "post_value":"1" + } + }, + { + "map_fieldvalue":{ + "pre_value":"400 Bad Request", + "post_value":"0" + } + }, + { + "map_fieldvalue":{ + "pre_value":"401 Unauthorized", + "post_value":"0" + } + }, + { + "map_fieldvalue":{ + "pre_value":"403 Forbidden", + "post_value":"0" + } + }, + { + "map_fieldvalue":{ + "pre_value":"404 Not Found", + "post_value":"0" + } + }, + { + "map_fieldvalue":{ + "pre_value":"409 Resource Conflict", + "post_value":"0" + } + }, + { + "map_fieldvalue":{ + "pre_value":"500 Internal Server Error", + "post_value":"0" + } + }, + { + "map_fieldname":{ + "new_fieldname":"result" + } + } + ] + + } + + } + + ] + +} http://git-wip-us.apache.org/repos/asf/ambari/blob/c295941b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/output.config.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/output.config.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/output.config.json.j2 new file mode 100644 index 0000000..062d636 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/output.config.json.j2 @@ -0,0 +1,61 @@ +{# + # Licensed to the Apache Software Foundation (ASF) under one + # or more contributor license agreements. See the NOTICE file + # distributed with this work for additional information + # regarding copyright ownership. The ASF licenses this file + # to you under the Apache License, Version 2.0 (the + # "License"); you may not use this file except in compliance + # with the License. You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + #} +{ + "output":[ + { + "comment":"Output to solr for service logs", + "is_enabled":"{{solr_service_logs_enable}}", + "destination":"solr", + "zk_connect_string":"{{zookeeper_quorum}}{{infra_solr_znode}}", + "collection":"{{logsearch_solr_collection_service_logs}}", + "number_of_shards": "{{logsearch_collection_service_logs_numshards}}", + "splits_interval_mins": "{{logsearch_service_logs_split_interval_mins}}", + "conditions":{ + "fields":{ + "rowtype":[ + "service" + ] + + } + + } + + }, + { + "comment":"Output to solr for audit records", + "is_enabled":"{{solr_audit_logs_enable}}", + "destination":"solr", + "zk_connect_string":"{{zookeeper_quorum}}{{infra_solr_znode}}", + "collection":"{{logsearch_solr_collection_audit_logs}}", + "number_of_shards": "{{logsearch_collection_audit_logs_numshards}}", + "splits_interval_mins": "{{logsearch_audit_logs_split_interval_mins}}", + "conditions":{ + "fields":{ + "rowtype":[ + "audit" + ] + + } + + } + + } + + ] + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/c295941b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py index 773b75a..02570e2 100644 --- a/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py +++ b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py @@ -62,7 +62,7 @@ class TestLogFeeder(RMFTestCase): ) self.assertResourceCalled('PropertiesFile', '/etc/ambari-logsearch-logfeeder/conf/logfeeder.properties', properties={'logfeeder.checkpoint.folder': '/etc/ambari-logsearch-logfeeder/conf/checkpoints', - 'logfeeder.config.files': 'global.config.json,output.config.json,input.config-ambari.json,input.config-logsearch.json,input.config-zookeeper.json', + 'logfeeder.config.files': 'output.config.json,input.config-ambari.json,global.config.json,input.config-logsearch.json,input.config-zookeeper.json', 'logfeeder.metrics.collector.hosts': '', 'logfeeder.solr.core.config.name': 'history', 'logfeeder.solr.zk_connect_string': 'c6401.ambari.apache.org:2181/infra-solr' @@ -79,10 +79,18 @@ class TestLogFeeder(RMFTestCase): content=InlineTemplate('GP'), encoding='utf-8' ) + self.assertResourceCalled('File', '/etc/ambari-logsearch-logfeeder/conf/input.config-ambari.json', + content=InlineTemplate('ambari-grok-filter'), + encoding='utf-8' + ) + self.assertResourceCalled('File', '/etc/ambari-logsearch-logfeeder/conf/output.config.json', + content=InlineTemplate('output-grok-filter'), + encoding='utf-8' + ) - logfeeder_supported_services = ['ambari','logsearch'] + logfeeder_supported_services = ['logsearch'] - logfeeder_config_file_names = ['global.config.json', 'output.config.json'] + \ + logfeeder_config_file_names = ['global.config.json'] + \ ['input.config-%s.json' % (tag) for tag in logfeeder_supported_services] for file_name in logfeeder_config_file_names: http://git-wip-us.apache.org/repos/asf/ambari/blob/c295941b/ambari-server/src/test/python/stacks/2.4/configs/default.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.4/configs/default.json b/ambari-server/src/test/python/stacks/2.4/configs/default.json index c3eba53..a6e2478 100644 --- a/ambari-server/src/test/python/stacks/2.4/configs/default.json +++ b/ambari-server/src/test/python/stacks/2.4/configs/default.json @@ -313,6 +313,12 @@ "logfeeder_max_mem": "512m", "content": "# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements. See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License. You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nlogsearch.solr.metrics.collector.hosts=http://{{metrics_collector_hosts}}:{{metrics_collector_port}}/ws/v1/timeline/metrics\n{% if logsearch_solr_ssl_enabled %}\nexport LOGFEED ER_SSL=\"true\"\nexport LOGFEEDER_KEYSTORE_LOCATION={{logfeeder_keystore_location}}\nexport LOGFEEDER_KEYSTORE_PASSWORD={{logfeeder_keystore_password}}\nexport LOGFEEDER_KEYSTORE_TYPE={{logfeeder_keystore_type}}\nexport LOGFEEDER_TRUSTSTORE_LOCATION={{logfeeder_truststore_location}}\nexport LOGFEEDER_TRUSTSTORE_PASSWORD={{logfeeder_truststore_password}}\nexport LOGFEEDER_TRUSTSTORE_TYPE={{logfeeder_truststore_type}}\n{% endif %}" }, + "logfeeder-output-config" : { + "content" : "output-grok-filter" + }, + "logfeeder-ambari-config" : { + "content" : "ambari-grok-filter" + }, "logfeeder-grok": { "default_grok_patterns": "GP", "custom_grok_patterns": ""