Github user james-sirota commented on a diff in the pull request: https://github.com/apache/incubator-metron/pull/308#discussion_r83144872 --- Diff: metron-platform/metron-parsers/src/main/config/zookeeper/parsers/websphere.json --- @@ -3,7 +3,7 @@ "sensorTopic":"websphere", "parserConfig": { - "grokPath":"/patterns/websphere", + "grokPattern":"# Days - two digit number is used\nDAY \\d{1,2}\n# Time - two digit hour, minute, and second\nTIME \\d{2}:\\d{2}:\\d{2}\n# Timestamp - month, day, and time\nTIMESTAMP %{MONTH:UNWANTED}\\s+%{DAY:UNWANTED} %{TIME:UNWANTED}\n# Generic word field\nWORD \\w+\n# Priority\nPRIORITY \\d+\n# Log start - the first part of the log line\nLOGSTART <%{PRIORITY:priority}>?%{TIMESTAMP:timestamp_string} %{WORD:hostname}\n# Security domain\nSECURITY_DOMAIN [%{WORD:security_domain}]\n# Log middle - the middle part of the log line\nLOGMIDDLE (\\[%{WORD:security_domain}\\])?\\[%{WORD:event_code}\\]\\[%{WORD:event_type}\\]\\[%{WORD:severity}\\]\n# Define IP address formats\nIPV6 ((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d) ){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:)))(%.+)?\nIPV4 (?<![0-9])(?:(?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9] |[0-1]?[0-9]{1,2}))(?![0-9])\nIP (?:%{IPV6:UNWANTED}|%{IPV4:UNWANTED})\n# Message - the message body of the log\nMESSAGE .*\n# WebSphere - the entire log message\nWEBSPHERE %{LOGSTART:UNWANTED} %{LOGMIDDLE:UNWANTED} %{MESSAGE:message}", --- End diff -- we need to test the heck out of this, guys. i like the change because this makes things much simpler because there is no level of indirection anymore where you have to load the grok statement from a file system. but i see potential impacts on vagrant and management pack because we are now changing the way a setting is being interpreted. I didn't see any mods there and I would have expected that. this is one of those PRs we probably want 2 people to independently verify to make sure it's solid. i think there are two things that still need to be done. first, i think the patterns do have to be removed from HDFS because they are now redundant. second, ansible and message pack installers have to be modified to reflect the fact that the value is now in zookeeper. i volunteer to do the first pass and run it up on AWS once this is done. i also volunteer to clean up the docs and tutorials to account for this change.
--- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. ---