2015-09-24 14:21 GMT+02:00 Andre <andre-li...@fucs.org>: [...] > However the real advantage of grok is its user base and is reasonably > broad support by tools (.e.g. LogStash, FluentD, Graylog2, and > others)... > > This mean you gain an easy source of pre-defined parsing rules without > having to rewrite them as a sandbox LPEG, e.g: > > https://github.com/whyscream/postfix-grok-patterns
This one has already been ported (by me), see : https://gist.github.com/mozilla-services/heka/pull/1649 > https://github.com/elastic/logstash/tree/v1.4.2/patterns > http://www.gregmefford.com/blog/2014/09/24/analyzing-cisco-asa-firewall-logs-with-logstash/ This is now at <https://github.com/logstash-plugins/logstash-patterns-core>. Those are probably easy to port, I have a local lpeg decoder for vendor syslog, and will propose a PR with it once it is enough tested (it doesn't parse Cisco ASA yet, but it parses Cisco IOS and Nexus and various Netapp). See it attached for now. > http://seclists.org/snort/2013/q4/174 Very easy to do also. PR welcomes! Regards -- Mathieu
-- This Source Code Form is subject to the terms of the Mozilla Public -- License, v. 2.0. If a copy of the MPL was not distributed with this -- file, You can obtain one at http://mozilla.org/MPL/2.0/. -- Copyright 2015 Mathieu Parent <math.par...@gmail.com> --[[ Parses the syslog logs from vendor not respecting Syslog RFCs. Config: - hostname_keep (boolean, defaults to false) Always preserve the original 'Hostname' field set by Logstreamer's 'hostname' configuration setting. - cisco_ios (boolean, defaults to false) Try to parse as a Cisco IOS Payload. - cisco_nexus (boolean, defaults to false) Try to parse as a Nexus Payload. - cisco_wlc (boolean, defaults to false) Try to parse as a Wireless LAN Controller Payload. - netapp_cdot (boolean, defaults to false) Try to parse as a NetApp Clustered Data ONTAP Payload. - netapp_balance (boolean, defaults to false) Try to parse as a NetApp OnCommand Balance Payload. *Example Heka Configuration* .. code-block:: ini [FlexibleSyslogDecoder] type = "SandboxDecoder" filename = "lua_decoders/flexible_syslog.lua" [FlexibleSyslogDecoder.config] cisco_ios = true cisco_nexus = true cisco_wlc = true netapp_cdot = true netapp_balance = true *Example Heka Message* :Timestamp: 2014-01-10 07:04:56 -0800 PST :Type: netapp_cdot :Hostname: netapp1 :Pid: 0 :UUID: 8e414f01-9d7f-4a48-a5e1-ae92e5954df5 :Logger: SyslogInput :Payload: Volume test@vserver:42424242-4242-4242-4242-424242424242 destroyed. :EnvVersion: :Severity: 6 :Fields: | name:"syslogfacility" value:0 | name:"cdot_message" value:"wafl.vvol.destroyed" | name:"cdot_severity" value:"info" --]] local l = require 'lpeg' l.locale(l) local dt = require 'date_time' local math = require 'math' local os = require 'os' local hostname_keep = read_config('hostname_keep') local grammar_enable = {} grammar_enable.cisco_ios = read_config('cisco_ios') grammar_enable.cisco_nexus = read_config('cisco_nexus') grammar_enable.cisco_wlc = read_config('cisco_wlc') grammar_enable.netapp_cdot = read_config('netapp_cdot') grammar_enable.netapp_balance = read_config('netapp_balance') -- Build grammars local vendors = {} -- FIXME DUP of syslog local function convert_pri(pri) pri = tonumber(pri) local facility = math.floor(pri/8) local severity = pri % 8 return {facility = facility, severity = severity} end local pri = l.P'<' * l.Cg(l.digit^-3 / convert_pri, 'pri') * l.P'>' ------------------------------------------------------------------- -- Cisco IOS -- <157>6628: Aug 21 14:51:56: %SYS-5-CONFIG_I: Configured from console by admin onvty0 (10.20.30.42) -- <157>6629: SP: Aug 21 14:52:06: %PFINIT-5-CONFIG_SYNC: Sync'ing the startup configuration to the standby Router. -- <189>2105: *May 1 23:29:33: %LINEPROTO-5-UPDOWN: Line protocol on Interface FastEthernet0/7, changed state to up -- Ref http://www.cisco.com/c/en/us/products/collateral/services/high-availability/white_paper_c11-557812.html local ios_date = l.Cg(l.Ct( l.Cg(l.Cc('') / function() return os.date('%Y') end, 'year') * dt.date_mabbr * l.P' ' * dt.date_mday_sp * l.P' ' * dt.time_hour * l.P':' * dt.time_minute * l.P':' * dt.time_second * (l.P'.' * l.Cg(l.digit^1 / function(n) return tonumber(n)/1000 end, 'sec_frac'))^-1 ) / dt.time_to_ns, 'timestamp') vendors.cisco_ios = l.Ct( pri * l.digit^1 / tonumber -- message number since boot * l.P': ' * (l.P'SP' * l.P': ')^-1 -- supervisor * ( (l.P'*' * l.Cg(l.Cc('false'), 'ntp_authoritative') * l.Cg(l.Cc('false'), 'ntp_synchronized')) + (l.P'.' * l.Cg(l.Cc('false'), 'ntp_synchronized')) + l.P('') ) * ios_date * l.P': %' * l.Cg((l.P(1)-l.P': ')^1, 'ios_tag') * l.P': ' * l.Cg(l.P(1)^1, 'msg') ) ------------------------------------------------------------------- -- Cisco Nexus -- <PRI>2015 Aug 7 11:03:31 CEST: %VSHD-5-VSHD_SYSLOG_CONFIG_I: Configured from vty by admin on 10.20.30.40@pts/1 local nexus_date = l.Cg(l.Ct( dt.date_fullyear * l.P' ' * dt.date_mabbr * l.P' ' * l.Cg(l.digit^-2, 'day') * l.P' ' * dt.time_hour * l.P':' * dt.time_minute * l.P':' * dt.time_second * l.P' ' * dt.timezone ) / dt.time_to_ns, 'timestamp') vendors.cisco_nexus = l.Ct( pri * l.P': ' * nexus_date * l.P': %' * l.Cg((l.P(1)-l.P':')^1, 'nexus_tag') * l.P': ' * l.Cg(l.P(1)^1, 'msg') ) ------------------------------------------------------------------- -- Cisco Wireless LAN Controller -- <188>myhost: *apfMsConnTask_2: Sep 18 15:10:07.119: #APF-4-ASSOCREQ_PROC_FAILED: [PA] apf_80211.c:5233 Failed to process an association request from 74:de:2b:08:87:0b. WLAN:7, SSID:WAN@NM. message received from disabled WLAN. -- <189>myhost: -Traceback: 0x10e451e4 0x10135ffc 0x100f738c 0x100ff22c 0x10f323d0 0x10e53f08 0x12644620 0x126bc02c local wlc_date = l.Cg(l.Ct( l.Cg(l.Cc('') / function() return os.date('%Y') end, 'year') * dt.date_mabbr * l.P' ' * l.Cg(l.digit^-2, 'day') * l.P' ' * dt.time_hour * l.P':' * dt.time_minute * l.P':' * dt.time_second * l.P'.' * l.Cg(l.digit^1 / function(n) return tonumber(n)/1000 end, 'sec_frac') ) / dt.time_to_ns, 'timestamp') vendors.cisco_wlc = l.Ct( pri * l.Cg((l.P(1)-l.P': ')^1, 'hostname') * l.P': ' * (( l.P'*' * l.Cg((l.P(1)-l.P': ')^1, 'wlc_tag1') * l.P': ' * wlc_date * l.P': #' * l.Cg((l.P(1)-l.P':')^1, 'wlc_tag2') * l.P': [' * l.Cg((l.P(1)-l.P']')^1, 'wlc_tag3') * l.P'] ' * l.Cg((l.P(1)-l.P':')^1, 'wlc_file') * l.P':' * l.Cg(l.digit^1 / tonumber, 'wlc_line') * l.P' ' * l.Cg(l.P(1)^1, 'msg') ) + ( l.P'-Traceback: ' * l.Cg(l.Cc(true), '_drop') )) ) ------------------------------------------------------------------- -- NetApp cDOT -- <PRI>Aug 7 12:07:28 [cdot-1:wafl.vvol.destroyed:info]: Volume test@vserver:42424242-4242-4242-4242-424242424242 destroyed. local cdot_date = l.Cg(l.Ct( l.Cg(l.Cc('') / function() return os.date('%Y') end, 'year') * dt.date_mabbr * l.P' ' * dt.date_mday_sp * l.P' ' * dt.time_hour * l.P':' * dt.time_minute * l.P':' * dt.time_second ) / dt.time_to_ns, 'timestamp') vendors.netapp_cdot = l.Ct( pri * cdot_date * l.P' [' * l.Cg((l.P(1)-l.P':')^1, 'hostname') * l.P':' * l.Cg((l.P(1)-l.P':')^1, 'cdot_message') * l.P':' * l.Cg((l.P(1)-l.P']: ')^1, 'cdot_severity') * l.P']:' * (l.P' ' * l.Cg(l.P(1)^1, 'msg'))^-1 ) ------------------------------------------------------------------- -- NetApp OnCommand Balance -- <PRI>user:2015-08-07 11:14:11,032 WARN [sys.netapp] Syslog Test Message\0x0a local balance_date =l.Cg(l.Ct( dt.date_fullyear * l.P'-' * dt.date_month * l.P'-' * dt.date_mday * l.P' ' * dt.time_hour * l.P':' * dt.time_minute * l.P':' * dt.time_second * l.P',' * l.Cg(l.digit^1 / function(n) return tonumber(n)/1000 end, 'sec_frac') ) / dt.time_to_ns, 'timestamp') vendors.netapp_balance = l.Ct( pri * l.Cg((l.P(1)-l.P':')^1, 'balance_source') * l.P':' * balance_date * l.P' ' * l.Cg((l.P(1)-l.P' [')^1, 'balance_severity') * l.P' [' * l.Cg((l.P(1)-l.P'] ')^1, 'balance_category') * l.P'] ' * l.Cg((l.P(1) - l.P'\n')^1, 'msg') * l.P'\n'^-1 * l.P(-1) ) ------------------------------------------------------------------- local grammars = {} for k, v in pairs(grammar_enable) do if v then if not vendors[k] then error('Unknow grammar: ' .. k) else grammars[k] = vendors[k] end end end local msg = { Timestamp = nil, Type = nil, Hostname = nil, Payload = nil, Pid = nil, Severity = nil, Fields = nil } function process_message () local log = read_message('Payload') local fields = nil for k, grammar in pairs(grammars) do fields = grammar:match(log) if fields then -- FIXME Allow the sandbox decoder to drop messages https://github.com/mozilla-services/heka/issues/1398 -- if fields._drop then -- return -2 -- end msg.Timestamp = fields.timestamp fields.timestamp = nil msg.Type = k if fields.hostname and not hostname_keep then msg.Hostname = fields.hostname fields.hostname = nil else msg.Hostname = read_message('Hostname') end msg.Payload = fields.msg fields.msg = nil if fields.pri then msg.Severity = fields.pri.severity fields.syslogfacility = fields.pri.facility fields.pri = nil else msg.Severity = nil end if fields.ntp_synchronized == false then fields.timereported = msg.Timestamp msg.Timestamp = nil end msg.Fields = fields if not pcall(inject_message, msg) then return -1, 'Unable to inject message' end return 0 end end -- no match return -1, 'No match' end
_______________________________________________ Heka mailing list Heka@mozilla.org https://mail.mozilla.org/listinfo/heka
