Repository: metron Updated Branches: refs/heads/master de2c871ab -> b76bcd5f2
METRON-996 Performance improvement for ASA parser closes apache/incubator-metron#617 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/b76bcd5f Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/b76bcd5f Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/b76bcd5f Branch: refs/heads/master Commit: b76bcd5f2405db122c2bf9d19128f9efb20b8458 Parents: de2c871 Author: Simon Elliston Ball <[email protected]> Authored: Mon Jun 19 11:30:58 2017 -0400 Committer: cstella <[email protected]> Committed: Mon Jun 19 11:30:58 2017 -0400 ---------------------------------------------------------------------- .../metron/parsers/asa/BasicAsaParser.java | 366 ++++++++++--------- 1 file changed, 191 insertions(+), 175 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/b76bcd5f/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/asa/BasicAsaParser.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/asa/BasicAsaParser.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/asa/BasicAsaParser.java index ad1db83..daac141 100644 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/asa/BasicAsaParser.java +++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/asa/BasicAsaParser.java @@ -33,187 +33,203 @@ import java.io.*; import java.time.Clock; import java.time.ZoneId; import java.util.*; +import java.util.Map.Entry; public class BasicAsaParser extends BasicParser { - protected static final Logger LOG = LoggerFactory.getLogger(BasicAsaParser.class); - - private Grok asaGrok; - protected Clock deviceClock; - - private static final Map<String, String> patternMap = ImmutableMap.<String, String>builder() - .put("ASA-2-106001", "CISCOFW106001") - .put("ASA-2-106006", "CISCOFW106006_106007_106010") - .put("ASA-2-106007", "CISCOFW106006_106007_106010") - .put("ASA-2-106010", "CISCOFW106006_106007_106010") - .put("ASA-3-106014", "CISCOFW106014") - .put("ASA-6-106015", "CISCOFW106015") - .put("ASA-1-106021", "CISCOFW106021") - .put("ASA-4-106023", "CISCOFW106023") - .put("ASA-5-106100", "CISCOFW106100") - .put("ASA-6-110002", "CISCOFW110002") - .put("ASA-6-302010", "CISCOFW302010") - .put("ASA-6-302013", "CISCOFW302013_302014_302015_302016") - .put("ASA-6-302014", "CISCOFW302013_302014_302015_302016") - .put("ASA-6-302015", "CISCOFW302013_302014_302015_302016") - .put("ASA-6-302016", "CISCOFW302013_302014_302015_302016") - .put("ASA-6-302020", "CISCOFW302020_302021") - .put("ASA-6-302021", "CISCOFW302020_302021") - .put("ASA-6-305011", "CISCOFW305011") - .put("ASA-3-313001", "CISCOFW313001_313004_313008") - .put("ASA-3-313004", "CISCOFW313001_313004_313008") - .put("ASA-3-313008", "CISCOFW313001_313004_313008") - .put("ASA-4-313005", "CISCOFW313005") - .put("ASA-4-402117", "CISCOFW402117") - .put("ASA-4-402119", "CISCOFW402119") - .put("ASA-4-419001", "CISCOFW419001") - .put("ASA-4-419002", "CISCOFW419002") - .put("ASA-4-500004", "CISCOFW500004") - .put("ASA-6-602303", "CISCOFW602303_602304") - .put("ASA-6-602304", "CISCOFW602303_602304") - .put("ASA-7-710001", "CISCOFW710001_710002_710003_710005_710006") - .put("ASA-7-710002", "CISCOFW710001_710002_710003_710005_710006") - .put("ASA-7-710003", "CISCOFW710001_710002_710003_710005_710006") - .put("ASA-7-710005", "CISCOFW710001_710002_710003_710005_710006") - .put("ASA-7-710006", "CISCOFW710001_710002_710003_710005_710006") - .put("ASA-6-713172", "CISCOFW713172") - .put("ASA-4-733100", "CISCOFW733100") - .put("ASA-6-305012", "CISCOFW305012") - .put("ASA-7-609001", "CISCOFW609001") - .put("ASA-7-609002", "CISCOFW609002") - .put("ASA-5-713041", "CISCOFW713041") - .build(); - - @Override - public void configure(Map<String, Object> parserConfig) { - String timeZone = (String) parserConfig.get("deviceTimeZone"); - if (timeZone != null) - deviceClock = Clock.system(ZoneId.of(timeZone)); - else { - deviceClock = Clock.systemUTC(); - LOG.warn("[Metron] No device time zone provided; defaulting to UTC"); - } + protected static final Logger LOG = LoggerFactory.getLogger(BasicAsaParser.class); + + protected Clock deviceClock; + private String syslogPattern = "%{CISCO_TAGGED_SYSLOG}"; + + private Grok syslogGrok; + + private static final Map<String, String> patternMap = ImmutableMap.<String, String> builder() + .put("ASA-2-106001", "CISCOFW106001") + .put("ASA-2-106006", "CISCOFW106006_106007_106010") + .put("ASA-2-106007", "CISCOFW106006_106007_106010") + .put("ASA-2-106010", "CISCOFW106006_106007_106010") + .put("ASA-3-106014", "CISCOFW106014") + .put("ASA-6-106015", "CISCOFW106015") + .put("ASA-1-106021", "CISCOFW106021") + .put("ASA-4-106023", "CISCOFW106023") + .put("ASA-5-106100", "CISCOFW106100") + .put("ASA-6-110002", "CISCOFW110002") + .put("ASA-6-302010", "CISCOFW302010") + .put("ASA-6-302013", "CISCOFW302013_302014_302015_302016") + .put("ASA-6-302014", "CISCOFW302013_302014_302015_302016") + .put("ASA-6-302015", "CISCOFW302013_302014_302015_302016") + .put("ASA-6-302016", "CISCOFW302013_302014_302015_302016") + .put("ASA-6-302020", "CISCOFW302020_302021") + .put("ASA-6-302021", "CISCOFW302020_302021") + .put("ASA-6-305011", "CISCOFW305011") + .put("ASA-3-313001", "CISCOFW313001_313004_313008") + .put("ASA-3-313004", "CISCOFW313001_313004_313008") + .put("ASA-3-313008", "CISCOFW313001_313004_313008") + .put("ASA-4-313005", "CISCOFW313005") + .put("ASA-4-402117", "CISCOFW402117") + .put("ASA-4-402119", "CISCOFW402119") + .put("ASA-4-419001", "CISCOFW419001") + .put("ASA-4-419002", "CISCOFW419002") + .put("ASA-4-500004", "CISCOFW500004") + .put("ASA-6-602303", "CISCOFW602303_602304") + .put("ASA-6-602304", "CISCOFW602303_602304") + .put("ASA-7-710001", "CISCOFW710001_710002_710003_710005_710006") + .put("ASA-7-710002", "CISCOFW710001_710002_710003_710005_710006") + .put("ASA-7-710003", "CISCOFW710001_710002_710003_710005_710006") + .put("ASA-7-710005", "CISCOFW710001_710002_710003_710005_710006") + .put("ASA-7-710006", "CISCOFW710001_710002_710003_710005_710006") + .put("ASA-6-713172", "CISCOFW713172") + .put("ASA-4-733100", "CISCOFW733100") + .put("ASA-6-305012", "CISCOFW305012") + .put("ASA-7-609001", "CISCOFW609001") + .put("ASA-7-609002", "CISCOFW609002") + .put("ASA-5-713041", "CISCOFW713041") + .build(); + + private Map<String, Grok> grokers = new HashMap<String, Grok>(patternMap.size()); + + @Override + public void configure(Map<String, Object> parserConfig) { + String timeZone = (String) parserConfig.get("deviceTimeZone"); + if (timeZone != null) + deviceClock = Clock.system(ZoneId.of(timeZone)); + else { + deviceClock = Clock.systemUTC(); + LOG.warn("[Metron] No device time zone provided; defaulting to UTC"); + } + } + + private void addGrok(String key, String pattern) throws GrokException { + Grok grok = new Grok(); + InputStream patternStream = this.getClass().getResourceAsStream("/patterns/asa"); + grok.addPatternFromReader(new InputStreamReader(patternStream)); + grok.compile("%{" + pattern + "}"); + grokers.put(key, grok); + } + + @Override + public void init() { + syslogGrok = new Grok(); + InputStream syslogStream = this.getClass().getResourceAsStream("/patterns/asa"); + try { + syslogGrok.addPatternFromReader(new InputStreamReader(syslogStream)); + syslogGrok.compile(syslogPattern); + } catch (GrokException e) { + LOG.error("[Metron] Failed to load grok patterns from jar", e); + throw new RuntimeException(e.getMessage(), e); + } + + for (Entry<String, String> pattern : patternMap.entrySet()) { + try { + addGrok(pattern.getKey(), pattern.getValue()); + } catch (GrokException e) { + LOG.error("[Metron] Failed to load grok pattern %s for ASA tag %s", pattern.getValue(), pattern.getKey()); + } } - @Override - public void init() { - asaGrok = new Grok(); - InputStream patternStream = this.getClass().getResourceAsStream("/patterns/asa"); - try { - asaGrok.addPatternFromReader(new InputStreamReader(patternStream)); - } catch (GrokException e) { - LOG.error("[Metron] Failed to load grok patterns from jar", e); - throw new RuntimeException(e.getMessage(), e); - } - LOG.info("[Metron] CISCO ASA Parser Initialized"); + LOG.info("[Metron] CISCO ASA Parser Initialized"); + } + + @Override + public List<JSONObject> parse(byte[] rawMessage) { + String logLine = ""; + String messagePattern = ""; + JSONObject metronJson = new JSONObject(); + List<JSONObject> messages = new ArrayList<>(); + Map<String, Object> syslogJson = new HashMap<String, Object>(); + + try { + logLine = new String(rawMessage, "UTF-8"); + } catch (UnsupportedEncodingException e) { + LOG.error("[Metron] Could not read raw message", e); + throw new RuntimeException(e.getMessage(), e); } - @Override - public List<JSONObject> parse(byte[] rawMessage) { - String logLine = ""; - String syslogPattern = "%{CISCO_TAGGED_SYSLOG}"; - String messagePattern = ""; - JSONObject metronJson = new JSONObject(); - List<JSONObject> messages = new ArrayList<>(); - Map<String, Object> syslogJson = new HashMap<String, Object>(); - - try { - logLine = new String(rawMessage, "UTF-8"); - } catch (UnsupportedEncodingException e) { - LOG.error("[Metron] Could not read raw message", e); - throw new RuntimeException(e.getMessage(), e); - } - - try { - LOG.debug("[Metron] Started parsing raw message: {}", logLine); - - asaGrok.compile(syslogPattern); - Match syslogMatch = asaGrok.match(logLine); - syslogMatch.captures(); - if(!syslogMatch.isNull()) { - syslogJson = syslogMatch.toMap(); - LOG.trace("[Metron] Grok CISCO ASA syslog matches: {}", syslogMatch.toJson()); - - metronJson.put(Constants.Fields.ORIGINAL.getName(), logLine); - metronJson.put(Constants.Fields.TIMESTAMP.getName(), - SyslogUtils.parseTimestampToEpochMillis((String) syslogJson.get("CISCOTIMESTAMP"), deviceClock)); - metronJson.put("ciscotag", syslogJson.get("CISCOTAG")); - metronJson.put("syslog_severity", SyslogUtils.getSeverityFromPriority((int) syslogJson.get("syslog_pri"))); - metronJson.put("syslog_facility", SyslogUtils.getFacilityFromPriority((int) syslogJson.get("syslog_pri"))); - - - if (syslogJson.get("syslog_host")!=null) { - metronJson.put("syslog_host", syslogJson.get("syslog_host")); - } - if (syslogJson.get("syslog_prog")!=null) { - metronJson.put("syslog_prog", syslogJson.get("syslog_prog")); - } - - } - else - throw new RuntimeException(String.format("[Metron] Message '%s' does not match pattern '%s'", logLine, syslogPattern)); - } catch (GrokException e) { - LOG.error(String.format("[Metron] Could not compile grok pattern '%s'", syslogPattern), e); - throw new RuntimeException(e.getMessage(), e); - } catch (ParseException e) { - LOG.error("[Metron] Could not parse message timestamp", e); - throw new RuntimeException(e.getMessage(), e); - } catch (RuntimeException e) { - LOG.error(e.getMessage(), e); - throw new RuntimeException(e.getMessage(), e); - } - - try { - messagePattern = patternMap.get(syslogJson.get("CISCOTAG")); - if (messagePattern == null) - LOG.info("[Metron] No pattern for ciscotag '{}'", syslogJson.get("CISCOTAG")); - else { - asaGrok.compile("%{" + messagePattern + "}"); - Match messageMatch = asaGrok.match((String) syslogJson.get("message")); - messageMatch.captures(); - if (!messageMatch.isNull()) { - Map<String, Object> messageJson = messageMatch.toMap(); - LOG.trace("[Metron] Grok CISCO ASA message matches: {}", messageMatch.toJson()); - - String src_ip = (String) messageJson.get("src_ip"); - if (src_ip != null) - metronJson.put(Constants.Fields.SRC_ADDR.getName(), src_ip); - - Integer src_port = (Integer) messageJson.get("src_port"); - if (src_port != null) - metronJson.put(Constants.Fields.SRC_PORT.getName(), src_port); - - String dst_ip = (String) messageJson.get("dst_ip"); - if (dst_ip != null) - metronJson.put(Constants.Fields.DST_ADDR.getName(), dst_ip); - - Integer dst_port = (Integer) messageJson.get("dst_port"); - if (dst_port != null) - metronJson.put(Constants.Fields.DST_PORT.getName(), dst_port); - - String protocol = (String) messageJson.get("protocol"); - if (protocol != null) - metronJson.put(Constants.Fields.PROTOCOL.getName(), protocol.toLowerCase()); - - String action = (String) messageJson.get("action"); - if (action != null) - metronJson.put("action", action.toLowerCase()); - } - else - LOG.warn("[Metron] Message '{}' did not match pattern for ciscotag '{}'", logLine, syslogJson.get("CISCOTAG")); - } - - LOG.debug("[Metron] Final normalized message: {}", metronJson.toString()); - - } catch (GrokException e) { - LOG.error(String.format("[Metron] Could not compile grok pattern '%s'", messagePattern), e); - throw new RuntimeException(e.getMessage(), e); - } catch (RuntimeException e) { - LOG.error(e.getMessage(), e); - throw new RuntimeException(e.getMessage(), e); - } - - messages.add(metronJson); - return messages; + try { + LOG.debug("[Metron] Started parsing raw message: {}", logLine); + Match syslogMatch = syslogGrok.match(logLine); + syslogMatch.captures(); + if (!syslogMatch.isNull()) { + syslogJson = syslogMatch.toMap(); + LOG.trace("[Metron] Grok CISCO ASA syslog matches: {}", syslogMatch.toJson()); + + metronJson.put(Constants.Fields.ORIGINAL.getName(), logLine); + metronJson.put(Constants.Fields.TIMESTAMP.getName(), + SyslogUtils.parseTimestampToEpochMillis((String) syslogJson.get("CISCOTIMESTAMP"), deviceClock)); + metronJson.put("ciscotag", syslogJson.get("CISCOTAG")); + metronJson.put("syslog_severity", SyslogUtils.getSeverityFromPriority((int) syslogJson.get("syslog_pri"))); + metronJson.put("syslog_facility", SyslogUtils.getFacilityFromPriority((int) syslogJson.get("syslog_pri"))); + + if (syslogJson.get("syslog_host") != null) { + metronJson.put("syslog_host", syslogJson.get("syslog_host")); + } + if (syslogJson.get("syslog_prog") != null) { + metronJson.put("syslog_prog", syslogJson.get("syslog_prog")); + } + + } else + throw new RuntimeException( + String.format("[Metron] Message '%s' does not match pattern '%s'", logLine, syslogPattern)); + } catch (ParseException e) { + LOG.error("[Metron] Could not parse message timestamp", e); + throw new RuntimeException(e.getMessage(), e); + } catch (RuntimeException e) { + LOG.error(e.getMessage(), e); + throw new RuntimeException(e.getMessage(), e); } + + try { + messagePattern = (String) syslogJson.get("CISCOTAG"); + Grok asaGrok = grokers.get(messagePattern); + + if (asaGrok == null) + LOG.info("[Metron] No pattern for ciscotag '{}'", syslogJson.get("CISCOTAG")); + else { + + String messageContent = (String) syslogJson.get("message"); + Match messageMatch = asaGrok.match(messageContent); + messageMatch.captures(); + if (!messageMatch.isNull()) { + Map<String, Object> messageJson = messageMatch.toMap(); + LOG.trace("[Metron] Grok CISCO ASA message matches: {}", messageMatch.toJson()); + + String src_ip = (String) messageJson.get("src_ip"); + if (src_ip != null) + metronJson.put(Constants.Fields.SRC_ADDR.getName(), src_ip); + + Integer src_port = (Integer) messageJson.get("src_port"); + if (src_port != null) + metronJson.put(Constants.Fields.SRC_PORT.getName(), src_port); + + String dst_ip = (String) messageJson.get("dst_ip"); + if (dst_ip != null) + metronJson.put(Constants.Fields.DST_ADDR.getName(), dst_ip); + + Integer dst_port = (Integer) messageJson.get("dst_port"); + if (dst_port != null) + metronJson.put(Constants.Fields.DST_PORT.getName(), dst_port); + + String protocol = (String) messageJson.get("protocol"); + if (protocol != null) + metronJson.put(Constants.Fields.PROTOCOL.getName(), protocol.toLowerCase()); + + String action = (String) messageJson.get("action"); + if (action != null) + metronJson.put("action", action.toLowerCase()); + } else + LOG.warn("[Metron] Message '{}' did not match pattern for ciscotag '{}'", logLine, + syslogJson.get("CISCOTAG")); + } + + LOG.debug("[Metron] Final normalized message: {}", metronJson.toString()); + + } catch (RuntimeException e) { + LOG.error(e.getMessage(), e); + throw new RuntimeException(e.getMessage(), e); + } + + messages.add(metronJson); + return messages; + } }
