Repository: metron Updated Branches: refs/heads/master 42068d951 -> 61b1ede0b
METRON-1740 Improve Palo Alto parser to handle CONFIG and SYSTEM syslog messages (liuy-tnz via nickwallen) closes apache/metron#1171 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/61b1ede0 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/61b1ede0 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/61b1ede0 Branch: refs/heads/master Commit: 61b1ede0bdfd28b04a3783371deb935d293df998 Parents: 42068d9 Author: liuy-tnz <[email protected]> Authored: Tue Nov 13 07:00:51 2018 -0500 Committer: nickallen <[email protected]> Committed: Tue Nov 13 07:00:51 2018 -0500 ---------------------------------------------------------------------- .../paloalto/BasicPaloAltoFirewallParser.java | 413 ++++++++++++------- .../apache/metron/parsers/paloalto/README.md | 32 ++ .../BasicPaloAltoFirewallParserTest.java | 244 +++++++++++ 3 files changed, 541 insertions(+), 148 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/61b1ede0/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParser.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParser.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParser.java index 9051f09..c8e8b62 100644 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParser.java +++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParser.java @@ -34,21 +34,26 @@ import java.util.regex.Pattern; public class BasicPaloAltoFirewallParser extends BasicParser { - private static boolean empty_attribute( final String s ) { + private static boolean empty_attribute(final String s) { return s == null || s.trim().isEmpty() || s.equals("\"\""); } - private static String unquoted_attribute( String s ) { + private static String unquoted_attribute(String s) { s = s.trim(); - if ( s.startsWith( "\"" ) && s.endsWith( "\"" ) ) - return s.substring( 1, s.length( ) - 1 ); + if (s.startsWith("\"") && s.endsWith("\"")) + return s.substring(1, s.length() - 1); return s; } - private static final Logger _LOG = LoggerFactory.getLogger - (BasicPaloAltoFirewallParser.class); + private static final Logger _LOG = LoggerFactory.getLogger(BasicPaloAltoFirewallParser.class); private static final long serialVersionUID = 3147090149725343999L; + + private static final String LogTypeConfig = "CONFIG"; + private static final String LogTypeSystem = "SYSTEM"; + private static final String LogTypeThreat = "THREAT"; + private static final String LogTypeTraffic = "TRAFFIC"; + public static final String PaloAltoDomain = "palo_alto_domain"; public static final String ReceiveTime = "receive_time"; public static final String SerialNum = "serial"; @@ -101,6 +106,21 @@ public class BasicPaloAltoFirewallParser extends BasicParser { public static final String ParentSessionStartTime = "parent_session_start_time"; public static final String TunnelType = "tunnel_type"; + //System + public static final String EventId = "event_id"; + public static final String Object = "object"; + public static final String Module = "module"; + public static final String Description = "description"; + + //Config + public static final String Command = "command"; + public static final String Admin = "admin"; + public static final String Client = "client"; + public static final String Result = "result"; + public static final String ConfigurationPath = "configuration_path"; + public static final String BeforeChangeDetail = "before_change_detail"; + public static final String AfterChangeDetail = "after_change_detail"; + //Threat public static final String URL = "url"; public static final String HOST = "host"; @@ -113,7 +133,7 @@ public class BasicPaloAltoFirewallParser extends BasicParser { public static final String PCAPID = "pcap_id"; public static final String WFFileDigest = "filedigest"; public static final String WFCloud = "cloud"; - public static final String UserAgent= "user_agent"; + public static final String UserAgent = "user_agent"; public static final String WFFileType = "filetype"; public static final String XForwardedFor = "xff"; public static final String Referer = "referer"; @@ -159,8 +179,6 @@ public class BasicPaloAltoFirewallParser extends BasicParser { toParse = new String(msg, "UTF-8"); _LOG.debug("Received message: {}", toParse); - - parseMessage(toParse, outputMessage); long timestamp = System.currentTimeMillis(); outputMessage.put("timestamp", System.currentTimeMillis()); @@ -178,159 +196,258 @@ public class BasicPaloAltoFirewallParser extends BasicParser { private void parseMessage(String message, JSONObject outputMessage) { String[] tokens = Iterables.toArray(Splitter.on(Pattern.compile(",(?=(?:[^\"]*\"[^\"]*\")*[^\"]*$)")).split(message), String.class); + int parser_version = 0; String type = tokens[3].trim(); + //validate log types + if (!type.equals(LogTypeConfig) && + !type.equals(LogTypeThreat) && + !type.equals(LogTypeTraffic) && + !type.equals(LogTypeSystem)) { + throw new UnsupportedOperationException("Unsupported log type."); + } + //populate common objects - if( !empty_attribute( tokens[0] ) ) outputMessage.put(PaloAltoDomain, tokens[0].trim()); - if( !empty_attribute( tokens[1] ) ) outputMessage.put(ReceiveTime, tokens[1].trim()); - if( !empty_attribute( tokens[2] ) ) outputMessage.put(SerialNum, tokens[2].trim()); + if (!empty_attribute(tokens[0])) outputMessage.put(PaloAltoDomain, tokens[0].trim()); + if (!empty_attribute(tokens[1])) outputMessage.put(ReceiveTime, tokens[1].trim()); + if (!empty_attribute(tokens[2])) outputMessage.put(SerialNum, tokens[2].trim()); outputMessage.put(Type, type); - if( !empty_attribute( tokens[4] ) ) outputMessage.put(ThreatContentType, unquoted_attribute(tokens[4])); - if( !empty_attribute( tokens[5] ) ) outputMessage.put(ConfigVersion, tokens[5].trim()); - if( !empty_attribute( tokens[6] ) ) outputMessage.put(GenerateTime, tokens[6].trim()); - if( !empty_attribute( tokens[7] ) ) outputMessage.put(SourceAddress, tokens[7].trim()); - if( !empty_attribute( tokens[8] ) ) outputMessage.put(DestinationAddress, tokens[8].trim()); - if( !empty_attribute( tokens[9] ) ) outputMessage.put(NATSourceIP, tokens[9].trim()); - if( !empty_attribute( tokens[10] ) ) outputMessage.put(NATDestinationIP, tokens[10].trim()); - if( !empty_attribute( tokens[11] ) ) outputMessage.put(Rule, unquoted_attribute(tokens[11])); - if( !empty_attribute( tokens[12] ) ) outputMessage.put(SourceUser, unquoted_attribute(tokens[12])); - if( !empty_attribute( tokens[13] ) ) outputMessage.put(DestinationUser, unquoted_attribute(tokens[13])); - if( !empty_attribute( tokens[14] ) ) outputMessage.put(Application, unquoted_attribute(tokens[14])); - if( !empty_attribute( tokens[15] ) ) outputMessage.put(VirtualSystem, unquoted_attribute(tokens[15])); - if( !empty_attribute( tokens[16] ) ) outputMessage.put(SourceZone, unquoted_attribute(tokens[16])); - if( !empty_attribute( tokens[17] ) ) outputMessage.put(DestinationZone, unquoted_attribute(tokens[17])); - if( !empty_attribute( tokens[18] ) ) outputMessage.put(InboundInterface, unquoted_attribute(tokens[18])); - if( !empty_attribute( tokens[19] ) ) outputMessage.put(OutboundInterface, unquoted_attribute(tokens[19])); - if( !empty_attribute( tokens[20] ) ) outputMessage.put(LogAction, unquoted_attribute(tokens[20])); - if( !empty_attribute( tokens[21] ) ) outputMessage.put(TimeLogged, tokens[21].trim()); - if( !empty_attribute( tokens[22] ) ) outputMessage.put(SessionID, tokens[22].trim()); - if( !empty_attribute( tokens[23] ) ) outputMessage.put(RepeatCount, tokens[23].trim()); - if( !empty_attribute( tokens[24] ) ) outputMessage.put(SourcePort, tokens[24].trim()); - if( !empty_attribute( tokens[25] ) ) outputMessage.put(DestinationPort, tokens[25].trim()); - if( !empty_attribute( tokens[26] ) ) outputMessage.put(NATSourcePort, tokens[26].trim()); - if( !empty_attribute( tokens[27] ) ) outputMessage.put(NATDestinationPort, tokens[27].trim()); - if( !empty_attribute( tokens[28] ) ) outputMessage.put(Flags, tokens[28].trim()); - if( !empty_attribute( tokens[29] ) ) outputMessage.put(IPProtocol, unquoted_attribute(tokens[29])); - if( !empty_attribute( tokens[30] ) ) outputMessage.put(Action, unquoted_attribute(tokens[30])); - - - if ("THREAT".equals(type.toUpperCase())) { - int p1_offset = 0; - if (tokens.length == 45) parser_version = 60; - else if (tokens.length == 53) parser_version = 61; - else if (tokens.length == 61) { - parser_version = 70; - p1_offset = 1; - } - else if (tokens.length == 72) { - parser_version = 80; - p1_offset =1; + if (!empty_attribute(tokens[4])) outputMessage.put(ThreatContentType, unquoted_attribute(tokens[4])); + if (!empty_attribute(tokens[5])) outputMessage.put(ConfigVersion, tokens[5].trim()); + if (!empty_attribute(tokens[6])) outputMessage.put(GenerateTime, tokens[6].trim()); + + if (LogTypeConfig.equals(type.toUpperCase())) { + // There are two fields in custom logs only and they are not in the default format. + // But we need to parse them if they exist + if (tokens.length == 16 || tokens.length == 18) parser_version = 61; + else if (tokens.length == 22 || tokens.length == 24) parser_version = 80; + + if (parser_version >= 61) { + if (!empty_attribute(tokens[7])) outputMessage.put(HOST, tokens[7].trim()); + if (!empty_attribute(tokens[8])) outputMessage.put(VirtualSystem, tokens[8].trim()); + if (!empty_attribute(tokens[9])) outputMessage.put(Command, tokens[9].trim()); + if (!empty_attribute(tokens[10])) outputMessage.put(Admin, tokens[10].trim()); + if (!empty_attribute(tokens[11])) outputMessage.put(Client, unquoted_attribute(tokens[11])); + if (!empty_attribute(tokens[12])) outputMessage.put(Result, unquoted_attribute(tokens[12])); + if (!empty_attribute(tokens[13])) outputMessage.put(ConfigurationPath, unquoted_attribute(tokens[13])); } - outputMessage.put(ParserVersion, parser_version); - if( !empty_attribute( tokens[31] ) ) { - outputMessage.put(URL, unquoted_attribute(tokens[31])); - try { - URL url = new URL(unquoted_attribute(tokens[31])); - outputMessage.put(HOST, url.getHost()); - } catch (MalformedURLException e) { + + if (parser_version == 61) { + if (!empty_attribute(tokens[14])) outputMessage.put(Seqno, unquoted_attribute(tokens[14])); + if (!empty_attribute(tokens[15])) outputMessage.put(ActionFlags, unquoted_attribute(tokens[15])); + if (tokens.length == 18) { + if (!empty_attribute(tokens[16])) + outputMessage.put(BeforeChangeDetail, unquoted_attribute(tokens[16])); + if (!empty_attribute(tokens[17])) + outputMessage.put(AfterChangeDetail, unquoted_attribute(tokens[17])); } } - if( !empty_attribute( tokens[32] ) ) outputMessage.put(ThreatID, tokens[32].trim()); - if( !empty_attribute( tokens[33] ) ) outputMessage.put(Category, unquoted_attribute(tokens[33])); - if( !empty_attribute( tokens[34] ) ) outputMessage.put(Severity, unquoted_attribute(tokens[34])); - if( !empty_attribute( tokens[35] ) ) outputMessage.put(Direction, unquoted_attribute(tokens[35])); - if( !empty_attribute( tokens[36] ) ) outputMessage.put(Seqno, tokens[36].trim()); - if( !empty_attribute( tokens[37] ) ) outputMessage.put(ActionFlags, unquoted_attribute(tokens[37])); - if( !empty_attribute( tokens[38] ) ) outputMessage.put(SourceLocation, unquoted_attribute(tokens[38])); - if( !empty_attribute( tokens[39] ) ) outputMessage.put(DestinationLocation, unquoted_attribute(tokens[39])); - if( !empty_attribute( tokens[41] ) ) outputMessage.put(ContentType, unquoted_attribute(tokens[41])); - if( !empty_attribute( tokens[42] ) ) outputMessage.put(PCAPID, tokens[42].trim()); - if( !empty_attribute( tokens[43] ) ) outputMessage.put(WFFileDigest, unquoted_attribute(tokens[43])); - if( !empty_attribute( tokens[44] ) ) outputMessage.put(WFCloud, unquoted_attribute(tokens[44])); - if ( parser_version >= 61) { - if( !empty_attribute( tokens[(45 + p1_offset)] ) ) outputMessage.put(UserAgent, unquoted_attribute(tokens[(45 + p1_offset)])); - if( !empty_attribute( tokens[(46 + p1_offset)] ) ) outputMessage.put(WFFileType, unquoted_attribute(tokens[(46 + p1_offset)])); - if( !empty_attribute( tokens[(47 + p1_offset)] ) ) outputMessage.put(XForwardedFor, unquoted_attribute(tokens[(47 + p1_offset)])); - if( !empty_attribute( tokens[(48 + p1_offset)] ) ) outputMessage.put(Referer, unquoted_attribute(tokens[(48 + p1_offset)])); - if( !empty_attribute( tokens[(49 + p1_offset)] ) ) outputMessage.put(WFSender, unquoted_attribute(tokens[(49 + p1_offset)])); - if( !empty_attribute( tokens[(50 + p1_offset)] ) ) outputMessage.put(WFSubject, unquoted_attribute(tokens[(50 + p1_offset)])); - if( !empty_attribute( tokens[(51 + p1_offset)] ) ) outputMessage.put(WFRecipient, unquoted_attribute(tokens[(51 + p1_offset)])); - if( !empty_attribute( tokens[(52 + p1_offset)] ) ) outputMessage.put(WFReportID, unquoted_attribute(tokens[(52 + p1_offset)])); - } - if ( parser_version >= 70) { - if( !empty_attribute( tokens[45] ) ) outputMessage.put(URLIndex, tokens[45].trim()); - if( !empty_attribute( tokens[54] ) ) outputMessage.put(DGH1, tokens[54].trim()); - if( !empty_attribute( tokens[55] ) ) outputMessage.put(DGH2, tokens[55].trim()); - if( !empty_attribute( tokens[56] ) ) outputMessage.put(DGH3, tokens[56].trim()); - if( !empty_attribute( tokens[57] ) ) outputMessage.put(DGH4, tokens[57].trim()); - if( !empty_attribute( tokens[58] ) ) outputMessage.put(VSYSName, unquoted_attribute(tokens[58])); - if( !empty_attribute( tokens[59] ) ) outputMessage.put(DeviceName, unquoted_attribute(tokens[59])); - } - if ( parser_version >= 80) { - if( !empty_attribute( tokens[61] ) ) outputMessage.put(SourceVmUuid, tokens[61].trim()); - if( !empty_attribute( tokens[62] ) ) outputMessage.put(DestinationVmUuid, tokens[62].trim()); - if( !empty_attribute( tokens[63] ) ) outputMessage.put(HTTPMethod, tokens[63].trim()); - if( !empty_attribute( tokens[64] ) ) outputMessage.put(TunnelId, tokens[64].trim()); - if( !empty_attribute( tokens[65] ) ) outputMessage.put(MonitorTag, tokens[65].trim()); - if( !empty_attribute( tokens[66] ) ) outputMessage.put(ParentSessionId, tokens[66].trim()); - if( !empty_attribute( tokens[67] ) ) outputMessage.put(ParentSessionStartTime, tokens[67].trim()); - if( !empty_attribute( tokens[68] ) ) outputMessage.put(TunnelType, tokens[68].trim()); - if( !empty_attribute( tokens[69] ) ) outputMessage.put(ThreatCategory, tokens[69].trim()); - if( !empty_attribute( tokens[70] ) ) outputMessage.put(ContentVersion, tokens[70].trim()); - } - if ( parser_version == 0) { - outputMessage.put(Tokens, tokens.length); - } - - } else if ("TRAFFIC".equals(type.toUpperCase())) { - if (tokens.length == 46) parser_version = 60; - else if (tokens.length == 47) parser_version = 61; - else if (tokens.length == 54) parser_version = 70; - else if (tokens.length == 61) parser_version = 80; - outputMessage.put(ParserVersion, parser_version); - if( !empty_attribute( tokens[31] ) ) outputMessage.put(Bytes, tokens[31].trim()); - if( !empty_attribute( tokens[32] ) ) outputMessage.put(BytesSent, tokens[32].trim()); - if( !empty_attribute( tokens[33] ) ) outputMessage.put(BytesReceived, tokens[33].trim()); - if( !empty_attribute( tokens[34] ) ) outputMessage.put(Packets, tokens[34].trim()); - if( !empty_attribute( tokens[35] ) ) outputMessage.put(StartTime, tokens[35].trim()); - if( !empty_attribute( tokens[36] ) ) outputMessage.put(ElapsedTimeInSec, tokens[36].trim()); - if( !empty_attribute( tokens[37] ) ) outputMessage.put(Category, unquoted_attribute(tokens[37])); - if( !empty_attribute( tokens[39] ) ) outputMessage.put(Seqno, tokens[39].trim()); - if( !empty_attribute( tokens[40] ) ) outputMessage.put(ActionFlags, unquoted_attribute(tokens[40])); - if( !empty_attribute( tokens[41] ) ) outputMessage.put(SourceLocation, unquoted_attribute(tokens[41])); - if( !empty_attribute( tokens[42] ) ) outputMessage.put(DestinationLocation, unquoted_attribute(tokens[42])); - if( !empty_attribute( tokens[44] ) ) outputMessage.put(PktsSent, tokens[44].trim()); - if( !empty_attribute( tokens[45] ) ) outputMessage.put(PktsReceived, tokens[45].trim()); - if ( parser_version >= 61) { - if( !empty_attribute( tokens[46] ) ) outputMessage.put(EndReason, unquoted_attribute(tokens[46])); + if (parser_version >= 70) { + int custom_fields_offset = 0; + if (tokens.length == 24) { + if (!empty_attribute(tokens[14])) { + outputMessage.put(BeforeChangeDetail, unquoted_attribute(tokens[14 + custom_fields_offset])); + } + if (!empty_attribute(tokens[15])) { + outputMessage.put(AfterChangeDetail, unquoted_attribute(tokens[15 + custom_fields_offset])); + } + custom_fields_offset = 2; + } + if (!empty_attribute(tokens[14 + custom_fields_offset])) { + outputMessage.put(Seqno, unquoted_attribute(tokens[14 + custom_fields_offset])); + } + if (!empty_attribute(tokens[15 + custom_fields_offset])) { + outputMessage.put(ActionFlags, unquoted_attribute(tokens[15 + custom_fields_offset])); + } + if (!empty_attribute(tokens[16 + custom_fields_offset])) { + outputMessage.put(DGH1, unquoted_attribute(tokens[16 + custom_fields_offset])); + } + if (!empty_attribute(tokens[17 + custom_fields_offset])) { + outputMessage.put(DGH2, unquoted_attribute(tokens[17 + custom_fields_offset])); + } + if (!empty_attribute(tokens[18 + custom_fields_offset])) { + outputMessage.put(DGH3, unquoted_attribute(tokens[18 + custom_fields_offset])); + } + if (!empty_attribute(tokens[19 + custom_fields_offset])) { + outputMessage.put(DGH4, unquoted_attribute(tokens[19 + custom_fields_offset])); + } + if (!empty_attribute(tokens[20 + custom_fields_offset])) { + outputMessage.put(VSYSName, unquoted_attribute(tokens[20 + custom_fields_offset])); + } + if (!empty_attribute(tokens[21 + custom_fields_offset])) { + outputMessage.put(DeviceName, unquoted_attribute(tokens[21 + custom_fields_offset])); + } } - if ( parser_version >= 70) { - if( !empty_attribute( tokens[47] ) ) outputMessage.put(DGH1, tokens[47].trim()); - if( !empty_attribute( tokens[48] ) ) outputMessage.put(DGH2, tokens[48].trim()); - if( !empty_attribute( tokens[49] ) ) outputMessage.put(DGH3, tokens[49].trim()); - if( !empty_attribute( tokens[50] ) ) outputMessage.put(DGH4, tokens[50].trim()); - if( !empty_attribute( tokens[51] ) ) outputMessage.put(VSYSName, unquoted_attribute(tokens[51])); - if( !empty_attribute( tokens[52] ) ) outputMessage.put(DeviceName, unquoted_attribute(tokens[52])); - if( !empty_attribute( tokens[53] ) ) outputMessage.put(ActionSource, unquoted_attribute(tokens[53])); + } else if (LogTypeSystem.equals(type.toUpperCase())) { + if (tokens.length == 17) parser_version = 61; + else if (tokens.length == 23) parser_version = 80; + + if (parser_version >= 61) { + if (!empty_attribute(tokens[7])) outputMessage.put(VirtualSystem, tokens[7].trim()); + if (!empty_attribute(tokens[8])) outputMessage.put(EventId, tokens[8].trim()); + if (!empty_attribute(tokens[9])) outputMessage.put(Object, tokens[9].trim()); + + if (!empty_attribute(tokens[12])) outputMessage.put(Module, tokens[12].trim()); + if (!empty_attribute(tokens[13])) outputMessage.put(Severity, unquoted_attribute(tokens[13])); + if (!empty_attribute(tokens[14])) outputMessage.put(Description, unquoted_attribute(tokens[14])); + if (!empty_attribute(tokens[15])) outputMessage.put(Seqno, unquoted_attribute(tokens[15])); + if (!empty_attribute(tokens[16])) outputMessage.put(ActionFlags, unquoted_attribute(tokens[16])); } - if ( parser_version >= 80) { - if( !empty_attribute( tokens[54] ) ) outputMessage.put(SourceVmUuid, tokens[54].trim()); - if( !empty_attribute( tokens[55] ) ) outputMessage.put(DestinationVmUuid, tokens[55].trim()); - if( !empty_attribute( tokens[56] ) ) outputMessage.put(TunnelId, tokens[56].trim()); - if( !empty_attribute( tokens[57] ) ) outputMessage.put(MonitorTag, tokens[57].trim()); - if( !empty_attribute( tokens[58] ) ) outputMessage.put(ParentSessionId, tokens[58].trim()); - if( !empty_attribute( tokens[59] ) ) outputMessage.put(ParentSessionStartTime, tokens[59].trim()); - if( !empty_attribute( tokens[60] ) ) outputMessage.put(TunnelType, tokens[60].trim()); + + if (parser_version == 80) { + if (!empty_attribute(tokens[17])) outputMessage.put(DGH1, tokens[17].trim()); + if (!empty_attribute(tokens[18])) outputMessage.put(DGH2, tokens[18].trim()); + if (!empty_attribute(tokens[19])) outputMessage.put(DGH3, tokens[19].trim()); + if (!empty_attribute(tokens[20])) outputMessage.put(DGH4, tokens[20].trim()); + if (!empty_attribute(tokens[21])) outputMessage.put(VSYSName, unquoted_attribute(tokens[21])); + if (!empty_attribute(tokens[22])) outputMessage.put(DeviceName, unquoted_attribute(tokens[22])); } - if ( parser_version == 0) { - outputMessage.put(Tokens, tokens.length); + } else if (LogTypeThreat.equals(type.toUpperCase()) || + LogTypeTraffic.equals(type.toUpperCase())) { + if (!empty_attribute(tokens[7])) outputMessage.put(SourceAddress, tokens[7].trim()); + if (!empty_attribute(tokens[8])) outputMessage.put(DestinationAddress, tokens[8].trim()); + if (!empty_attribute(tokens[9])) outputMessage.put(NATSourceIP, tokens[9].trim()); + if (!empty_attribute(tokens[10])) outputMessage.put(NATDestinationIP, tokens[10].trim()); + if (!empty_attribute(tokens[11])) outputMessage.put(Rule, unquoted_attribute(tokens[11])); + if (!empty_attribute(tokens[12])) outputMessage.put(SourceUser, unquoted_attribute(tokens[12])); + if (!empty_attribute(tokens[13])) outputMessage.put(DestinationUser, unquoted_attribute(tokens[13])); + if (!empty_attribute(tokens[14])) outputMessage.put(Application, unquoted_attribute(tokens[14])); + if (!empty_attribute(tokens[15])) outputMessage.put(VirtualSystem, unquoted_attribute(tokens[15])); + if (!empty_attribute(tokens[16])) outputMessage.put(SourceZone, unquoted_attribute(tokens[16])); + if (!empty_attribute(tokens[17])) outputMessage.put(DestinationZone, unquoted_attribute(tokens[17])); + if (!empty_attribute(tokens[18])) outputMessage.put(InboundInterface, unquoted_attribute(tokens[18])); + if (!empty_attribute(tokens[19])) outputMessage.put(OutboundInterface, unquoted_attribute(tokens[19])); + if (!empty_attribute(tokens[20])) outputMessage.put(LogAction, unquoted_attribute(tokens[20])); + if (!empty_attribute(tokens[21])) outputMessage.put(TimeLogged, tokens[21].trim()); + if (!empty_attribute(tokens[22])) outputMessage.put(SessionID, tokens[22].trim()); + if (!empty_attribute(tokens[23])) outputMessage.put(RepeatCount, tokens[23].trim()); + if (!empty_attribute(tokens[24])) outputMessage.put(SourcePort, tokens[24].trim()); + if (!empty_attribute(tokens[25])) outputMessage.put(DestinationPort, tokens[25].trim()); + if (!empty_attribute(tokens[26])) outputMessage.put(NATSourcePort, tokens[26].trim()); + if (!empty_attribute(tokens[27])) outputMessage.put(NATDestinationPort, tokens[27].trim()); + if (!empty_attribute(tokens[28])) outputMessage.put(Flags, tokens[28].trim()); + if (!empty_attribute(tokens[29])) outputMessage.put(IPProtocol, unquoted_attribute(tokens[29])); + if (!empty_attribute(tokens[30])) outputMessage.put(Action, unquoted_attribute(tokens[30])); + + if (LogTypeThreat.equals(type.toUpperCase())) { + int p1_offset = 0; + if (tokens.length == 45) parser_version = 60; + else if (tokens.length == 53) parser_version = 61; + else if (tokens.length == 61) { + parser_version = 70; + p1_offset = 1; + } else if (tokens.length == 72) { + parser_version = 80; + p1_offset = 1; + } + if (!empty_attribute(tokens[31])) { + outputMessage.put(URL, unquoted_attribute(tokens[31])); + try { + URL url = new URL(unquoted_attribute(tokens[31])); + outputMessage.put(HOST, url.getHost()); + } catch (MalformedURLException e) { + } + } + if (!empty_attribute(tokens[32])) outputMessage.put(ThreatID, tokens[32].trim()); + if (!empty_attribute(tokens[33])) outputMessage.put(Category, unquoted_attribute(tokens[33])); + if (!empty_attribute(tokens[34])) outputMessage.put(Severity, unquoted_attribute(tokens[34])); + if (!empty_attribute(tokens[35])) outputMessage.put(Direction, unquoted_attribute(tokens[35])); + if (!empty_attribute(tokens[36])) outputMessage.put(Seqno, tokens[36].trim()); + if (!empty_attribute(tokens[37])) outputMessage.put(ActionFlags, unquoted_attribute(tokens[37])); + if (!empty_attribute(tokens[38])) outputMessage.put(SourceLocation, unquoted_attribute(tokens[38])); + if (!empty_attribute(tokens[39])) + outputMessage.put(DestinationLocation, unquoted_attribute(tokens[39])); + if (!empty_attribute(tokens[41])) outputMessage.put(ContentType, unquoted_attribute(tokens[41])); + if (!empty_attribute(tokens[42])) outputMessage.put(PCAPID, tokens[42].trim()); + if (!empty_attribute(tokens[43])) outputMessage.put(WFFileDigest, unquoted_attribute(tokens[43])); + if (!empty_attribute(tokens[44])) outputMessage.put(WFCloud, unquoted_attribute(tokens[44])); + if (parser_version >= 61) { + if (!empty_attribute(tokens[(45 + p1_offset)])) + outputMessage.put(UserAgent, unquoted_attribute(tokens[(45 + p1_offset)])); + if (!empty_attribute(tokens[(46 + p1_offset)])) + outputMessage.put(WFFileType, unquoted_attribute(tokens[(46 + p1_offset)])); + if (!empty_attribute(tokens[(47 + p1_offset)])) + outputMessage.put(XForwardedFor, unquoted_attribute(tokens[(47 + p1_offset)])); + if (!empty_attribute(tokens[(48 + p1_offset)])) + outputMessage.put(Referer, unquoted_attribute(tokens[(48 + p1_offset)])); + if (!empty_attribute(tokens[(49 + p1_offset)])) + outputMessage.put(WFSender, unquoted_attribute(tokens[(49 + p1_offset)])); + if (!empty_attribute(tokens[(50 + p1_offset)])) + outputMessage.put(WFSubject, unquoted_attribute(tokens[(50 + p1_offset)])); + if (!empty_attribute(tokens[(51 + p1_offset)])) + outputMessage.put(WFRecipient, unquoted_attribute(tokens[(51 + p1_offset)])); + if (!empty_attribute(tokens[(52 + p1_offset)])) + outputMessage.put(WFReportID, unquoted_attribute(tokens[(52 + p1_offset)])); + } + if (parser_version >= 70) { + if (!empty_attribute(tokens[45])) outputMessage.put(URLIndex, tokens[45].trim()); + if (!empty_attribute(tokens[54])) outputMessage.put(DGH1, tokens[54].trim()); + if (!empty_attribute(tokens[55])) outputMessage.put(DGH2, tokens[55].trim()); + if (!empty_attribute(tokens[56])) outputMessage.put(DGH3, tokens[56].trim()); + if (!empty_attribute(tokens[57])) outputMessage.put(DGH4, tokens[57].trim()); + if (!empty_attribute(tokens[58])) outputMessage.put(VSYSName, unquoted_attribute(tokens[58])); + if (!empty_attribute(tokens[59])) outputMessage.put(DeviceName, unquoted_attribute(tokens[59])); + } + if (parser_version >= 80) { + if (!empty_attribute(tokens[61])) outputMessage.put(SourceVmUuid, tokens[61].trim()); + if (!empty_attribute(tokens[62])) outputMessage.put(DestinationVmUuid, tokens[62].trim()); + if (!empty_attribute(tokens[63])) outputMessage.put(HTTPMethod, tokens[63].trim()); + if (!empty_attribute(tokens[64])) outputMessage.put(TunnelId, tokens[64].trim()); + if (!empty_attribute(tokens[65])) outputMessage.put(MonitorTag, tokens[65].trim()); + if (!empty_attribute(tokens[66])) outputMessage.put(ParentSessionId, tokens[66].trim()); + if (!empty_attribute(tokens[67])) outputMessage.put(ParentSessionStartTime, tokens[67].trim()); + if (!empty_attribute(tokens[68])) outputMessage.put(TunnelType, tokens[68].trim()); + if (!empty_attribute(tokens[69])) outputMessage.put(ThreatCategory, tokens[69].trim()); + if (!empty_attribute(tokens[70])) outputMessage.put(ContentVersion, tokens[70].trim()); + } + } else if (LogTypeTraffic.equals(type.toUpperCase())) { + if (tokens.length == 46) parser_version = 60; + else if (tokens.length == 47) parser_version = 61; + else if (tokens.length == 54) parser_version = 70; + else if (tokens.length == 61) parser_version = 80; + if (!empty_attribute(tokens[31])) outputMessage.put(Bytes, tokens[31].trim()); + if (!empty_attribute(tokens[32])) outputMessage.put(BytesSent, tokens[32].trim()); + if (!empty_attribute(tokens[33])) outputMessage.put(BytesReceived, tokens[33].trim()); + if (!empty_attribute(tokens[34])) outputMessage.put(Packets, tokens[34].trim()); + if (!empty_attribute(tokens[35])) outputMessage.put(StartTime, tokens[35].trim()); + if (!empty_attribute(tokens[36])) outputMessage.put(ElapsedTimeInSec, tokens[36].trim()); + if (!empty_attribute(tokens[37])) outputMessage.put(Category, unquoted_attribute(tokens[37])); + if (!empty_attribute(tokens[39])) outputMessage.put(Seqno, tokens[39].trim()); + if (!empty_attribute(tokens[40])) outputMessage.put(ActionFlags, unquoted_attribute(tokens[40])); + if (!empty_attribute(tokens[41])) outputMessage.put(SourceLocation, unquoted_attribute(tokens[41])); + if (!empty_attribute(tokens[42])) + outputMessage.put(DestinationLocation, unquoted_attribute(tokens[42])); + if (!empty_attribute(tokens[44])) outputMessage.put(PktsSent, tokens[44].trim()); + if (!empty_attribute(tokens[45])) outputMessage.put(PktsReceived, tokens[45].trim()); + if (parser_version >= 61) { + if (!empty_attribute(tokens[46])) outputMessage.put(EndReason, unquoted_attribute(tokens[46])); + } + if (parser_version >= 70) { + if (!empty_attribute(tokens[47])) outputMessage.put(DGH1, tokens[47].trim()); + if (!empty_attribute(tokens[48])) outputMessage.put(DGH2, tokens[48].trim()); + if (!empty_attribute(tokens[49])) outputMessage.put(DGH3, tokens[49].trim()); + if (!empty_attribute(tokens[50])) outputMessage.put(DGH4, tokens[50].trim()); + if (!empty_attribute(tokens[51])) outputMessage.put(VSYSName, unquoted_attribute(tokens[51])); + if (!empty_attribute(tokens[52])) outputMessage.put(DeviceName, unquoted_attribute(tokens[52])); + if (!empty_attribute(tokens[53])) outputMessage.put(ActionSource, unquoted_attribute(tokens[53])); + } + if (parser_version >= 80) { + if (!empty_attribute(tokens[54])) outputMessage.put(SourceVmUuid, tokens[54].trim()); + if (!empty_attribute(tokens[55])) outputMessage.put(DestinationVmUuid, tokens[55].trim()); + if (!empty_attribute(tokens[56])) outputMessage.put(TunnelId, tokens[56].trim()); + if (!empty_attribute(tokens[57])) outputMessage.put(MonitorTag, tokens[57].trim()); + if (!empty_attribute(tokens[58])) outputMessage.put(ParentSessionId, tokens[58].trim()); + if (!empty_attribute(tokens[59])) outputMessage.put(ParentSessionStartTime, tokens[59].trim()); + if (!empty_attribute(tokens[60])) outputMessage.put(TunnelType, tokens[60].trim()); + } } } - + outputMessage.put(ParserVersion, parser_version); + if (parser_version == 0) { + outputMessage.put(Tokens, tokens.length); + } } - - } http://git-wip-us.apache.org/repos/asf/metron/blob/61b1ede0/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/paloalto/README.md ---------------------------------------------------------------------- diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/paloalto/README.md b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/paloalto/README.md new file mode 100644 index 0000000..b6a1230 --- /dev/null +++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/paloalto/README.md @@ -0,0 +1,32 @@ +<!-- +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +--> +# BasicPaloAltoFirewallParser +## Introduction +The parser is able to parse logs generated by Palo Alto firewall devices powered by Pan OS. The supported log format is CSV. + +The supported log types and versions + +| Log type | Pan OS version | +|----------|----------------| +| Traffic | 6.0, 6.1, 7.0, 8.0 | +| Threat | 6.0, 6.1, 7.0, 8.0 | +| Config | 6.1, 7.0, 8.0 | +| System | 6.1, 7.0, 8.0 | + + + http://git-wip-us.apache.org/repos/asf/metron/blob/61b1ede0/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParserTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParserTest.java index 2c90b1e..cc6191c 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParserTest.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParserTest.java @@ -18,6 +18,7 @@ package org.apache.metron.parsers.paloalto; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; import org.apache.metron.parsers.AbstractParserConfigTest; import org.json.simple.JSONObject; @@ -25,6 +26,8 @@ import org.json.simple.parser.ParseException; import org.junit.Before; import org.junit.Test; +import java.util.List; + public class BasicPaloAltoFirewallParserTest extends AbstractParserConfigTest { @Before @@ -32,6 +35,221 @@ public class BasicPaloAltoFirewallParserTest extends AbstractParserConfigTest { parser = new BasicPaloAltoFirewallParser(); } + @SuppressWarnings("unchecked") + @Test + public void testParseSystem61() throws ParseException { + final String SYSTEM_61 = "1,2017/08/11 12:37:58,008900008659,SYSTEM,general,1,2017/08/11 11:37:58,vsys1,eventId_test,object_test,Futureuse1_test,futureuse2_test,management,high,Description_test,1354,0x0"; + + JSONObject actual = parser.parse(SYSTEM_61.getBytes()).get(0); + + JSONObject expected = new JSONObject(); + expected.put(BasicPaloAltoFirewallParser.PaloAltoDomain, "1"); + expected.put(BasicPaloAltoFirewallParser.ReceiveTime, "2017/08/11 12:37:58"); + expected.put(BasicPaloAltoFirewallParser.SerialNum, "008900008659"); + expected.put(BasicPaloAltoFirewallParser.Type, "SYSTEM"); + expected.put(BasicPaloAltoFirewallParser.ThreatContentType, "general"); + expected.put(BasicPaloAltoFirewallParser.ConfigVersion, "1"); + expected.put(BasicPaloAltoFirewallParser.GenerateTime, "2017/08/11 11:37:58"); + expected.put(BasicPaloAltoFirewallParser.VirtualSystem, "vsys1"); + expected.put(BasicPaloAltoFirewallParser.EventId, "eventId_test"); + expected.put(BasicPaloAltoFirewallParser.Object, "object_test"); + expected.put(BasicPaloAltoFirewallParser.Module, "management"); + expected.put(BasicPaloAltoFirewallParser.Severity, "high"); + expected.put(BasicPaloAltoFirewallParser.Description, "Description_test"); + expected.put(BasicPaloAltoFirewallParser.Seqno, "1354"); + expected.put(BasicPaloAltoFirewallParser.ActionFlags, "0x0"); + expected.put(BasicPaloAltoFirewallParser.ParserVersion, 61); + expected.put("original_string", SYSTEM_61); + expected.put("timestamp", actual.get("timestamp")); + + assertEquals(expected, actual); + } + + @SuppressWarnings("unchecked") + @Test + public void testParseSystem80() throws ParseException { + final String SYSTEM_80 = "1,2017/08/11 12:37:58,008900008659,SYSTEM,general,1,2017/08/11 11:37:58,vsys1,eventId_test,object_test,Futureuse1_test,futureuse2_test,management,high,Description_test,1354,0x0,12,34,45,0,virSys1,dev-something200-01"; + + JSONObject actual = parser.parse(SYSTEM_80.getBytes()).get(0); + + JSONObject expected = new JSONObject(); + expected.put(BasicPaloAltoFirewallParser.PaloAltoDomain, "1"); + expected.put(BasicPaloAltoFirewallParser.ReceiveTime, "2017/08/11 12:37:58"); + expected.put(BasicPaloAltoFirewallParser.SerialNum, "008900008659"); + expected.put(BasicPaloAltoFirewallParser.Type, "SYSTEM"); + expected.put(BasicPaloAltoFirewallParser.ThreatContentType, "general"); + expected.put(BasicPaloAltoFirewallParser.ConfigVersion, "1"); + expected.put(BasicPaloAltoFirewallParser.GenerateTime, "2017/08/11 11:37:58"); + expected.put(BasicPaloAltoFirewallParser.VirtualSystem, "vsys1"); + expected.put(BasicPaloAltoFirewallParser.EventId, "eventId_test"); + expected.put(BasicPaloAltoFirewallParser.Object, "object_test"); + expected.put(BasicPaloAltoFirewallParser.Module, "management"); + expected.put(BasicPaloAltoFirewallParser.Severity, "high"); + expected.put(BasicPaloAltoFirewallParser.Description, "Description_test"); + expected.put(BasicPaloAltoFirewallParser.Seqno, "1354"); + expected.put(BasicPaloAltoFirewallParser.ActionFlags, "0x0"); + expected.put(BasicPaloAltoFirewallParser.DGH1, "12"); + expected.put(BasicPaloAltoFirewallParser.DGH2, "34"); + expected.put(BasicPaloAltoFirewallParser.DGH3, "45"); + expected.put(BasicPaloAltoFirewallParser.DGH4, "0"); + expected.put(BasicPaloAltoFirewallParser.VSYSName, "virSys1"); + expected.put(BasicPaloAltoFirewallParser.DeviceName, "dev-something200-01"); + + expected.put(BasicPaloAltoFirewallParser.ParserVersion, 80); + expected.put("original_string", SYSTEM_80); + expected.put("timestamp", actual.get("timestamp")); + + assertEquals(expected, actual); + } + + @SuppressWarnings("unchecked") + @Test + public void testParseConfig61NoCustomFields() throws ParseException { + final String CONFIG_61_customFields = "1,2017/08/11 12:37:58,008900008659,CONFIG,0,1,2017/08/11 11:37:58,192.168.14.162,vsys1,edit,admin,Web,Succeeded, config shared log-settings config,1354,0x0"; + + JSONObject actual = parser.parse(CONFIG_61_customFields.getBytes()).get(0); + + JSONObject expected = new JSONObject(); + expected.put(BasicPaloAltoFirewallParser.PaloAltoDomain, "1"); + expected.put(BasicPaloAltoFirewallParser.ReceiveTime, "2017/08/11 12:37:58"); + expected.put(BasicPaloAltoFirewallParser.SerialNum, "008900008659"); + expected.put(BasicPaloAltoFirewallParser.Type, "CONFIG"); + expected.put(BasicPaloAltoFirewallParser.ThreatContentType, "0"); + expected.put(BasicPaloAltoFirewallParser.ConfigVersion, "1"); + expected.put(BasicPaloAltoFirewallParser.GenerateTime, "2017/08/11 11:37:58"); + + expected.put(BasicPaloAltoFirewallParser.HOST, "192.168.14.162"); + expected.put(BasicPaloAltoFirewallParser.VirtualSystem, "vsys1"); + expected.put(BasicPaloAltoFirewallParser.Command, "edit"); + expected.put(BasicPaloAltoFirewallParser.Admin, "admin"); + expected.put(BasicPaloAltoFirewallParser.Client, "Web"); + expected.put(BasicPaloAltoFirewallParser.Result, "Succeeded"); + expected.put(BasicPaloAltoFirewallParser.ConfigurationPath, "config shared log-settings config"); + expected.put(BasicPaloAltoFirewallParser.Seqno, "1354"); + expected.put(BasicPaloAltoFirewallParser.ActionFlags, "0x0"); + + expected.put(BasicPaloAltoFirewallParser.ParserVersion, 61); + expected.put("original_string", CONFIG_61_customFields); + expected.put("timestamp", actual.get("timestamp")); + + assertEquals(expected, actual); + } + + @SuppressWarnings("unchecked") + @Test + public void testParseConfig61CustomFields() throws ParseException { + final String CONFIG_61_noCustomFields = "1,2017/08/11 12:37:58,008900008659,CONFIG,0,1,2017/08/11 11:37:58,192.168.14.162,vsys1,edit,admin,Web,Succeeded, config shared log-settings config,1354,0x0,/FatherNode/KidNode/GrandsonNode1,/FatherNode/KidNode/GrandsonNode2"; + + JSONObject actual = parser.parse(CONFIG_61_noCustomFields.getBytes()).get(0); + + JSONObject expected = new JSONObject(); + expected.put(BasicPaloAltoFirewallParser.PaloAltoDomain, "1"); + expected.put(BasicPaloAltoFirewallParser.ReceiveTime, "2017/08/11 12:37:58"); + expected.put(BasicPaloAltoFirewallParser.SerialNum, "008900008659"); + expected.put(BasicPaloAltoFirewallParser.Type, "CONFIG"); + expected.put(BasicPaloAltoFirewallParser.ThreatContentType, "0"); + expected.put(BasicPaloAltoFirewallParser.ConfigVersion, "1"); + expected.put(BasicPaloAltoFirewallParser.GenerateTime, "2017/08/11 11:37:58"); + + expected.put(BasicPaloAltoFirewallParser.HOST, "192.168.14.162"); + expected.put(BasicPaloAltoFirewallParser.VirtualSystem, "vsys1"); + expected.put(BasicPaloAltoFirewallParser.Command, "edit"); + expected.put(BasicPaloAltoFirewallParser.Admin, "admin"); + expected.put(BasicPaloAltoFirewallParser.Client, "Web"); + expected.put(BasicPaloAltoFirewallParser.Result, "Succeeded"); + expected.put(BasicPaloAltoFirewallParser.ConfigurationPath, "config shared log-settings config"); + expected.put(BasicPaloAltoFirewallParser.Seqno, "1354"); + expected.put(BasicPaloAltoFirewallParser.ActionFlags, "0x0"); + expected.put(BasicPaloAltoFirewallParser.BeforeChangeDetail, "/FatherNode/KidNode/GrandsonNode1"); + expected.put(BasicPaloAltoFirewallParser.AfterChangeDetail, "/FatherNode/KidNode/GrandsonNode2"); + + expected.put(BasicPaloAltoFirewallParser.ParserVersion, 61); + expected.put("original_string", CONFIG_61_noCustomFields); + expected.put("timestamp", actual.get("timestamp")); + + assertEquals(expected, actual); + } + + @SuppressWarnings("unchecked") + @Test + public void testParseConfig70And80NoCustomFields() throws ParseException { + final String CONFIG_70_80_noCustomFields = "1,2017/08/11 12:37:58,008900008659,CONFIG,0,1,2017/08/11 11:37:58,192.168.14.162,vsys1,edit,admin,Web,Succeeded, config shared log-settings config,1354,0x0,12,34,45,0,virSys1,dev-something200-01"; + + JSONObject actual = parser.parse(CONFIG_70_80_noCustomFields.getBytes()).get(0); + + JSONObject expected = new JSONObject(); + expected.put(BasicPaloAltoFirewallParser.PaloAltoDomain, "1"); + expected.put(BasicPaloAltoFirewallParser.ReceiveTime, "2017/08/11 12:37:58"); + expected.put(BasicPaloAltoFirewallParser.SerialNum, "008900008659"); + expected.put(BasicPaloAltoFirewallParser.Type, "CONFIG"); + expected.put(BasicPaloAltoFirewallParser.ThreatContentType, "0"); + expected.put(BasicPaloAltoFirewallParser.ConfigVersion, "1"); + expected.put(BasicPaloAltoFirewallParser.GenerateTime, "2017/08/11 11:37:58"); + + expected.put(BasicPaloAltoFirewallParser.HOST, "192.168.14.162"); + expected.put(BasicPaloAltoFirewallParser.VirtualSystem, "vsys1"); + expected.put(BasicPaloAltoFirewallParser.Command, "edit"); + expected.put(BasicPaloAltoFirewallParser.Admin, "admin"); + expected.put(BasicPaloAltoFirewallParser.Client, "Web"); + expected.put(BasicPaloAltoFirewallParser.Result, "Succeeded"); + expected.put(BasicPaloAltoFirewallParser.ConfigurationPath, "config shared log-settings config"); + expected.put(BasicPaloAltoFirewallParser.Seqno, "1354"); + expected.put(BasicPaloAltoFirewallParser.ActionFlags, "0x0"); + expected.put(BasicPaloAltoFirewallParser.DGH1, "12"); + expected.put(BasicPaloAltoFirewallParser.DGH2, "34"); + expected.put(BasicPaloAltoFirewallParser.DGH3, "45"); + expected.put(BasicPaloAltoFirewallParser.DGH4, "0"); + expected.put(BasicPaloAltoFirewallParser.VSYSName, "virSys1"); + expected.put(BasicPaloAltoFirewallParser.DeviceName, "dev-something200-01"); + + expected.put(BasicPaloAltoFirewallParser.ParserVersion, 80); + expected.put("original_string", CONFIG_70_80_noCustomFields); + expected.put("timestamp", actual.get("timestamp")); + + assertEquals(expected, actual); + } + + @SuppressWarnings("unchecked") + @Test + public void testParseConfig70And80CustomFields() throws ParseException { + final String CONFIG_70_80_customFields = "1,2017/08/11 12:37:58,008900008659,CONFIG,0,1,2017/08/11 11:37:58,192.168.14.162,vsys1,edit,admin,Web,Succeeded,config shared log-settings config,/FatherNode/KidNode/GrandsonNode1,/FatherNode/KidNode/GrandsonNode2,1354,0x0,12,34,45,0,virSys1,dev-something200-01"; + + JSONObject actual = parser.parse(CONFIG_70_80_customFields.getBytes()).get(0); + + JSONObject expected = new JSONObject(); + expected.put(BasicPaloAltoFirewallParser.PaloAltoDomain, "1"); + expected.put(BasicPaloAltoFirewallParser.ReceiveTime, "2017/08/11 12:37:58"); + expected.put(BasicPaloAltoFirewallParser.SerialNum, "008900008659"); + expected.put(BasicPaloAltoFirewallParser.Type, "CONFIG"); + expected.put(BasicPaloAltoFirewallParser.ThreatContentType, "0"); + expected.put(BasicPaloAltoFirewallParser.ConfigVersion, "1"); + expected.put(BasicPaloAltoFirewallParser.GenerateTime, "2017/08/11 11:37:58"); + + expected.put(BasicPaloAltoFirewallParser.HOST, "192.168.14.162"); + expected.put(BasicPaloAltoFirewallParser.VirtualSystem, "vsys1"); + expected.put(BasicPaloAltoFirewallParser.Command, "edit"); + expected.put(BasicPaloAltoFirewallParser.Admin, "admin"); + expected.put(BasicPaloAltoFirewallParser.Client, "Web"); + expected.put(BasicPaloAltoFirewallParser.Result, "Succeeded"); + expected.put(BasicPaloAltoFirewallParser.ConfigurationPath, "config shared log-settings config"); + expected.put(BasicPaloAltoFirewallParser.BeforeChangeDetail, "/FatherNode/KidNode/GrandsonNode1"); + expected.put(BasicPaloAltoFirewallParser.AfterChangeDetail, "/FatherNode/KidNode/GrandsonNode2"); + expected.put(BasicPaloAltoFirewallParser.Seqno, "1354"); + expected.put(BasicPaloAltoFirewallParser.ActionFlags, "0x0"); + expected.put(BasicPaloAltoFirewallParser.DGH1, "12"); + expected.put(BasicPaloAltoFirewallParser.DGH2, "34"); + expected.put(BasicPaloAltoFirewallParser.DGH3, "45"); + expected.put(BasicPaloAltoFirewallParser.DGH4, "0"); + expected.put(BasicPaloAltoFirewallParser.VSYSName, "virSys1"); + expected.put(BasicPaloAltoFirewallParser.DeviceName, "dev-something200-01"); + + expected.put(BasicPaloAltoFirewallParser.ParserVersion, 80); + expected.put("original_string", CONFIG_70_80_customFields); + expected.put("timestamp", actual.get("timestamp")); + + assertEquals(expected, actual); + } + public static final String THREAT_60 = "1,2015/01/05 05:38:58,0006C110285,THREAT,vulnerability,1,2015/01/05 05:38:58,10.0.0.115,216.0.10.198,0.0.0.0,0.0.0.0,EX-Allow,example\\user.name,,web-browsing,vsys1,internal,external,ethernet1/2,ethernet1/1,LOG-Default,2015/01/05 05:38:58,12031,1,54180,80,0,0,0x80004000,tcp,reset-both,\"ad.aspx?f=300x250&id=12;tile=1;ord=67AF705D60B1119C0F18BEA336F9\",HTTP: IIS Denial Of Service Attempt(40019),any,high,client-to-server,347368099,0x0,10.0.0.0-10.255.255.255,US,0,,1200568889751109656,,"; @SuppressWarnings("unchecked") @@ -88,6 +306,7 @@ public class BasicPaloAltoFirewallParserTest extends AbstractParserConfigTest { } public static final String TRAFFIC_60 = "1,2015/01/05 12:51:33,0011C103117,TRAFFIC,end,1,2015/01/05 12:51:33,10.0.0.39,10.1.0.163,0.0.0.0,0.0.0.0,EX-Allow,,example\\\\user.name,ms-ds-smb,vsys1,v_external,v_internal,ethernet1/2,ethernet1/1,LOG-Default,2015/01/05 12:51:33,33760927,1,52688,445,0,0,0x401a,tcp,allow,2229,1287,942,10,2015/01/05 12:51:01,30,any,0,17754932062,0x0,10.0.0.0-10.255.255.255,10.0.0.0-10.255.255.255,0,6,"; + @SuppressWarnings("unchecked") @Test public void testParseTraffic60() throws ParseException { @@ -142,6 +361,7 @@ public class BasicPaloAltoFirewallParserTest extends AbstractParserConfigTest { } public static final String THREAT_70 = "1,2017/05/24 09:53:10,001801000001,THREAT,virus,0,2017/05/24 09:53:10,217.1.2.3,10.1.8.7,217.1.2.3,214.123.1.2,WLAN-Internet,,user,web-browsing,vsys1,Untrust,wifi_zone,ethernet1/1,vlan.1,Std-Log-Forward,2017/05/24 09:53:10,49567,1,80,51787,80,25025,0x400000,tcp,reset-both,\"abcdef310.exe\",Virus/Win32.WGeneric.lumeo(2457399),computer-and-internet-info,medium,server-to-client,329423829,0x0,DE,10.0.0.0-10.255.255.255,0,,0,,,1,,,\"\",\"\",,,,0,19,0,0,0,,PAN1,"; + @SuppressWarnings("unchecked") @Test public void testParseThreat70() throws ParseException { @@ -202,6 +422,7 @@ public class BasicPaloAltoFirewallParserTest extends AbstractParserConfigTest { } public static final String TRAFFIC_70 = "1,2017/05/25 21:38:13,001606000003,TRAFFIC,drop,1,2017/05/25 21:38:13,10.2.1.8,192.168.1.10,0.0.0.0,0.0.0.0,DropLog,,,not-applicable,vsys1,intern,VPN,vlan.1,,Std-Log-Forward,2017/05/25 21:38:13,0,1,137,137,0,0,0x0,udp,deny,114,114,0,1,2017/05/25 21:38:12,0,any,0,9953744,0x0,192.168.0.0-192.168.255.255,DE,0,1,0,policy-deny,19,0,0,0,,PAN1,from-policy"; + @SuppressWarnings("unchecked") @Test public void testParseTraffic70() throws ParseException { @@ -262,6 +483,7 @@ public class BasicPaloAltoFirewallParserTest extends AbstractParserConfigTest { } public static final String TRAFFIC_71 = "1,2017/05/31 23:59:57,0006C000005,TRAFFIC,drop,0,2017/05/31 23:59:57,185.94.1.1,201.1.4.5,0.0.0.0,0.0.0.0,DropLog,,,not-applicable,vsys1,untrust,untrust,vlan.1,,Standard-Syslog,2017/05/31 23:59:57,0,1,59836,123,0,0,0x0,udp,deny,60,60,0,1,2017/05/31 23:59:57,0,any,0,3433072193,0x0,RU,DE,0,1,0,policy-deny,16,11,0,0,,PAN1,from-policy"; + @SuppressWarnings("unchecked") @Test public void testParseTraffic71() throws ParseException { @@ -322,6 +544,7 @@ public class BasicPaloAltoFirewallParserTest extends AbstractParserConfigTest { } public static final String THREAT_71 = "1,2017/05/25 19:31:13,0006C000005,THREAT,url,0,2017/05/25 19:31:13,192.168.1.7,140.177.26.29,201.1.4.5,140.177.26.29,ms_out,,,ssl,vsys1,mgmt,untrust,vlan.199,vlan.1,Standard-Syslog,2017/05/25 19:31:13,50556,1,56059,443,14810,443,0x40b000,tcp,alert,\"settings-win.data.microsoft.com/\",(9999),computer-and-internet-info,informational,client-to-server,10030265,0x0,192.168.0.0-192.168.255.255,IE,0,,0,,,0,,,,,,,,0,16,11,0,0,,PAN1,"; + @SuppressWarnings("unchecked") @Test public void testParseThreat71() throws ParseException { @@ -381,6 +604,7 @@ public class BasicPaloAltoFirewallParserTest extends AbstractParserConfigTest { } public static final String THREAT_80 = "1,2018/02/01 21:29:03,001606000007,THREAT,vulnerability,1,2018/02/01 21:29:03,213.211.198.62,172.16.2.6,213.211.198.62,192.168.178.202,Outgoing,,,web-browsing,vsys1,internet,guest,ethernet1/1,ethernet1/2.2,test,2018/02/01 21:29:03,18720,1,80,53161,80,32812,0x402000,tcp,reset-server,\"www.eicar.org/download/eicar.com\",Eicar File Detected(39040),computer-and-internet-info,medium,server-to-client,27438839,0x0,Germany,172.16.0.0-172.31.255.255,0,,0,,,9,,,,,,,,0,0,0,0,0,,PAN1,,,,,0,,0,,N/A,code-execution,AppThreat-771-4450,0x0"; + @SuppressWarnings("unchecked") @Test public void testParseThreat80() throws ParseException { @@ -445,6 +669,7 @@ public class BasicPaloAltoFirewallParserTest extends AbstractParserConfigTest { } public static final String TRAFFIC_80 = "1,2018/02/01 21:24:11,001606000007,TRAFFIC,end,1,2018/02/01 21:24:11,172.16.2.31,134.19.6.22,192.168.18.2,134.19.6.22,Outgoing,,,ssl,vsys1,guest,internet,ethernet1/2.2,ethernet1/1,test,2018/02/01 21:24:11,19468,1,41537,443,12211,443,0x40001c,tcp,allow,7936,1731,6205,24,2018/02/01 21:00:42,1395,computer-and-internet-info,0,62977478,0x0,172.16.0.0-172.31.255.255,United States,0,14,10,tcp-rst-from-client,0,0,0,0,,PAN1,from-policy,,,0,,0,,N/A"; + @SuppressWarnings("unchecked") @Test public void testParseTraffic80() throws ParseException { @@ -507,4 +732,23 @@ public class BasicPaloAltoFirewallParserTest extends AbstractParserConfigTest { expected.put(BasicPaloAltoFirewallParser.DeviceName, "PAN1"); assertEquals(expected, actual); } + + @SuppressWarnings("unchecked") + @Test + public void testParseInvalidLogTypeMessage() throws ParseException { + final String unsupportedLogTypeMessage = "1,2017/08/11 12:37:58,008900008659,INVALIDlogType,0,1,2017/08/11 11:37:58,192.168.14.162,vsys1,edit,admin,Web,Succeeded, config shared log-settings config,1354,0x0"; + List<JSONObject> actual = parser.parse(unsupportedLogTypeMessage.getBytes()); + + assertNull(actual); + } + + @SuppressWarnings("unchecked") + @Test + public void testParseInvalidVersionMessage() throws ParseException { + final String invalidLengthMessage = "1,2017/08/11 12:37:58,008900008659,CONFIG,0,1,2017/08/11 11:37:58,192.168.14.162,vsys1,edit,admin,Web,Succeeded, config shared log-settings config"; + + JSONObject actual = parser.parse(invalidLengthMessage.getBytes()).get(0); + String expectedParserVersion = actual.get(BasicPaloAltoFirewallParser.ParserVersion).toString(); + assertEquals(expectedParserVersion, "0"); + } }
