METRON-941 native PaloAlto parser corrupts message when having a comma in the payload (ctramnitz via justinleet) closes apache/metron#579
Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/5f08ba0b Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/5f08ba0b Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/5f08ba0b Branch: refs/heads/feature/METRON-1416-upgrade-solr Commit: 5f08ba0b1dbe6ba19e8525055f639ecdb85291fc Parents: fa5cff2 Author: ctramnitz <[email protected]> Authored: Fri Feb 16 13:05:06 2018 -0500 Committer: leet <[email protected]> Committed: Fri Feb 16 13:05:06 2018 -0500 ---------------------------------------------------------------------- Upgrading.md | 18 + .../paloalto/BasicPaloAltoFirewallParser.java | 333 +++++++++---- .../BasicPaloAltoFirewallParserTest.java | 493 ++++++++++++++++++- .../logData/PaloAltoFirewallParserTest.txt | 2 - 4 files changed, 718 insertions(+), 128 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/5f08ba0b/Upgrading.md ---------------------------------------------------------------------- diff --git a/Upgrading.md b/Upgrading.md index 047b68e..19da992 100644 --- a/Upgrading.md +++ b/Upgrading.md @@ -19,6 +19,24 @@ limitations under the License. This document constitutes a per-version listing of changes of configuration which are non-backwards compatible. +## 0.4.2 to 0.4.3 + +### [METRON-941: native PaloAlto parser corrupts message when having a comma in the payload](https://issues.apache.org/jira/browse/METRON-941) +While modifying the PaloAlto log parser to support logs from newer +PAN-OS version and to not break when a message payload contains a +comma, some field names were changed to extend the coverage, fix some +duplicate names and change some field names to the Metron standard +message format. + +Installations making use of this parser should check, if the resulting +messages still meet their expectations and adjust downstream configurations +(i.e. ElasticSearch template) accordingly. + +*Note:* Previously, the samples for the test contained a full syslog line +(including syslog header). This did - and will continue to - create a +broken "domain" field in the parsed message. It is recommended to only feed +the syslog message part to the parser for now. + ## 0.4.1 to 0.4.2 ### [METRON-1277: STELLAR Add Match functionality to language](https://issues.apache.org/jira/browse/METRON-1277) http://git-wip-us.apache.org/repos/asf/metron/blob/5f08ba0b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParser.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParser.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParser.java index 46155b3..9051f09 100644 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParser.java +++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParser.java @@ -18,6 +18,8 @@ package org.apache.metron.parsers.paloalto; +import com.google.common.base.Splitter; +import com.google.common.collect.Iterables; import org.apache.metron.parsers.BasicParser; import org.json.simple.JSONObject; import org.slf4j.Logger; @@ -28,68 +30,113 @@ import java.net.URL; import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.regex.Pattern; public class BasicPaloAltoFirewallParser extends BasicParser { + private static boolean empty_attribute( final String s ) { + return s == null || s.trim().isEmpty() || s.equals("\"\""); + } + + private static String unquoted_attribute( String s ) { + s = s.trim(); + if ( s.startsWith( "\"" ) && s.endsWith( "\"" ) ) + return s.substring( 1, s.length( ) - 1 ); + return s; + } + private static final Logger _LOG = LoggerFactory.getLogger (BasicPaloAltoFirewallParser.class); private static final long serialVersionUID = 3147090149725343999L; public static final String PaloAltoDomain = "palo_alto_domain"; public static final String ReceiveTime = "receive_time"; - public static final String SerialNum = "serial_num"; + public static final String SerialNum = "serial"; public static final String Type = "type"; - public static final String ThreatContentType = "threat_content_type"; + public static final String ThreatContentType = "subtype"; public static final String ConfigVersion = "config_version"; - public static final String GenerateTime = "generate_time"; - public static final String SourceAddress = "source_address"; - public static final String DestinationAddress = "destination_address"; - public static final String NATSourceIP = "nat_source_ip"; - public static final String NATDestinationIP = "nat_destination_ip"; + public static final String GenerateTime = "time_generated"; + public static final String SourceAddress = "ip_src_addr"; // Palo Alto name: "src" + public static final String DestinationAddress = "ip_dst_addr"; // Palo Alto name: "dst" + public static final String NATSourceIP = "natsrc"; + public static final String NATDestinationIP = "natdst"; public static final String Rule = "rule"; - public static final String SourceUser = "source_user"; - public static final String DestinationUser = "destination_user"; - public static final String Application = "application"; - public static final String VirtualSystem = "virtual_system"; - public static final String SourceZone = "source_zone"; - public static final String DestinationZone = "destination_zone"; - public static final String InboundInterface = "inbound_interface"; - public static final String OutboundInterface = "outbound_interface"; + public static final String SourceUser = "srcuser"; + public static final String DestinationUser = "dstuser"; + public static final String Application = "app"; + public static final String VirtualSystem = "vsys"; + public static final String SourceZone = "from"; + public static final String DestinationZone = "to"; + public static final String InboundInterface = "inbound_if"; + public static final String OutboundInterface = "outbound_if"; public static final String LogAction = "log_action"; - public static final String TimeLogged = "time_logged"; - public static final String SessionID = "session_id"; - public static final String RepeatCount = "repeat_count"; - public static final String SourcePort = "source_port"; - public static final String DestinationPort = "destination_port"; - public static final String NATSourcePort = "nats_source_port"; - public static final String NATDestinationPort = "nats_destination_port"; + public static final String TimeLogged = "start"; + public static final String SessionID = "sessionid"; + public static final String RepeatCount = "repeatcnt"; + public static final String SourcePort = "ip_src_port"; // Palo Alto name: "sport" + public static final String DestinationPort = "ip_dst_port"; // Palo Alto name: "dport" + public static final String NATSourcePort = "natsport"; + public static final String NATDestinationPort = "natdport"; public static final String Flags = "flags"; - public static final String IPProtocol = "ip_protocol"; + public static final String IPProtocol = "protocol"; // Palo Alto name: "proto" public static final String Action = "action"; + public static final String Seqno = "seqno"; + public static final String ActionFlags = "actionflags"; + public static final String Category = "category"; + public static final String DGH1 = "dg_hier_level_1"; + public static final String DGH2 = "dg_hier_level_2"; + public static final String DGH3 = "dg_hier_level_3"; + public static final String DGH4 = "dg_hier_level_4"; + public static final String VSYSName = "vsys_name"; + public static final String DeviceName = "device_name"; + public static final String ActionSource = "action_source"; + public static final String ParserVersion = "parser_version"; + public static final String Tokens = "tokens_seen"; + + public static final String SourceVmUuid = "source_vm_uuid"; + public static final String DestinationVmUuid = "destination_vm_uuid"; + public static final String TunnelId = "tunnel_id"; + public static final String MonitorTag = "monitor_tag"; + public static final String ParentSessionId = "parent_session_id"; + public static final String ParentSessionStartTime = "parent_session_start_time"; + public static final String TunnelType = "tunnel_type"; //Threat public static final String URL = "url"; public static final String HOST = "host"; - public static final String ThreatContentName = "threat_content_name"; - public static final String Category = "category"; + public static final String ThreatID = "threatid"; + public static final String Severity = "severity"; public static final String Direction = "direction"; - public static final String Seqno = "seqno"; - public static final String ActionFlags = "action_flags"; - public static final String SourceCountry = "source_country"; - public static final String DestinationCountry = "destination_country"; - public static final String Cpadding = "cpadding"; - public static final String ContentType = "content_type"; + public static final String SourceLocation = "srcloc"; + public static final String DestinationLocation = "dstloc"; + public static final String ContentType = "contenttype"; + public static final String PCAPID = "pcap_id"; + public static final String WFFileDigest = "filedigest"; + public static final String WFCloud = "cloud"; + public static final String UserAgent= "user_agent"; + public static final String WFFileType = "filetype"; + public static final String XForwardedFor = "xff"; + public static final String Referer = "referer"; + public static final String WFSender = "sender"; + public static final String WFSubject = "subject"; + public static final String WFRecipient = "recipient"; + public static final String WFReportID = "reportid"; + public static final String URLIndex = "url_idx"; + public static final String HTTPMethod = "http_method"; + public static final String ThreatCategory = "threat_category"; + public static final String ContentVersion = "content_version"; + //Traffic - public static final String Bytes = "content_type"; - public static final String BytesSent = "content_type"; - public static final String BytesReceived = "content_type"; - public static final String Packets = "content_type"; - public static final String StartTime = "content_type"; - public static final String ElapsedTimeInSec = "content_type"; - public static final String Padding = "content_type"; + public static final String Bytes = "bytes"; + public static final String BytesSent = "bytes_sent"; + public static final String BytesReceived = "bytes_received"; + public static final String Packets = "packets"; + public static final String StartTime = "start"; + public static final String ElapsedTimeInSec = "elapsed"; public static final String PktsSent = "pkts_sent"; public static final String PktsReceived = "pkts_received"; + public static final String EndReason = "session_end_reason"; @Override public void configure(Map<String, Object> parserConfig) { @@ -117,12 +164,6 @@ public class BasicPaloAltoFirewallParser extends BasicParser { parseMessage(toParse, outputMessage); long timestamp = System.currentTimeMillis(); outputMessage.put("timestamp", System.currentTimeMillis()); - outputMessage.put("ip_src_addr", outputMessage.remove("source_address")); - outputMessage.put("ip_src_port", outputMessage.remove("source_port")); - outputMessage.put("ip_dst_addr", outputMessage.remove("destination_address")); - outputMessage.put("ip_dst_port", outputMessage.remove("destination_port")); - outputMessage.put("protocol", outputMessage.remove("ip_protocol")); - outputMessage.put("original_string", toParse); messages.add(outputMessage); return messages; @@ -136,77 +177,157 @@ public class BasicPaloAltoFirewallParser extends BasicParser { @SuppressWarnings("unchecked") private void parseMessage(String message, JSONObject outputMessage) { - String[] tokens = message.split(","); + String[] tokens = Iterables.toArray(Splitter.on(Pattern.compile(",(?=(?:[^\"]*\"[^\"]*\")*[^\"]*$)")).split(message), String.class); + int parser_version = 0; String type = tokens[3].trim(); //populate common objects - outputMessage.put(PaloAltoDomain, tokens[0].trim()); - outputMessage.put(ReceiveTime, tokens[1].trim()); - outputMessage.put(SerialNum, tokens[2].trim()); + if( !empty_attribute( tokens[0] ) ) outputMessage.put(PaloAltoDomain, tokens[0].trim()); + if( !empty_attribute( tokens[1] ) ) outputMessage.put(ReceiveTime, tokens[1].trim()); + if( !empty_attribute( tokens[2] ) ) outputMessage.put(SerialNum, tokens[2].trim()); outputMessage.put(Type, type); - outputMessage.put(ThreatContentType, tokens[4].trim()); - outputMessage.put(ConfigVersion, tokens[5].trim()); - outputMessage.put(GenerateTime, tokens[6].trim()); - outputMessage.put(SourceAddress, tokens[7].trim()); - outputMessage.put(DestinationAddress, tokens[8].trim()); - outputMessage.put(NATSourceIP, tokens[9].trim()); - outputMessage.put(NATDestinationIP, tokens[10].trim()); - outputMessage.put(Rule, tokens[11].trim()); - outputMessage.put(SourceUser, tokens[12].trim()); - outputMessage.put(DestinationUser, tokens[13].trim()); - outputMessage.put(Application, tokens[14].trim()); - outputMessage.put(VirtualSystem, tokens[15].trim()); - outputMessage.put(SourceZone, tokens[16].trim()); - outputMessage.put(DestinationZone, tokens[17].trim()); - outputMessage.put(InboundInterface, tokens[18].trim()); - outputMessage.put(OutboundInterface, tokens[19].trim()); - outputMessage.put(LogAction, tokens[20].trim()); - outputMessage.put(TimeLogged, tokens[21].trim()); - outputMessage.put(SessionID, tokens[22].trim()); - outputMessage.put(RepeatCount, tokens[23].trim()); - outputMessage.put(SourcePort, tokens[24].trim()); - outputMessage.put(DestinationPort, tokens[25].trim()); - outputMessage.put(NATSourcePort, tokens[26].trim()); - outputMessage.put(NATDestinationPort, tokens[27].trim()); - outputMessage.put(Flags, tokens[28].trim()); - outputMessage.put(IPProtocol, tokens[29].trim()); - outputMessage.put(Action, tokens[30].trim()); + if( !empty_attribute( tokens[4] ) ) outputMessage.put(ThreatContentType, unquoted_attribute(tokens[4])); + if( !empty_attribute( tokens[5] ) ) outputMessage.put(ConfigVersion, tokens[5].trim()); + if( !empty_attribute( tokens[6] ) ) outputMessage.put(GenerateTime, tokens[6].trim()); + if( !empty_attribute( tokens[7] ) ) outputMessage.put(SourceAddress, tokens[7].trim()); + if( !empty_attribute( tokens[8] ) ) outputMessage.put(DestinationAddress, tokens[8].trim()); + if( !empty_attribute( tokens[9] ) ) outputMessage.put(NATSourceIP, tokens[9].trim()); + if( !empty_attribute( tokens[10] ) ) outputMessage.put(NATDestinationIP, tokens[10].trim()); + if( !empty_attribute( tokens[11] ) ) outputMessage.put(Rule, unquoted_attribute(tokens[11])); + if( !empty_attribute( tokens[12] ) ) outputMessage.put(SourceUser, unquoted_attribute(tokens[12])); + if( !empty_attribute( tokens[13] ) ) outputMessage.put(DestinationUser, unquoted_attribute(tokens[13])); + if( !empty_attribute( tokens[14] ) ) outputMessage.put(Application, unquoted_attribute(tokens[14])); + if( !empty_attribute( tokens[15] ) ) outputMessage.put(VirtualSystem, unquoted_attribute(tokens[15])); + if( !empty_attribute( tokens[16] ) ) outputMessage.put(SourceZone, unquoted_attribute(tokens[16])); + if( !empty_attribute( tokens[17] ) ) outputMessage.put(DestinationZone, unquoted_attribute(tokens[17])); + if( !empty_attribute( tokens[18] ) ) outputMessage.put(InboundInterface, unquoted_attribute(tokens[18])); + if( !empty_attribute( tokens[19] ) ) outputMessage.put(OutboundInterface, unquoted_attribute(tokens[19])); + if( !empty_attribute( tokens[20] ) ) outputMessage.put(LogAction, unquoted_attribute(tokens[20])); + if( !empty_attribute( tokens[21] ) ) outputMessage.put(TimeLogged, tokens[21].trim()); + if( !empty_attribute( tokens[22] ) ) outputMessage.put(SessionID, tokens[22].trim()); + if( !empty_attribute( tokens[23] ) ) outputMessage.put(RepeatCount, tokens[23].trim()); + if( !empty_attribute( tokens[24] ) ) outputMessage.put(SourcePort, tokens[24].trim()); + if( !empty_attribute( tokens[25] ) ) outputMessage.put(DestinationPort, tokens[25].trim()); + if( !empty_attribute( tokens[26] ) ) outputMessage.put(NATSourcePort, tokens[26].trim()); + if( !empty_attribute( tokens[27] ) ) outputMessage.put(NATDestinationPort, tokens[27].trim()); + if( !empty_attribute( tokens[28] ) ) outputMessage.put(Flags, tokens[28].trim()); + if( !empty_attribute( tokens[29] ) ) outputMessage.put(IPProtocol, unquoted_attribute(tokens[29])); + if( !empty_attribute( tokens[30] ) ) outputMessage.put(Action, unquoted_attribute(tokens[30])); if ("THREAT".equals(type.toUpperCase())) { - outputMessage.put(URL, tokens[31].trim()); - try { - URL url = new URL(tokens[31].trim()); - outputMessage.put(HOST, url.getHost()); - } catch (MalformedURLException e) { + int p1_offset = 0; + if (tokens.length == 45) parser_version = 60; + else if (tokens.length == 53) parser_version = 61; + else if (tokens.length == 61) { + parser_version = 70; + p1_offset = 1; + } + else if (tokens.length == 72) { + parser_version = 80; + p1_offset =1; + } + outputMessage.put(ParserVersion, parser_version); + if( !empty_attribute( tokens[31] ) ) { + outputMessage.put(URL, unquoted_attribute(tokens[31])); + try { + URL url = new URL(unquoted_attribute(tokens[31])); + outputMessage.put(HOST, url.getHost()); + } catch (MalformedURLException e) { + } + } + if( !empty_attribute( tokens[32] ) ) outputMessage.put(ThreatID, tokens[32].trim()); + if( !empty_attribute( tokens[33] ) ) outputMessage.put(Category, unquoted_attribute(tokens[33])); + if( !empty_attribute( tokens[34] ) ) outputMessage.put(Severity, unquoted_attribute(tokens[34])); + if( !empty_attribute( tokens[35] ) ) outputMessage.put(Direction, unquoted_attribute(tokens[35])); + if( !empty_attribute( tokens[36] ) ) outputMessage.put(Seqno, tokens[36].trim()); + if( !empty_attribute( tokens[37] ) ) outputMessage.put(ActionFlags, unquoted_attribute(tokens[37])); + if( !empty_attribute( tokens[38] ) ) outputMessage.put(SourceLocation, unquoted_attribute(tokens[38])); + if( !empty_attribute( tokens[39] ) ) outputMessage.put(DestinationLocation, unquoted_attribute(tokens[39])); + if( !empty_attribute( tokens[41] ) ) outputMessage.put(ContentType, unquoted_attribute(tokens[41])); + if( !empty_attribute( tokens[42] ) ) outputMessage.put(PCAPID, tokens[42].trim()); + if( !empty_attribute( tokens[43] ) ) outputMessage.put(WFFileDigest, unquoted_attribute(tokens[43])); + if( !empty_attribute( tokens[44] ) ) outputMessage.put(WFCloud, unquoted_attribute(tokens[44])); + if ( parser_version >= 61) { + if( !empty_attribute( tokens[(45 + p1_offset)] ) ) outputMessage.put(UserAgent, unquoted_attribute(tokens[(45 + p1_offset)])); + if( !empty_attribute( tokens[(46 + p1_offset)] ) ) outputMessage.put(WFFileType, unquoted_attribute(tokens[(46 + p1_offset)])); + if( !empty_attribute( tokens[(47 + p1_offset)] ) ) outputMessage.put(XForwardedFor, unquoted_attribute(tokens[(47 + p1_offset)])); + if( !empty_attribute( tokens[(48 + p1_offset)] ) ) outputMessage.put(Referer, unquoted_attribute(tokens[(48 + p1_offset)])); + if( !empty_attribute( tokens[(49 + p1_offset)] ) ) outputMessage.put(WFSender, unquoted_attribute(tokens[(49 + p1_offset)])); + if( !empty_attribute( tokens[(50 + p1_offset)] ) ) outputMessage.put(WFSubject, unquoted_attribute(tokens[(50 + p1_offset)])); + if( !empty_attribute( tokens[(51 + p1_offset)] ) ) outputMessage.put(WFRecipient, unquoted_attribute(tokens[(51 + p1_offset)])); + if( !empty_attribute( tokens[(52 + p1_offset)] ) ) outputMessage.put(WFReportID, unquoted_attribute(tokens[(52 + p1_offset)])); + } + if ( parser_version >= 70) { + if( !empty_attribute( tokens[45] ) ) outputMessage.put(URLIndex, tokens[45].trim()); + if( !empty_attribute( tokens[54] ) ) outputMessage.put(DGH1, tokens[54].trim()); + if( !empty_attribute( tokens[55] ) ) outputMessage.put(DGH2, tokens[55].trim()); + if( !empty_attribute( tokens[56] ) ) outputMessage.put(DGH3, tokens[56].trim()); + if( !empty_attribute( tokens[57] ) ) outputMessage.put(DGH4, tokens[57].trim()); + if( !empty_attribute( tokens[58] ) ) outputMessage.put(VSYSName, unquoted_attribute(tokens[58])); + if( !empty_attribute( tokens[59] ) ) outputMessage.put(DeviceName, unquoted_attribute(tokens[59])); + } + if ( parser_version >= 80) { + if( !empty_attribute( tokens[61] ) ) outputMessage.put(SourceVmUuid, tokens[61].trim()); + if( !empty_attribute( tokens[62] ) ) outputMessage.put(DestinationVmUuid, tokens[62].trim()); + if( !empty_attribute( tokens[63] ) ) outputMessage.put(HTTPMethod, tokens[63].trim()); + if( !empty_attribute( tokens[64] ) ) outputMessage.put(TunnelId, tokens[64].trim()); + if( !empty_attribute( tokens[65] ) ) outputMessage.put(MonitorTag, tokens[65].trim()); + if( !empty_attribute( tokens[66] ) ) outputMessage.put(ParentSessionId, tokens[66].trim()); + if( !empty_attribute( tokens[67] ) ) outputMessage.put(ParentSessionStartTime, tokens[67].trim()); + if( !empty_attribute( tokens[68] ) ) outputMessage.put(TunnelType, tokens[68].trim()); + if( !empty_attribute( tokens[69] ) ) outputMessage.put(ThreatCategory, tokens[69].trim()); + if( !empty_attribute( tokens[70] ) ) outputMessage.put(ContentVersion, tokens[70].trim()); + } + if ( parser_version == 0) { + outputMessage.put(Tokens, tokens.length); + } + + + } else if ("TRAFFIC".equals(type.toUpperCase())) { + if (tokens.length == 46) parser_version = 60; + else if (tokens.length == 47) parser_version = 61; + else if (tokens.length == 54) parser_version = 70; + else if (tokens.length == 61) parser_version = 80; + outputMessage.put(ParserVersion, parser_version); + if( !empty_attribute( tokens[31] ) ) outputMessage.put(Bytes, tokens[31].trim()); + if( !empty_attribute( tokens[32] ) ) outputMessage.put(BytesSent, tokens[32].trim()); + if( !empty_attribute( tokens[33] ) ) outputMessage.put(BytesReceived, tokens[33].trim()); + if( !empty_attribute( tokens[34] ) ) outputMessage.put(Packets, tokens[34].trim()); + if( !empty_attribute( tokens[35] ) ) outputMessage.put(StartTime, tokens[35].trim()); + if( !empty_attribute( tokens[36] ) ) outputMessage.put(ElapsedTimeInSec, tokens[36].trim()); + if( !empty_attribute( tokens[37] ) ) outputMessage.put(Category, unquoted_attribute(tokens[37])); + if( !empty_attribute( tokens[39] ) ) outputMessage.put(Seqno, tokens[39].trim()); + if( !empty_attribute( tokens[40] ) ) outputMessage.put(ActionFlags, unquoted_attribute(tokens[40])); + if( !empty_attribute( tokens[41] ) ) outputMessage.put(SourceLocation, unquoted_attribute(tokens[41])); + if( !empty_attribute( tokens[42] ) ) outputMessage.put(DestinationLocation, unquoted_attribute(tokens[42])); + if( !empty_attribute( tokens[44] ) ) outputMessage.put(PktsSent, tokens[44].trim()); + if( !empty_attribute( tokens[45] ) ) outputMessage.put(PktsReceived, tokens[45].trim()); + if ( parser_version >= 61) { + if( !empty_attribute( tokens[46] ) ) outputMessage.put(EndReason, unquoted_attribute(tokens[46])); + } + if ( parser_version >= 70) { + if( !empty_attribute( tokens[47] ) ) outputMessage.put(DGH1, tokens[47].trim()); + if( !empty_attribute( tokens[48] ) ) outputMessage.put(DGH2, tokens[48].trim()); + if( !empty_attribute( tokens[49] ) ) outputMessage.put(DGH3, tokens[49].trim()); + if( !empty_attribute( tokens[50] ) ) outputMessage.put(DGH4, tokens[50].trim()); + if( !empty_attribute( tokens[51] ) ) outputMessage.put(VSYSName, unquoted_attribute(tokens[51])); + if( !empty_attribute( tokens[52] ) ) outputMessage.put(DeviceName, unquoted_attribute(tokens[52])); + if( !empty_attribute( tokens[53] ) ) outputMessage.put(ActionSource, unquoted_attribute(tokens[53])); + } + if ( parser_version >= 80) { + if( !empty_attribute( tokens[54] ) ) outputMessage.put(SourceVmUuid, tokens[54].trim()); + if( !empty_attribute( tokens[55] ) ) outputMessage.put(DestinationVmUuid, tokens[55].trim()); + if( !empty_attribute( tokens[56] ) ) outputMessage.put(TunnelId, tokens[56].trim()); + if( !empty_attribute( tokens[57] ) ) outputMessage.put(MonitorTag, tokens[57].trim()); + if( !empty_attribute( tokens[58] ) ) outputMessage.put(ParentSessionId, tokens[58].trim()); + if( !empty_attribute( tokens[59] ) ) outputMessage.put(ParentSessionStartTime, tokens[59].trim()); + if( !empty_attribute( tokens[60] ) ) outputMessage.put(TunnelType, tokens[60].trim()); + } + if ( parser_version == 0) { + outputMessage.put(Tokens, tokens.length); } - outputMessage.put(ThreatContentName, tokens[32].trim()); - outputMessage.put(Category, tokens[33].trim()); - outputMessage.put(Direction, tokens[34].trim()); - outputMessage.put(Seqno, tokens[35].trim()); - outputMessage.put(ActionFlags, tokens[36].trim()); - outputMessage.put(SourceCountry, tokens[37].trim()); - outputMessage.put(DestinationCountry, tokens[38].trim()); - outputMessage.put(Cpadding, tokens[39].trim()); - outputMessage.put(ContentType, tokens[40].trim()); - - } else { - outputMessage.put(Bytes, tokens[31].trim()); - outputMessage.put(BytesSent, tokens[32].trim()); - outputMessage.put(BytesReceived, tokens[33].trim()); - outputMessage.put(Packets, tokens[34].trim()); - outputMessage.put(StartTime, tokens[35].trim()); - outputMessage.put(ElapsedTimeInSec, tokens[36].trim()); - outputMessage.put(Category, tokens[37].trim()); - outputMessage.put(Padding, tokens[38].trim()); - outputMessage.put(Seqno, tokens[39].trim()); - outputMessage.put(ActionFlags, tokens[40].trim()); - outputMessage.put(SourceCountry, tokens[41].trim()); - outputMessage.put(DestinationCountry, tokens[42].trim()); - outputMessage.put(Cpadding, tokens[43].trim()); - outputMessage.put(PktsSent, tokens[44].trim()); - outputMessage.put(PktsReceived, tokens[45].trim()); } } http://git-wip-us.apache.org/repos/asf/metron/blob/5f08ba0b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParserTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParserTest.java index cf93c92..2c90b1e 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParserTest.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParserTest.java @@ -17,13 +17,11 @@ */ package org.apache.metron.parsers.paloalto; -import java.util.Map; -import java.util.Map.Entry; +import static org.junit.Assert.assertEquals; + import org.apache.metron.parsers.AbstractParserConfigTest; import org.json.simple.JSONObject; -import org.json.simple.parser.JSONParser; import org.json.simple.parser.ParseException; -import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -31,27 +29,482 @@ public class BasicPaloAltoFirewallParserTest extends AbstractParserConfigTest { @Before public void setUp() throws Exception { - inputStrings = readTestDataFromFile( - "src/test/resources/logData/PaloAltoFirewallParserTest.txt"); parser = new BasicPaloAltoFirewallParser(); } - @SuppressWarnings({"rawtypes"}) + public static final String THREAT_60 = "1,2015/01/05 05:38:58,0006C110285,THREAT,vulnerability,1,2015/01/05 05:38:58,10.0.0.115,216.0.10.198,0.0.0.0,0.0.0.0,EX-Allow,example\\user.name,,web-browsing,vsys1,internal,external,ethernet1/2,ethernet1/1,LOG-Default,2015/01/05 05:38:58,12031,1,54180,80,0,0,0x80004000,tcp,reset-both,\"ad.aspx?f=300x250&id=12;tile=1;ord=67AF705D60B1119C0F18BEA336F9\",HTTP: IIS Denial Of Service Attempt(40019),any,high,client-to-server,347368099,0x0,10.0.0.0-10.255.255.255,US,0,,1200568889751109656,,"; + + @SuppressWarnings("unchecked") + @Test + public void testParseThreat60() throws ParseException { + JSONObject actual = parser.parse(THREAT_60.getBytes()).get(0); + + JSONObject expected = new JSONObject(); + expected.put(BasicPaloAltoFirewallParser.Action, "reset-both"); + expected.put(BasicPaloAltoFirewallParser.ActionFlags, "0x0"); + expected.put(BasicPaloAltoFirewallParser.Application, "web-browsing"); + expected.put(BasicPaloAltoFirewallParser.Category, "any"); + + expected.put(BasicPaloAltoFirewallParser.ConfigVersion, "1"); + expected.put(BasicPaloAltoFirewallParser.Direction, "client-to-server"); + expected.put(BasicPaloAltoFirewallParser.DestinationLocation, "US"); + expected.put(BasicPaloAltoFirewallParser.Flags, "0x80004000"); + expected.put(BasicPaloAltoFirewallParser.SourceZone, "internal"); + expected.put(BasicPaloAltoFirewallParser.InboundInterface, "ethernet1/2"); + expected.put(BasicPaloAltoFirewallParser.DestinationAddress, "216.0.10.198"); + expected.put(BasicPaloAltoFirewallParser.DestinationPort, "80"); + expected.put(BasicPaloAltoFirewallParser.SourceAddress, "10.0.0.115"); + expected.put(BasicPaloAltoFirewallParser.SourcePort, "54180"); + expected.put(BasicPaloAltoFirewallParser.LogAction, "LOG-Default"); + expected.put(BasicPaloAltoFirewallParser.NATDestinationPort, "0"); + expected.put(BasicPaloAltoFirewallParser.NATDestinationIP, "0.0.0.0"); + expected.put(BasicPaloAltoFirewallParser.NATSourcePort, "0"); + expected.put(BasicPaloAltoFirewallParser.NATSourceIP, "0.0.0.0"); + expected.put("original_string", THREAT_60); + expected.put(BasicPaloAltoFirewallParser.OutboundInterface, "ethernet1/1"); + expected.put(BasicPaloAltoFirewallParser.PaloAltoDomain, "1"); + expected.put(BasicPaloAltoFirewallParser.ParserVersion, 60); + expected.put(BasicPaloAltoFirewallParser.PCAPID, "1200568889751109656"); + expected.put(BasicPaloAltoFirewallParser.IPProtocol, "tcp"); + expected.put(BasicPaloAltoFirewallParser.ReceiveTime, "2015/01/05 05:38:58"); + expected.put(BasicPaloAltoFirewallParser.RepeatCount, "1"); + expected.put(BasicPaloAltoFirewallParser.Rule, "EX-Allow"); + expected.put(BasicPaloAltoFirewallParser.Seqno, "347368099"); + expected.put(BasicPaloAltoFirewallParser.SerialNum, "0006C110285"); + expected.put(BasicPaloAltoFirewallParser.SessionID, "12031"); + expected.put(BasicPaloAltoFirewallParser.Severity, "high"); + expected.put(BasicPaloAltoFirewallParser.SourceLocation, "10.0.0.0-10.255.255.255"); + expected.put(BasicPaloAltoFirewallParser.SourceUser, "example\\user.name"); + expected.put(BasicPaloAltoFirewallParser.StartTime, "2015/01/05 05:38:58"); + expected.put(BasicPaloAltoFirewallParser.ThreatContentType, "vulnerability"); + expected.put(BasicPaloAltoFirewallParser.ThreatID, "HTTP: IIS Denial Of Service Attempt(40019)"); + expected.put(BasicPaloAltoFirewallParser.GenerateTime, "2015/01/05 05:38:58"); + expected.put("timestamp", actual.get("timestamp")); + expected.put(BasicPaloAltoFirewallParser.DestinationZone, "external"); + expected.put(BasicPaloAltoFirewallParser.Type, "THREAT"); + expected.put(BasicPaloAltoFirewallParser.URL, "ad.aspx?f=300x250&id=12;tile=1;ord=67AF705D60B1119C0F18BEA336F9"); + expected.put(BasicPaloAltoFirewallParser.VirtualSystem, "vsys1"); + assertEquals(expected, actual); + } + + public static final String TRAFFIC_60 = "1,2015/01/05 12:51:33,0011C103117,TRAFFIC,end,1,2015/01/05 12:51:33,10.0.0.39,10.1.0.163,0.0.0.0,0.0.0.0,EX-Allow,,example\\\\user.name,ms-ds-smb,vsys1,v_external,v_internal,ethernet1/2,ethernet1/1,LOG-Default,2015/01/05 12:51:33,33760927,1,52688,445,0,0,0x401a,tcp,allow,2229,1287,942,10,2015/01/05 12:51:01,30,any,0,17754932062,0x0,10.0.0.0-10.255.255.255,10.0.0.0-10.255.255.255,0,6,"; + @SuppressWarnings("unchecked") + @Test + public void testParseTraffic60() throws ParseException { + JSONObject actual = parser.parse(TRAFFIC_60.getBytes()).get(0); + + JSONObject expected = new JSONObject(); + expected.put(BasicPaloAltoFirewallParser.Action, "allow"); + expected.put(BasicPaloAltoFirewallParser.ActionFlags, "0x0"); + expected.put(BasicPaloAltoFirewallParser.Application, "ms-ds-smb"); + expected.put(BasicPaloAltoFirewallParser.Bytes, "2229"); + expected.put(BasicPaloAltoFirewallParser.BytesReceived, "942"); + expected.put(BasicPaloAltoFirewallParser.BytesSent, "1287"); + expected.put(BasicPaloAltoFirewallParser.Category, "any"); + expected.put(BasicPaloAltoFirewallParser.ConfigVersion, "1"); + expected.put(BasicPaloAltoFirewallParser.DestinationLocation, "10.0.0.0-10.255.255.255"); + expected.put(BasicPaloAltoFirewallParser.DestinationUser, "example\\\\user.name"); + expected.put(BasicPaloAltoFirewallParser.ElapsedTimeInSec, "30"); + expected.put(BasicPaloAltoFirewallParser.Flags, "0x401a"); + expected.put(BasicPaloAltoFirewallParser.SourceZone, "v_external"); + expected.put(BasicPaloAltoFirewallParser.InboundInterface, "ethernet1/2"); + expected.put(BasicPaloAltoFirewallParser.DestinationAddress, "10.1.0.163"); + expected.put(BasicPaloAltoFirewallParser.DestinationPort, "445"); + expected.put(BasicPaloAltoFirewallParser.SourceAddress, "10.0.0.39"); + expected.put(BasicPaloAltoFirewallParser.SourcePort, "52688"); + expected.put(BasicPaloAltoFirewallParser.LogAction, "LOG-Default"); + expected.put(BasicPaloAltoFirewallParser.NATDestinationPort, "0"); + expected.put(BasicPaloAltoFirewallParser.NATDestinationIP, "0.0.0.0"); + expected.put(BasicPaloAltoFirewallParser.NATSourcePort, "0"); + expected.put(BasicPaloAltoFirewallParser.NATSourceIP, "0.0.0.0"); + expected.put("original_string", TRAFFIC_60); + expected.put(BasicPaloAltoFirewallParser.OutboundInterface, "ethernet1/1"); + expected.put(BasicPaloAltoFirewallParser.Packets, "10"); + expected.put(BasicPaloAltoFirewallParser.PaloAltoDomain, "1"); + expected.put(BasicPaloAltoFirewallParser.ParserVersion, 60); + expected.put(BasicPaloAltoFirewallParser.PktsSent, "6"); + expected.put(BasicPaloAltoFirewallParser.IPProtocol, "tcp"); + expected.put(BasicPaloAltoFirewallParser.ReceiveTime, "2015/01/05 12:51:33"); + expected.put(BasicPaloAltoFirewallParser.RepeatCount, "1"); + expected.put(BasicPaloAltoFirewallParser.Rule, "EX-Allow"); + expected.put(BasicPaloAltoFirewallParser.Seqno, "17754932062"); + expected.put(BasicPaloAltoFirewallParser.SerialNum, "0011C103117"); + expected.put(BasicPaloAltoFirewallParser.SessionID, "33760927"); + expected.put(BasicPaloAltoFirewallParser.SourceLocation, "10.0.0.0-10.255.255.255"); + expected.put(BasicPaloAltoFirewallParser.StartTime, "2015/01/05 12:51:01"); + expected.put(BasicPaloAltoFirewallParser.ThreatContentType, "end"); + expected.put(BasicPaloAltoFirewallParser.GenerateTime, "2015/01/05 12:51:33"); + expected.put("timestamp", actual.get("timestamp")); + expected.put(BasicPaloAltoFirewallParser.DestinationZone, "v_internal"); + expected.put(BasicPaloAltoFirewallParser.Type, "TRAFFIC"); + expected.put(BasicPaloAltoFirewallParser.VirtualSystem, "vsys1"); + assertEquals(expected, actual); + } + + public static final String THREAT_70 = "1,2017/05/24 09:53:10,001801000001,THREAT,virus,0,2017/05/24 09:53:10,217.1.2.3,10.1.8.7,217.1.2.3,214.123.1.2,WLAN-Internet,,user,web-browsing,vsys1,Untrust,wifi_zone,ethernet1/1,vlan.1,Std-Log-Forward,2017/05/24 09:53:10,49567,1,80,51787,80,25025,0x400000,tcp,reset-both,\"abcdef310.exe\",Virus/Win32.WGeneric.lumeo(2457399),computer-and-internet-info,medium,server-to-client,329423829,0x0,DE,10.0.0.0-10.255.255.255,0,,0,,,1,,,\"\",\"\",,,,0,19,0,0,0,,PAN1,"; + @SuppressWarnings("unchecked") + @Test + public void testParseThreat70() throws ParseException { + JSONObject actual = parser.parse(THREAT_70.getBytes()).get(0); + + JSONObject expected = new JSONObject(); + expected.put(BasicPaloAltoFirewallParser.Action, "reset-both"); + expected.put(BasicPaloAltoFirewallParser.ActionFlags, "0x0"); + expected.put(BasicPaloAltoFirewallParser.Application, "web-browsing"); + expected.put(BasicPaloAltoFirewallParser.Category, "computer-and-internet-info"); + expected.put(BasicPaloAltoFirewallParser.ConfigVersion, "0"); + expected.put(BasicPaloAltoFirewallParser.Direction, "server-to-client"); + expected.put(BasicPaloAltoFirewallParser.DestinationLocation, "10.0.0.0-10.255.255.255"); + expected.put(BasicPaloAltoFirewallParser.DestinationUser, "user"); + expected.put(BasicPaloAltoFirewallParser.Flags, "0x400000"); + expected.put(BasicPaloAltoFirewallParser.SourceZone, "Untrust"); + expected.put(BasicPaloAltoFirewallParser.InboundInterface, "ethernet1/1"); + expected.put(BasicPaloAltoFirewallParser.DestinationAddress, "10.1.8.7"); + expected.put(BasicPaloAltoFirewallParser.DestinationPort, "51787"); + expected.put(BasicPaloAltoFirewallParser.SourceAddress, "217.1.2.3"); + expected.put(BasicPaloAltoFirewallParser.SourcePort, "80"); + expected.put(BasicPaloAltoFirewallParser.LogAction, "Std-Log-Forward"); + expected.put(BasicPaloAltoFirewallParser.NATDestinationPort, "25025"); + expected.put(BasicPaloAltoFirewallParser.NATDestinationIP, "214.123.1.2"); + expected.put(BasicPaloAltoFirewallParser.NATSourcePort, "80"); + expected.put(BasicPaloAltoFirewallParser.NATSourceIP, "217.1.2.3"); + expected.put("original_string", THREAT_70); + expected.put(BasicPaloAltoFirewallParser.OutboundInterface, "vlan.1"); + expected.put(BasicPaloAltoFirewallParser.PaloAltoDomain, "1"); + expected.put(BasicPaloAltoFirewallParser.ParserVersion, 70); + expected.put(BasicPaloAltoFirewallParser.PCAPID, "0"); + expected.put(BasicPaloAltoFirewallParser.IPProtocol, "tcp"); + expected.put(BasicPaloAltoFirewallParser.ReceiveTime, "2017/05/24 09:53:10"); + expected.put(BasicPaloAltoFirewallParser.RepeatCount, "1"); + expected.put(BasicPaloAltoFirewallParser.Rule, "WLAN-Internet"); + expected.put(BasicPaloAltoFirewallParser.Seqno, "329423829"); + expected.put(BasicPaloAltoFirewallParser.SerialNum, "001801000001"); + expected.put(BasicPaloAltoFirewallParser.SessionID, "49567"); + expected.put(BasicPaloAltoFirewallParser.Severity, "medium"); + expected.put(BasicPaloAltoFirewallParser.SourceLocation, "DE"); + expected.put(BasicPaloAltoFirewallParser.StartTime, "2017/05/24 09:53:10"); + expected.put(BasicPaloAltoFirewallParser.ThreatContentType, "virus"); + expected.put(BasicPaloAltoFirewallParser.ThreatID, "Virus/Win32.WGeneric.lumeo(2457399)"); + expected.put(BasicPaloAltoFirewallParser.GenerateTime, "2017/05/24 09:53:10"); + expected.put("timestamp", actual.get("timestamp")); + expected.put(BasicPaloAltoFirewallParser.DestinationZone, "wifi_zone"); + expected.put(BasicPaloAltoFirewallParser.Type, "THREAT"); + expected.put(BasicPaloAltoFirewallParser.URL, "abcdef310.exe"); + expected.put(BasicPaloAltoFirewallParser.VirtualSystem, "vsys1"); + expected.put(BasicPaloAltoFirewallParser.URLIndex, "1"); + expected.put(BasicPaloAltoFirewallParser.WFReportID, "0"); + expected.put(BasicPaloAltoFirewallParser.DGH1, "19"); + expected.put(BasicPaloAltoFirewallParser.DGH2, "0"); + expected.put(BasicPaloAltoFirewallParser.DGH3, "0"); + expected.put(BasicPaloAltoFirewallParser.DGH4, "0"); + expected.put(BasicPaloAltoFirewallParser.DeviceName, "PAN1"); + assertEquals(expected, actual); + } + + public static final String TRAFFIC_70 = "1,2017/05/25 21:38:13,001606000003,TRAFFIC,drop,1,2017/05/25 21:38:13,10.2.1.8,192.168.1.10,0.0.0.0,0.0.0.0,DropLog,,,not-applicable,vsys1,intern,VPN,vlan.1,,Std-Log-Forward,2017/05/25 21:38:13,0,1,137,137,0,0,0x0,udp,deny,114,114,0,1,2017/05/25 21:38:12,0,any,0,9953744,0x0,192.168.0.0-192.168.255.255,DE,0,1,0,policy-deny,19,0,0,0,,PAN1,from-policy"; + @SuppressWarnings("unchecked") @Test - public void testParse() throws ParseException { - for (String inputString : inputStrings) { - JSONObject parsed = parser.parse(inputString.getBytes()).get(0); - Assert.assertNotNull(parsed); + public void testParseTraffic70() throws ParseException { + JSONObject actual = parser.parse(TRAFFIC_70.getBytes()).get(0); - JSONParser parser = new JSONParser(); - Map json = (Map) parser.parse(parsed.toJSONString()); + JSONObject expected = new JSONObject(); + expected.put(BasicPaloAltoFirewallParser.Action, "deny"); + expected.put(BasicPaloAltoFirewallParser.ActionFlags, "0x0"); + expected.put(BasicPaloAltoFirewallParser.ActionSource, "from-policy"); + expected.put(BasicPaloAltoFirewallParser.Application, "not-applicable"); + expected.put(BasicPaloAltoFirewallParser.Bytes, "114"); + expected.put(BasicPaloAltoFirewallParser.BytesReceived, "0"); + expected.put(BasicPaloAltoFirewallParser.BytesSent, "114"); + expected.put(BasicPaloAltoFirewallParser.Category, "any"); + expected.put(BasicPaloAltoFirewallParser.ConfigVersion, "1"); + expected.put(BasicPaloAltoFirewallParser.DestinationLocation, "DE"); + expected.put(BasicPaloAltoFirewallParser.ElapsedTimeInSec, "0"); + expected.put(BasicPaloAltoFirewallParser.Flags, "0x0"); + expected.put(BasicPaloAltoFirewallParser.SourceZone, "intern"); + expected.put(BasicPaloAltoFirewallParser.InboundInterface, "vlan.1"); + expected.put(BasicPaloAltoFirewallParser.DestinationAddress, "192.168.1.10"); + expected.put(BasicPaloAltoFirewallParser.DestinationPort, "137"); + expected.put(BasicPaloAltoFirewallParser.SourceAddress, "10.2.1.8"); + expected.put(BasicPaloAltoFirewallParser.SourcePort, "137"); + expected.put(BasicPaloAltoFirewallParser.LogAction, "Std-Log-Forward"); + expected.put(BasicPaloAltoFirewallParser.NATDestinationPort, "0"); + expected.put(BasicPaloAltoFirewallParser.NATDestinationIP, "0.0.0.0"); + expected.put(BasicPaloAltoFirewallParser.NATSourcePort, "0"); + expected.put(BasicPaloAltoFirewallParser.NATSourceIP, "0.0.0.0"); + expected.put("original_string", TRAFFIC_70); + expected.put(BasicPaloAltoFirewallParser.Packets, "1"); + expected.put(BasicPaloAltoFirewallParser.PaloAltoDomain, "1"); + expected.put(BasicPaloAltoFirewallParser.ParserVersion, 70); + expected.put(BasicPaloAltoFirewallParser.PktsReceived, "0"); + expected.put(BasicPaloAltoFirewallParser.PktsSent, "1"); + expected.put(BasicPaloAltoFirewallParser.IPProtocol, "udp"); + expected.put(BasicPaloAltoFirewallParser.ReceiveTime, "2017/05/25 21:38:13"); + expected.put(BasicPaloAltoFirewallParser.RepeatCount, "1"); + expected.put(BasicPaloAltoFirewallParser.Rule, "DropLog"); + expected.put(BasicPaloAltoFirewallParser.Seqno, "9953744"); + expected.put(BasicPaloAltoFirewallParser.SerialNum, "001606000003"); + expected.put(BasicPaloAltoFirewallParser.EndReason, "policy-deny"); + expected.put(BasicPaloAltoFirewallParser.SessionID, "0"); + expected.put(BasicPaloAltoFirewallParser.SourceLocation, "192.168.0.0-192.168.255.255"); + expected.put(BasicPaloAltoFirewallParser.StartTime, "2017/05/25 21:38:12"); + expected.put(BasicPaloAltoFirewallParser.ThreatContentType, "drop"); + expected.put(BasicPaloAltoFirewallParser.GenerateTime, "2017/05/25 21:38:13"); + expected.put("timestamp", actual.get("timestamp")); + expected.put(BasicPaloAltoFirewallParser.DestinationZone, "VPN"); + expected.put(BasicPaloAltoFirewallParser.Type, "TRAFFIC"); + expected.put(BasicPaloAltoFirewallParser.VirtualSystem, "vsys1"); + expected.put(BasicPaloAltoFirewallParser.DGH1, "19"); + expected.put(BasicPaloAltoFirewallParser.DGH2, "0"); + expected.put(BasicPaloAltoFirewallParser.DGH3, "0"); + expected.put(BasicPaloAltoFirewallParser.DGH4, "0"); + expected.put(BasicPaloAltoFirewallParser.DeviceName, "PAN1"); + assertEquals(expected, actual); + } + + public static final String TRAFFIC_71 = "1,2017/05/31 23:59:57,0006C000005,TRAFFIC,drop,0,2017/05/31 23:59:57,185.94.1.1,201.1.4.5,0.0.0.0,0.0.0.0,DropLog,,,not-applicable,vsys1,untrust,untrust,vlan.1,,Standard-Syslog,2017/05/31 23:59:57,0,1,59836,123,0,0,0x0,udp,deny,60,60,0,1,2017/05/31 23:59:57,0,any,0,3433072193,0x0,RU,DE,0,1,0,policy-deny,16,11,0,0,,PAN1,from-policy"; + @SuppressWarnings("unchecked") + @Test + public void testParseTraffic71() throws ParseException { + JSONObject actual = parser.parse(TRAFFIC_71.getBytes()).get(0); + + JSONObject expected = new JSONObject(); + expected.put(BasicPaloAltoFirewallParser.Action, "deny"); + expected.put(BasicPaloAltoFirewallParser.ActionFlags, "0x0"); + expected.put(BasicPaloAltoFirewallParser.ActionSource, "from-policy"); + expected.put(BasicPaloAltoFirewallParser.Application, "not-applicable"); + expected.put(BasicPaloAltoFirewallParser.Bytes, "60"); + expected.put(BasicPaloAltoFirewallParser.BytesReceived, "0"); + expected.put(BasicPaloAltoFirewallParser.BytesSent, "60"); + expected.put(BasicPaloAltoFirewallParser.Category, "any"); + expected.put(BasicPaloAltoFirewallParser.ConfigVersion, "0"); + expected.put(BasicPaloAltoFirewallParser.DestinationLocation, "DE"); + expected.put(BasicPaloAltoFirewallParser.ElapsedTimeInSec, "0"); + expected.put(BasicPaloAltoFirewallParser.Flags, "0x0"); + expected.put(BasicPaloAltoFirewallParser.SourceZone, "untrust"); + expected.put(BasicPaloAltoFirewallParser.InboundInterface, "vlan.1"); + expected.put(BasicPaloAltoFirewallParser.DestinationAddress, "201.1.4.5"); + expected.put(BasicPaloAltoFirewallParser.DestinationPort, "123"); + expected.put(BasicPaloAltoFirewallParser.SourceAddress, "185.94.1.1"); + expected.put(BasicPaloAltoFirewallParser.SourcePort, "59836"); + expected.put(BasicPaloAltoFirewallParser.LogAction, "Standard-Syslog"); + expected.put(BasicPaloAltoFirewallParser.NATDestinationPort, "0"); + expected.put(BasicPaloAltoFirewallParser.NATDestinationIP, "0.0.0.0"); + expected.put(BasicPaloAltoFirewallParser.NATSourcePort, "0"); + expected.put(BasicPaloAltoFirewallParser.NATSourceIP, "0.0.0.0"); + expected.put("original_string", TRAFFIC_71); + expected.put(BasicPaloAltoFirewallParser.Packets, "1"); + expected.put(BasicPaloAltoFirewallParser.PaloAltoDomain, "1"); + expected.put(BasicPaloAltoFirewallParser.ParserVersion, 70); + expected.put(BasicPaloAltoFirewallParser.PktsReceived, "0"); + expected.put(BasicPaloAltoFirewallParser.PktsSent, "1"); + expected.put(BasicPaloAltoFirewallParser.IPProtocol, "udp"); + expected.put(BasicPaloAltoFirewallParser.ReceiveTime, "2017/05/31 23:59:57"); + expected.put(BasicPaloAltoFirewallParser.RepeatCount, "1"); + expected.put(BasicPaloAltoFirewallParser.Rule, "DropLog"); + expected.put(BasicPaloAltoFirewallParser.Seqno, "3433072193"); + expected.put(BasicPaloAltoFirewallParser.SerialNum, "0006C000005"); + expected.put(BasicPaloAltoFirewallParser.EndReason, "policy-deny"); + expected.put(BasicPaloAltoFirewallParser.SessionID, "0"); + expected.put(BasicPaloAltoFirewallParser.SourceLocation, "RU"); + expected.put(BasicPaloAltoFirewallParser.StartTime, "2017/05/31 23:59:57"); + expected.put(BasicPaloAltoFirewallParser.ThreatContentType, "drop"); + expected.put(BasicPaloAltoFirewallParser.GenerateTime, "2017/05/31 23:59:57"); + expected.put("timestamp", actual.get("timestamp")); + expected.put(BasicPaloAltoFirewallParser.DestinationZone, "untrust"); + expected.put(BasicPaloAltoFirewallParser.Type, "TRAFFIC"); + expected.put(BasicPaloAltoFirewallParser.VirtualSystem, "vsys1"); + expected.put(BasicPaloAltoFirewallParser.DGH1, "16"); + expected.put(BasicPaloAltoFirewallParser.DGH2, "11"); + expected.put(BasicPaloAltoFirewallParser.DGH3, "0"); + expected.put(BasicPaloAltoFirewallParser.DGH4, "0"); + expected.put(BasicPaloAltoFirewallParser.DeviceName, "PAN1"); + assertEquals(expected, actual); + } + + public static final String THREAT_71 = "1,2017/05/25 19:31:13,0006C000005,THREAT,url,0,2017/05/25 19:31:13,192.168.1.7,140.177.26.29,201.1.4.5,140.177.26.29,ms_out,,,ssl,vsys1,mgmt,untrust,vlan.199,vlan.1,Standard-Syslog,2017/05/25 19:31:13,50556,1,56059,443,14810,443,0x40b000,tcp,alert,\"settings-win.data.microsoft.com/\",(9999),computer-and-internet-info,informational,client-to-server,10030265,0x0,192.168.0.0-192.168.255.255,IE,0,,0,,,0,,,,,,,,0,16,11,0,0,,PAN1,"; + @SuppressWarnings("unchecked") + @Test + public void testParseThreat71() throws ParseException { + JSONObject actual = parser.parse(THREAT_71.getBytes()).get(0); + + JSONObject expected = new JSONObject(); + expected.put(BasicPaloAltoFirewallParser.Action, "alert"); + expected.put(BasicPaloAltoFirewallParser.ActionFlags, "0x0"); + expected.put(BasicPaloAltoFirewallParser.Application, "ssl"); + expected.put(BasicPaloAltoFirewallParser.Category, "computer-and-internet-info"); + expected.put(BasicPaloAltoFirewallParser.ConfigVersion, "0"); + expected.put(BasicPaloAltoFirewallParser.Direction, "client-to-server"); + expected.put(BasicPaloAltoFirewallParser.DestinationLocation, "IE"); + expected.put(BasicPaloAltoFirewallParser.Flags, "0x40b000"); + expected.put(BasicPaloAltoFirewallParser.SourceZone, "mgmt"); + expected.put(BasicPaloAltoFirewallParser.InboundInterface, "vlan.199"); + expected.put(BasicPaloAltoFirewallParser.DestinationAddress, "140.177.26.29"); + expected.put(BasicPaloAltoFirewallParser.DestinationPort, "443"); + expected.put(BasicPaloAltoFirewallParser.SourceAddress, "192.168.1.7"); + expected.put(BasicPaloAltoFirewallParser.SourcePort, "56059"); + expected.put(BasicPaloAltoFirewallParser.LogAction, "Standard-Syslog"); + expected.put(BasicPaloAltoFirewallParser.NATDestinationPort, "443"); + expected.put(BasicPaloAltoFirewallParser.NATDestinationIP, "140.177.26.29"); + expected.put(BasicPaloAltoFirewallParser.NATSourcePort, "14810"); + expected.put(BasicPaloAltoFirewallParser.NATSourceIP, "201.1.4.5"); + expected.put("original_string", THREAT_71); + expected.put(BasicPaloAltoFirewallParser.OutboundInterface, "vlan.1"); + expected.put(BasicPaloAltoFirewallParser.PaloAltoDomain, "1"); + expected.put(BasicPaloAltoFirewallParser.ParserVersion, 70); + expected.put(BasicPaloAltoFirewallParser.PCAPID, "0"); + expected.put(BasicPaloAltoFirewallParser.IPProtocol, "tcp"); + expected.put(BasicPaloAltoFirewallParser.ReceiveTime, "2017/05/25 19:31:13"); + expected.put(BasicPaloAltoFirewallParser.RepeatCount, "1"); + expected.put(BasicPaloAltoFirewallParser.Rule, "ms_out"); + expected.put(BasicPaloAltoFirewallParser.Seqno, "10030265"); + expected.put(BasicPaloAltoFirewallParser.SerialNum, "0006C000005"); + expected.put(BasicPaloAltoFirewallParser.SessionID, "50556"); + expected.put(BasicPaloAltoFirewallParser.Severity, "informational"); + expected.put(BasicPaloAltoFirewallParser.SourceLocation, "192.168.0.0-192.168.255.255"); + expected.put(BasicPaloAltoFirewallParser.StartTime, "2017/05/25 19:31:13"); + expected.put(BasicPaloAltoFirewallParser.ThreatContentType, "url"); + expected.put(BasicPaloAltoFirewallParser.ThreatID, "(9999)"); + expected.put(BasicPaloAltoFirewallParser.GenerateTime, "2017/05/25 19:31:13"); + expected.put("timestamp", actual.get("timestamp")); + expected.put(BasicPaloAltoFirewallParser.DestinationZone, "untrust"); + expected.put(BasicPaloAltoFirewallParser.Type, "THREAT"); + expected.put(BasicPaloAltoFirewallParser.URL, "settings-win.data.microsoft.com/"); + expected.put(BasicPaloAltoFirewallParser.VirtualSystem, "vsys1"); + expected.put(BasicPaloAltoFirewallParser.URLIndex, "0"); + expected.put(BasicPaloAltoFirewallParser.WFReportID, "0"); + expected.put(BasicPaloAltoFirewallParser.DGH1, "16"); + expected.put(BasicPaloAltoFirewallParser.DGH2, "11"); + expected.put(BasicPaloAltoFirewallParser.DGH3, "0"); + expected.put(BasicPaloAltoFirewallParser.DGH4, "0"); + expected.put(BasicPaloAltoFirewallParser.DeviceName, "PAN1"); + assertEquals(expected, actual); + } + + public static final String THREAT_80 = "1,2018/02/01 21:29:03,001606000007,THREAT,vulnerability,1,2018/02/01 21:29:03,213.211.198.62,172.16.2.6,213.211.198.62,192.168.178.202,Outgoing,,,web-browsing,vsys1,internet,guest,ethernet1/1,ethernet1/2.2,test,2018/02/01 21:29:03,18720,1,80,53161,80,32812,0x402000,tcp,reset-server,\"www.eicar.org/download/eicar.com\",Eicar File Detected(39040),computer-and-internet-info,medium,server-to-client,27438839,0x0,Germany,172.16.0.0-172.31.255.255,0,,0,,,9,,,,,,,,0,0,0,0,0,,PAN1,,,,,0,,0,,N/A,code-execution,AppThreat-771-4450,0x0"; + @SuppressWarnings("unchecked") + @Test + public void testParseThreat80() throws ParseException { + JSONObject actual = parser.parse(THREAT_80.getBytes()).get(0); + + JSONObject expected = new JSONObject(); + expected.put(BasicPaloAltoFirewallParser.Action, "reset-server"); + expected.put(BasicPaloAltoFirewallParser.ActionFlags, "0x0"); + expected.put(BasicPaloAltoFirewallParser.Application, "web-browsing"); + expected.put(BasicPaloAltoFirewallParser.Category, "computer-and-internet-info"); + expected.put(BasicPaloAltoFirewallParser.ConfigVersion, "1"); + expected.put(BasicPaloAltoFirewallParser.ContentVersion, "AppThreat-771-4450"); + expected.put(BasicPaloAltoFirewallParser.Direction, "server-to-client"); + expected.put(BasicPaloAltoFirewallParser.DestinationLocation, "172.16.0.0-172.31.255.255"); + expected.put(BasicPaloAltoFirewallParser.Flags, "0x402000"); + expected.put(BasicPaloAltoFirewallParser.SourceZone, "internet"); + expected.put(BasicPaloAltoFirewallParser.InboundInterface, "ethernet1/1"); + expected.put(BasicPaloAltoFirewallParser.DestinationAddress, "172.16.2.6"); + expected.put(BasicPaloAltoFirewallParser.DestinationPort, "53161"); + expected.put(BasicPaloAltoFirewallParser.SourceAddress, "213.211.198.62"); + expected.put(BasicPaloAltoFirewallParser.SourcePort, "80"); + expected.put(BasicPaloAltoFirewallParser.LogAction, "test"); + expected.put(BasicPaloAltoFirewallParser.NATDestinationPort, "32812"); + expected.put(BasicPaloAltoFirewallParser.NATDestinationIP, "192.168.178.202"); + expected.put(BasicPaloAltoFirewallParser.NATSourcePort, "80"); + expected.put(BasicPaloAltoFirewallParser.NATSourceIP, "213.211.198.62"); + expected.put("original_string", THREAT_80); + expected.put(BasicPaloAltoFirewallParser.OutboundInterface, "ethernet1/2.2"); + expected.put(BasicPaloAltoFirewallParser.PaloAltoDomain, "1"); + expected.put(BasicPaloAltoFirewallParser.ParentSessionId, "0"); + expected.put(BasicPaloAltoFirewallParser.ParserVersion, 80); + expected.put(BasicPaloAltoFirewallParser.PCAPID, "0"); + expected.put(BasicPaloAltoFirewallParser.IPProtocol, "tcp"); + expected.put(BasicPaloAltoFirewallParser.ReceiveTime, "2018/02/01 21:29:03"); + expected.put(BasicPaloAltoFirewallParser.RepeatCount, "1"); + expected.put(BasicPaloAltoFirewallParser.Rule, "Outgoing"); + expected.put(BasicPaloAltoFirewallParser.Seqno, "27438839"); + expected.put(BasicPaloAltoFirewallParser.SerialNum, "001606000007"); + expected.put(BasicPaloAltoFirewallParser.SessionID, "18720"); + expected.put(BasicPaloAltoFirewallParser.Severity, "medium"); + expected.put(BasicPaloAltoFirewallParser.SourceLocation, "Germany"); + expected.put(BasicPaloAltoFirewallParser.StartTime, "2018/02/01 21:29:03"); + expected.put(BasicPaloAltoFirewallParser.ThreatCategory, "code-execution"); + expected.put(BasicPaloAltoFirewallParser.ThreatContentType, "vulnerability"); + expected.put(BasicPaloAltoFirewallParser.ThreatID, "Eicar File Detected(39040)"); + expected.put(BasicPaloAltoFirewallParser.GenerateTime, "2018/02/01 21:29:03"); + expected.put("timestamp", actual.get("timestamp")); + expected.put(BasicPaloAltoFirewallParser.DestinationZone, "guest"); + expected.put(BasicPaloAltoFirewallParser.TunnelId, "0"); + expected.put(BasicPaloAltoFirewallParser.TunnelType, "N/A"); + expected.put(BasicPaloAltoFirewallParser.Type, "THREAT"); + expected.put(BasicPaloAltoFirewallParser.URL, "www.eicar.org/download/eicar.com"); + expected.put(BasicPaloAltoFirewallParser.VirtualSystem, "vsys1"); + expected.put(BasicPaloAltoFirewallParser.URLIndex, "9"); + expected.put(BasicPaloAltoFirewallParser.WFReportID, "0"); + expected.put(BasicPaloAltoFirewallParser.DGH1, "0"); + expected.put(BasicPaloAltoFirewallParser.DGH2, "0"); + expected.put(BasicPaloAltoFirewallParser.DGH3, "0"); + expected.put(BasicPaloAltoFirewallParser.DGH4, "0"); + expected.put(BasicPaloAltoFirewallParser.DeviceName, "PAN1"); + assertEquals(expected, actual); + } + + public static final String TRAFFIC_80 = "1,2018/02/01 21:24:11,001606000007,TRAFFIC,end,1,2018/02/01 21:24:11,172.16.2.31,134.19.6.22,192.168.18.2,134.19.6.22,Outgoing,,,ssl,vsys1,guest,internet,ethernet1/2.2,ethernet1/1,test,2018/02/01 21:24:11,19468,1,41537,443,12211,443,0x40001c,tcp,allow,7936,1731,6205,24,2018/02/01 21:00:42,1395,computer-and-internet-info,0,62977478,0x0,172.16.0.0-172.31.255.255,United States,0,14,10,tcp-rst-from-client,0,0,0,0,,PAN1,from-policy,,,0,,0,,N/A"; + @SuppressWarnings("unchecked") + @Test + public void testParseTraffic80() throws ParseException { + JSONObject actual = parser.parse(TRAFFIC_80.getBytes()).get(0); - for (Object o : json.entrySet()) { - Entry entry = (Entry) o; - String key = (String) entry.getKey(); - String value = json.get(key).toString(); - Assert.assertNotNull(value); - } - } + JSONObject expected = new JSONObject(); + expected.put(BasicPaloAltoFirewallParser.Action, "allow"); + expected.put(BasicPaloAltoFirewallParser.ActionFlags, "0x0"); + expected.put(BasicPaloAltoFirewallParser.ActionSource, "from-policy"); + expected.put(BasicPaloAltoFirewallParser.Application, "ssl"); + expected.put(BasicPaloAltoFirewallParser.Bytes, "7936"); + expected.put(BasicPaloAltoFirewallParser.BytesReceived, "6205"); + expected.put(BasicPaloAltoFirewallParser.BytesSent, "1731"); + expected.put(BasicPaloAltoFirewallParser.Category, "computer-and-internet-info"); + expected.put(BasicPaloAltoFirewallParser.ConfigVersion, "1"); + expected.put(BasicPaloAltoFirewallParser.DestinationLocation, "United States"); + expected.put(BasicPaloAltoFirewallParser.ElapsedTimeInSec, "1395"); + expected.put(BasicPaloAltoFirewallParser.Flags, "0x40001c"); + expected.put(BasicPaloAltoFirewallParser.SourceZone, "guest"); + expected.put(BasicPaloAltoFirewallParser.InboundInterface, "ethernet1/2.2"); + expected.put(BasicPaloAltoFirewallParser.DestinationAddress, "134.19.6.22"); + expected.put(BasicPaloAltoFirewallParser.DestinationPort, "443"); + expected.put(BasicPaloAltoFirewallParser.SourceAddress, "172.16.2.31"); + expected.put(BasicPaloAltoFirewallParser.SourcePort, "41537"); + expected.put(BasicPaloAltoFirewallParser.LogAction, "test"); + expected.put(BasicPaloAltoFirewallParser.NATDestinationPort, "443"); + expected.put(BasicPaloAltoFirewallParser.NATDestinationIP, "134.19.6.22"); + expected.put(BasicPaloAltoFirewallParser.NATSourcePort, "12211"); + expected.put(BasicPaloAltoFirewallParser.NATSourceIP, "192.168.18.2"); + expected.put("original_string", TRAFFIC_80); + expected.put(BasicPaloAltoFirewallParser.OutboundInterface, "ethernet1/1"); + expected.put(BasicPaloAltoFirewallParser.Packets, "24"); + expected.put(BasicPaloAltoFirewallParser.PaloAltoDomain, "1"); + expected.put(BasicPaloAltoFirewallParser.ParentSessionId, "0"); + expected.put(BasicPaloAltoFirewallParser.ParserVersion, 80); + expected.put(BasicPaloAltoFirewallParser.PktsReceived, "10"); + expected.put(BasicPaloAltoFirewallParser.PktsSent, "14"); + expected.put(BasicPaloAltoFirewallParser.IPProtocol, "tcp"); + expected.put(BasicPaloAltoFirewallParser.ReceiveTime, "2018/02/01 21:24:11"); + expected.put(BasicPaloAltoFirewallParser.RepeatCount, "1"); + expected.put(BasicPaloAltoFirewallParser.Rule, "Outgoing"); + expected.put(BasicPaloAltoFirewallParser.Seqno, "62977478"); + expected.put(BasicPaloAltoFirewallParser.SerialNum, "001606000007"); + expected.put(BasicPaloAltoFirewallParser.EndReason, "tcp-rst-from-client"); + expected.put(BasicPaloAltoFirewallParser.SessionID, "19468"); + expected.put(BasicPaloAltoFirewallParser.SourceLocation, "172.16.0.0-172.31.255.255"); + expected.put(BasicPaloAltoFirewallParser.StartTime, "2018/02/01 21:00:42"); + expected.put(BasicPaloAltoFirewallParser.ThreatContentType, "end"); + expected.put(BasicPaloAltoFirewallParser.GenerateTime, "2018/02/01 21:24:11"); + expected.put("timestamp", actual.get("timestamp")); + expected.put(BasicPaloAltoFirewallParser.DestinationZone, "internet"); + expected.put(BasicPaloAltoFirewallParser.TunnelId, "0"); + expected.put(BasicPaloAltoFirewallParser.TunnelType, "N/A"); + expected.put(BasicPaloAltoFirewallParser.Type, "TRAFFIC"); + expected.put(BasicPaloAltoFirewallParser.VirtualSystem, "vsys1"); + expected.put(BasicPaloAltoFirewallParser.DGH1, "0"); + expected.put(BasicPaloAltoFirewallParser.DGH2, "0"); + expected.put(BasicPaloAltoFirewallParser.DGH3, "0"); + expected.put(BasicPaloAltoFirewallParser.DGH4, "0"); + expected.put(BasicPaloAltoFirewallParser.DeviceName, "PAN1"); + assertEquals(expected, actual); } } http://git-wip-us.apache.org/repos/asf/metron/blob/5f08ba0b/metron-platform/metron-parsers/src/test/resources/logData/PaloAltoFirewallParserTest.txt ---------------------------------------------------------------------- diff --git a/metron-platform/metron-parsers/src/test/resources/logData/PaloAltoFirewallParserTest.txt b/metron-platform/metron-parsers/src/test/resources/logData/PaloAltoFirewallParserTest.txt deleted file mode 100644 index c58bcc8..0000000 --- a/metron-platform/metron-parsers/src/test/resources/logData/PaloAltoFirewallParserTest.txt +++ /dev/null @@ -1,2 +0,0 @@ -<11>Jan 5 05:38:59 PAN1.exampleCustomer.com 1,2015/01/05 05:38:58,0006C110285,THREAT,vulnerability,1,2015/01/05 05:38:58,10.0.0.115,216.0.10.198,0.0.0.0,0.0.0.0,EX-Allow,example\\user.name,,web-browsing,vsys1,internal,external,ethernet1/2,ethernet1/1,LOG-Default,2015/01/05 05:38:58,12031,1,54180,80,0,0,0x80004000,tcp,reset-both,\"ad.aspx?f=300x250&id=12;tile=1;ord=67AF705D60B1119C0F18BEA336F9\",HTTP: IIS Denial Of Service Attempt(40019),any,high,client-to-server,347368099,0x0,10.0.0.0-10.255.255.255,US,0,,1200568889751109656,, -<14>Jan 5 12:51:34 PAN1.exampleCustomer.com 1,2015/01/05 12:51:33,0011C103117,TRAFFIC,end,1,2015/01/05 12:51:33,10.0.0.39,10.1.0.163,0.0.0.0,0.0.0.0,EX-Allow,,example\\user.name,ms-ds-smb,vsys1,v_external,v_internal,ethernet1/2,ethernet1/1,LOG-Default,2015/01/05 12:51:33,33760927,1,52688,445,0,0,0x401a,tcp,allow,2229,1287,942,10,2015/01/05 12:51:01,30,any,0,17754932062,0x0,10.0.0.0-10.255.255.255,10.0.0.0-10.255.255.255,0,6,4 \ No newline at end of file
