Repository: incubator-metron Updated Branches: refs/heads/master 6f5c84ee2 -> 609ea40c2
METRON-488: Snort should use a proper CSV implementation (cestella via mmiklavc) closes apache/incubator-metron#297 Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/609ea40c Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/609ea40c Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/609ea40c Branch: refs/heads/master Commit: 609ea40c2e8ab3b35b645f4238d2f66afc120fb5 Parents: 6f5c84e Author: cstella <ceste...@gmail.com> Authored: Tue Oct 11 13:59:45 2016 -0400 Committer: Michael Miklavcic <michael.miklav...@gmail.com> Committed: Tue Oct 11 13:59:45 2016 -0400 ---------------------------------------------------------------------- .../main/sample/data/snort/parsed/SnortParsed | 6 +-- .../metron/parsers/snort/BasicSnortParser.java | 39 +++++++++++++++----- .../apache/metron/parsers/SnortParserTest.java | 6 ++- 3 files changed, 37 insertions(+), 14 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/609ea40c/metron-platform/metron-integration-test/src/main/sample/data/snort/parsed/SnortParsed ---------------------------------------------------------------------- diff --git a/metron-platform/metron-integration-test/src/main/sample/data/snort/parsed/SnortParsed b/metron-platform/metron-integration-test/src/main/sample/data/snort/parsed/SnortParsed index 318b158..e2894b3 100644 --- a/metron-platform/metron-integration-test/src/main/sample/data/snort/parsed/SnortParsed +++ b/metron-platform/metron-integration-test/src/main/sample/data/snort/parsed/SnortParsed @@ -1,3 +1,3 @@ -{"msg":"\"Consecutive TCP small segments exceeding threshold\"","sig_rev":"1","ip_dst_addr":"10.0.2.15","ip_dst_port":"22","ethsrc":"52:54:00:12:35:02","tcpseq":"0x9AFF3D7","dgmlen":"64","icmpid":"","tcplen":"","tcpwindow":"0xFFFF","icmpseq":"","tcpack":"0xC8761D52","original_string":"01\/27-16:01:04.877970 ,129,12,1,\"Consecutive TCP small segments exceeding threshold\",TCP,10.0.2.2,56642,10.0.2.15,22,52:54:00:12:35:02,08:00:27:7F:93:2D,0x4E,***AP***,0x9AFF3D7,0xC8761D52,,0xFFFF,64,0,59677,64,65536,,,,","icmpcode":"","tos":"0","id":"59677","timestamp":1453932941970,"ethdst":"08:00:27:7F:93:2D","ip_src_addr":"10.0.2.2","ttl":"64","source.type":"snort","ethlen":"0x4E","iplen":"65536","icmptype":"","protocol":"TCP","ip_src_port":"56642","tcpflags":"***AP***","sig_id":"12","sig_generator":"129", "is_alert" : "true"} -{"msg":"\"Consecutive TCP small segments exceeding threshold\"","sig_rev":"1","ip_dst_addr":"10.0.2.15","ip_dst_port":"50895","ethsrc":"52:54:00:12:35:02","tcpseq":"0xDB45F7A","dgmlen":"96","icmpid":"","tcplen":"","tcpwindow":"0xFFFF","icmpseq":"","tcpack":"0x7701DD5B","original_string":"02\/22-15:56:48.612494 ,129,12,1,\"Consecutive TCP small segments exceeding threshold\",TCP,96.44.142.5,80,10.0.2.15,50895,52:54:00:12:35:02,08:00:27:7F:93:2D,0x6E,***AP***,0xDB45F7A,0x7701DD5B,,0xFFFF,64,0,16785,96,98304,,,,","icmpcode":"","tos":"0","id":"16785","timestamp":1456178820494,"ethdst":"08:00:27:7F:93:2D","ip_src_addr":"96.44.142.5","ttl":"64","source.type":"snort","ethlen":"0x6E","iplen":"98304","icmptype":"","protocol":"TCP","ip_src_port":"80","tcpflags":"***AP***","sig_id":"12","sig_generator":"129", "is_alert" : "true"} -{"msg":"\"Consecutive TCP small segments exceeding threshold\"","sig_rev":"1","ip_dst_addr":"10.0.2.15","ip_dst_port":"50895","ethsrc":"52:54:00:12:35:02","tcpseq":"0xDB508F2","dgmlen":"152","icmpid":"","tcplen":"","tcpwindow":"0xFFFF","icmpseq":"","tcpack":"0x7701DD5B","original_string":"02\/22-15:56:48.616775 ,129,12,1,\"Consecutive TCP small segments exceeding threshold\",TCP,96.44.142.5,80,10.0.2.15,50895,52:54:00:12:35:02,08:00:27:7F:93:2D,0xA6,***AP***,0xDB508F2,0x7701DD5B,,0xFFFF,64,0,16824,152,155648,,,,","icmpcode":"","tos":"0","id":"16824","timestamp":1456178824775,"ethdst":"08:00:27:7F:93:2D","ip_src_addr":"96.44.142.5","ttl":"64","source.type":"snort","ethlen":"0xA6","iplen":"155648","icmptype":"","protocol":"TCP","ip_src_port":"80","tcpflags":"***AP***","sig_id":"12","sig_generator":"129", "is_alert" : "true"} +{"msg":"Consecutive TCP small segments exceeding threshold","sig_rev":"1","ip_dst_addr":"10.0.2.15","ip_dst_port":"22","ethsrc":"52:54:00:12:35:02","tcpseq":"0x9AFF3D7","dgmlen":"64","icmpid":"","tcplen":"","tcpwindow":"0xFFFF","icmpseq":"","tcpack":"0xC8761D52","original_string":"01\/27-16:01:04.877970 ,129,12,1,\"Consecutive TCP small segments exceeding threshold\",TCP,10.0.2.2,56642,10.0.2.15,22,52:54:00:12:35:02,08:00:27:7F:93:2D,0x4E,***AP***,0x9AFF3D7,0xC8761D52,,0xFFFF,64,0,59677,64,65536,,,,","icmpcode":"","tos":"0","id":"59677","timestamp":1453932941970,"ethdst":"08:00:27:7F:93:2D","ip_src_addr":"10.0.2.2","ttl":"64","source.type":"snort","ethlen":"0x4E","iplen":"65536","icmptype":"","protocol":"TCP","ip_src_port":"56642","tcpflags":"***AP***","sig_id":"12","sig_generator":"129", "is_alert" : "true"} +{"msg":"Consecutive TCP small segments exceeding threshold","sig_rev":"1","ip_dst_addr":"10.0.2.15","ip_dst_port":"50895","ethsrc":"52:54:00:12:35:02","tcpseq":"0xDB45F7A","dgmlen":"96","icmpid":"","tcplen":"","tcpwindow":"0xFFFF","icmpseq":"","tcpack":"0x7701DD5B","original_string":"02\/22-15:56:48.612494 ,129,12,1,\"Consecutive TCP small segments exceeding threshold\",TCP,96.44.142.5,80,10.0.2.15,50895,52:54:00:12:35:02,08:00:27:7F:93:2D,0x6E,***AP***,0xDB45F7A,0x7701DD5B,,0xFFFF,64,0,16785,96,98304,,,,","icmpcode":"","tos":"0","id":"16785","timestamp":1456178820494,"ethdst":"08:00:27:7F:93:2D","ip_src_addr":"96.44.142.5","ttl":"64","source.type":"snort","ethlen":"0x6E","iplen":"98304","icmptype":"","protocol":"TCP","ip_src_port":"80","tcpflags":"***AP***","sig_id":"12","sig_generator":"129", "is_alert" : "true"} +{"msg":"Consecutive TCP small segments exceeding threshold","sig_rev":"1","ip_dst_addr":"10.0.2.15","ip_dst_port":"50895","ethsrc":"52:54:00:12:35:02","tcpseq":"0xDB508F2","dgmlen":"152","icmpid":"","tcplen":"","tcpwindow":"0xFFFF","icmpseq":"","tcpack":"0x7701DD5B","original_string":"02\/22-15:56:48.616775 ,129,12,1,\"Consecutive TCP small segments exceeding threshold\",TCP,96.44.142.5,80,10.0.2.15,50895,52:54:00:12:35:02,08:00:27:7F:93:2D,0xA6,***AP***,0xDB508F2,0x7701DD5B,,0xFFFF,64,0,16824,152,155648,,,,","icmpcode":"","tos":"0","id":"16824","timestamp":1456178824775,"ethdst":"08:00:27:7F:93:2D","ip_src_addr":"96.44.142.5","ttl":"64","source.type":"snort","ethlen":"0xA6","iplen":"155648","icmptype":"","protocol":"TCP","ip_src_port":"80","tcpflags":"***AP***","sig_id":"12","sig_generator":"129", "is_alert" : "true"} http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/609ea40c/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/snort/BasicSnortParser.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/snort/BasicSnortParser.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/snort/BasicSnortParser.java index f295d4c..e50926c 100644 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/snort/BasicSnortParser.java +++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/snort/BasicSnortParser.java @@ -17,7 +17,10 @@ */ package org.apache.metron.parsers.snort; +import com.google.common.collect.Lists; +import jdk.nashorn.internal.runtime.arrays.ArrayIndex; import org.apache.metron.common.Constants; +import org.apache.metron.common.csv.CSVConverter; import org.apache.metron.parsers.BasicParser; import org.json.simple.JSONObject; import org.slf4j.Logger; @@ -72,14 +75,26 @@ public class BasicSnortParser extends BasicParser { */ private String recordDelimiter = ","; + private transient CSVConverter converter; + + public BasicSnortParser() { + + } + @Override public void configure(Map<String, Object> parserConfig) { - + init(); } @Override public void init() { - + if(converter == null) { + converter = new CSVConverter(); + Map<String, Object> config = new HashMap<>(); + config.put(CSVConverter.SEPARATOR_KEY, recordDelimiter); + config.put(CSVConverter.COLUMNS_KEY, Lists.newArrayList(fieldNames)); + converter.initialize(config); + } } @Override @@ -90,18 +105,24 @@ public class BasicSnortParser extends BasicParser { try { // snort alerts expected as csv records String csvMessage = new String(rawMessage, "UTF-8"); - String[] records = csvMessage.split(recordDelimiter, -1); + Map<String, String> records = null; + try { + records = converter.toMap(csvMessage); + } + catch(ArrayIndexOutOfBoundsException aioob) { + throw new IllegalArgumentException("Unexpected number of fields, expected: " + fieldNames.length + " in " + csvMessage); + } // validate the number of fields - if (records.length != fieldNames.length) { - throw new IllegalArgumentException("Unexpected number of fields, expected: " + fieldNames.length + " got: " + records.length); + if (records.size() != fieldNames.length) { + throw new IllegalArgumentException("Unexpected number of fields, expected: " + fieldNames.length + " got: " + records.size()); } long timestamp = 0L; // build the json record from each field - for (int i=0; i<records.length; i++) { + for (Map.Entry<String, String> kv : records.entrySet()) { - String field = fieldNames[i]; - String record = records[i]; + String field = kv.getKey(); + String record = kv.getValue(); if("timestamp".equals(field)) { @@ -119,7 +140,7 @@ public class BasicSnortParser extends BasicParser { jsonMessage.put("is_alert", "true"); messages.add(jsonMessage); } catch (Exception e) { - String message = "Unable to parse message: " + rawMessage; + String message = "Unable to parse message: " + (rawMessage == null?"null" : new String(rawMessage)); _LOG.error(message, e); throw new IllegalStateException(message, e); } http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/609ea40c/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/SnortParserTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/SnortParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/SnortParserTest.java index 6d777aa..8114e83 100644 --- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/SnortParserTest.java +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/SnortParserTest.java @@ -27,7 +27,7 @@ import java.util.Map; public class SnortParserTest { /** - 01/27-16:01:04.877970 ,129,12,1,"Consecutive TCP small segments exceeding threshold",TCP,10.0.2.2,56642,10.0.2.15,22,52:54:00:12:35:02,08:00:27:7F:93:2D,0x4E,***AP***,0x9AFF3D7,0xC8761D52,,0xFFFF,64,0,59677,64,65536,,,, + 01/27-16:01:04.877970 ,129,12,1,"Consecutive TCP small segments, exceeding threshold",TCP,10.0.2.2,56642,10.0.2.15,22,52:54:00:12:35:02,08:00:27:7F:93:2D,0x4E,***AP***,0x9AFF3D7,0xC8761D52,,0xFFFF,64,0,59677,64,65536,,,, **/ @Multiline public static String goodMessage; @@ -36,8 +36,9 @@ public class SnortParserTest { @Test public void testGoodMessage() { BasicSnortParser parser = new BasicSnortParser(); + parser.init(); Map out = parser.parse(goodMessage.getBytes()).get(0); - Assert.assertEquals(out.get("msg"),"\"Consecutive TCP small segments exceeding threshold\""); + Assert.assertEquals(out.get("msg"),"Consecutive TCP small segments, exceeding threshold"); Assert.assertEquals(out.get("sig_rev"), "1"); Assert.assertEquals(out.get("ip_dst_addr"), "10.0.2.15"); Assert.assertEquals(out.get("ip_dst_port"), "22"); @@ -69,6 +70,7 @@ public class SnortParserTest { @Test(expected=IllegalStateException.class) public void testBadMessage() { BasicSnortParser parser = new BasicSnortParser(); + parser.init(); parser.parse("foo bar".getBytes()); } }
