METRON-157 Create CEF Parser (simonellistonball via kylerichardson) closes apache/incubator-metron#451
Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/9e15cb6e Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/9e15cb6e Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/9e15cb6e Branch: refs/heads/Metron_0.3.1 Commit: 9e15cb6e24872620ec4bf3c183d15dd6292b153d Parents: 246acff Author: simonellistonball <si...@simonellistonball.com> Authored: Tue Feb 21 15:50:51 2017 -0500 Committer: Kyle Richardson <kylerichard...@apache.org> Committed: Tue Feb 21 15:50:51 2017 -0500 ---------------------------------------------------------------------- metron-platform/metron-parsers/pom.xml | 2 +- .../apache/metron/parsers/cef/CEFParser.java | 274 ++++++++++++++++++ .../apache/metron/parsers/utils/DateUtils.java | 115 ++++++++ .../metron/parsers/cef/CEFParserTest.java | 277 +++++++++++++++++++ .../org/apache/metron/parsers/cef/adallom.cef | 1 + .../apache/metron/parsers/cef/adallom.schema | 37 +++ .../org/apache/metron/parsers/cef/cyberark.cef | 1 + .../org/apache/metron/parsers/cef/cyberark.json | 21 ++ .../apache/metron/parsers/cef/cyberark.schema | 38 +++ .../org/apache/metron/parsers/cef/waf.cef | 1 + .../org/apache/metron/parsers/cef/waf.schema | 67 +++++ 11 files changed, 833 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9e15cb6e/metron-platform/metron-parsers/pom.xml ---------------------------------------------------------------------- diff --git a/metron-platform/metron-parsers/pom.xml b/metron-platform/metron-parsers/pom.xml index d8a77a0..3049a71 100644 --- a/metron-platform/metron-parsers/pom.xml +++ b/metron-platform/metron-parsers/pom.xml @@ -143,7 +143,7 @@ <dependency> <groupId>com.google.guava</groupId> <artifactId>guava</artifactId> - <version>${global_hbase_guava_version}</version> + <version>${global_guava_version}</version> </dependency> <dependency> <groupId>com.fasterxml.jackson.core</groupId> http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9e15cb6e/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/cef/CEFParser.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/cef/CEFParser.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/cef/CEFParser.java new file mode 100644 index 0000000..a765dd8 --- /dev/null +++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/cef/CEFParser.java @@ -0,0 +1,274 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.parsers.cef; + +import java.nio.charset.Charset; +import java.time.Clock; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.metron.parsers.BasicParser; +import org.apache.metron.parsers.ParseException; +import org.apache.metron.parsers.utils.DateUtils; +import org.apache.metron.parsers.utils.SyslogUtils; +import org.json.simple.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class CEFParser extends BasicParser { + private static final long serialVersionUID = 1L; + + protected static final Logger LOG = LoggerFactory.getLogger(CEFParser.class); + private static final String HEADER_CAPTURE_PATTERN = "[^\\|]*"; + private static final String EXTENSION_CAPTURE_PATTERN = "(?<!\\\\)="; + private static final Charset UTF_8 = Charset.forName("UTF-8"); + + private Pattern p; + private Pattern pext; + + public void init() { + + // CEF Headers: Device Vendor|Device Product|Device Version|Device Event + // Class ID|Name|Severity + + String syslogTime = "(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\\b +(?:(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9]) (?!<[0-9])(?:2[0123]|[01]?[0-9]):(?:[0-5][0-9])(?::(?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?))(?![0-9])?"; + String syslogTime5424 = "(?:\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d+)?(?:Z|[+-]\\d{2}:\\d{2}))"; + String syslogPriority = "<(?:[0-9]+)>"; + String syslogHost = "[a-z0-9\\.\\\\-_]+"; + + StringBuilder sb = new StringBuilder("(?<syslogTime>"); + sb.append(syslogTime); + sb.append("|"); + sb.append(syslogTime5424); + sb.append(")?"); + + sb.append("(?<syslogHost>"); + sb.append(syslogHost); + sb.append(")?"); + + sb.append("(?<syslogPriority>"); + sb.append(syslogPriority); + sb.append(")?"); + + sb.append(".*"); + + sb.append("CEF:0\\|"); + + headerBlock("DeviceVendor", sb); + sb.append("\\|"); + headerBlock("DeviceProduct", sb); + sb.append("\\|"); + headerBlock("DeviceVersion", sb); + sb.append("\\|"); + headerBlock("DeviceEvent", sb); + sb.append("\\|"); + headerBlock("Name", sb); + sb.append("\\|"); + headerBlock("Severity", sb); + sb.append("\\|"); + + // extension capture: + sb.append("(?<extensions>.*)"); + String pattern = sb.toString(); + + p = Pattern.compile(pattern); + + // key finder for extensions + pext = Pattern.compile(EXTENSION_CAPTURE_PATTERN); + } + + @SuppressWarnings("unchecked") + public List<JSONObject> parse(byte[] rawMessage) { + List<JSONObject> messages = new ArrayList<>(); + + String cefString = new String(rawMessage, UTF_8); + + Matcher matcher = p.matcher(cefString); + + while (matcher.find()) { + JSONObject obj = new JSONObject(); + if (matcher.matches()) { + LOG.info(String.format("Found %d groups", matcher.groupCount())); + obj.put("DeviceVendor", matcher.group("DeviceVendor")); + obj.put("DeviceProduct", matcher.group("DeviceProduct")); + obj.put("DeviceVersion", matcher.group("DeviceVersion")); + obj.put("DeviceEvent", matcher.group("DeviceEvent")); + obj.put("Name", matcher.group("Name")); + obj.put("Severity", standardizeSeverity(matcher.group("Severity"))); + } + + String ext = matcher.group("extensions"); + Matcher m = pext.matcher(ext); + + int index = 0; + String key = null; + String value = null; + Map<String, String> labelMap = new HashMap<String, String>(); + + while (m.find()) { + if (key == null) { + key = ext.substring(index, m.start()); + index = m.end(); + if (!m.find()) { + break; + } + } + value = ext.substring(index, m.start()); + index = m.end(); + int v = value.lastIndexOf(" "); + if (v > 0) { + String temp = value.substring(0, v).trim(); + if (key.endsWith("Label")) { + labelMap.put(key.substring(0, key.length() - 5), temp); + } else { + obj.put(key, temp); + } + key = value.substring(v).trim(); + } + } + value = ext.substring(index); + + // Build a map of Label extensions to apply later + if (key.endsWith("Label")) { + labelMap.put(key.substring(0, key.length() - 5), value); + } else { + obj.put(key, value); + } + + // Apply the labels to custom fields + for (Entry<String, String> label : labelMap.entrySet()) { + mutate(obj, label.getKey(), label.getValue()); + } + + // Rename standard CEF fields to comply with Metron standards + obj = mutate(obj, "dst", "ip_dst_addr"); + obj = mutate(obj, "dpt", "ip_dst_port"); + obj = convertToInt(obj, "ip_dst_port"); + + obj = mutate(obj, "src", "ip_src_addr"); + obj = mutate(obj, "spt", "ip_src_port"); + obj = convertToInt(obj, "ip_src_port"); + + obj = mutate(obj, "act", "deviceAction"); + // applicationProtocol + obj = mutate(obj, "app", "protocol"); + + obj.put("original_string", cefString); + + // apply timestamp from message if present, using rt, syslog + // timestamp, + // default to current system time + + if (obj.containsKey("rt")) { + String rt = (String) obj.get("rt"); + try { + obj.put("timestamp", DateUtils.parseMultiformat(rt, DateUtils.DATE_FORMATS_CEF)); + } catch (java.text.ParseException e) { + throw new IllegalStateException("rt field present in CEF but cannot be parsed", e); + } + } else { + String logTimestamp = matcher.group("syslogTime"); + if (!(logTimestamp == null || logTimestamp.isEmpty())) { + try { + obj.put("timestamp", SyslogUtils.parseTimestampToEpochMillis(logTimestamp, Clock.systemUTC())); + } catch (ParseException e) { + throw new IllegalStateException("Cannot parse syslog timestamp", e); + } + } else { + obj.put("timestamp", System.currentTimeMillis()); + } + } + + // add the host + String host = matcher.group("syslogHost"); + if (!(host == null || host.isEmpty())) { + obj.put("host", host); + } + + messages.add(obj); + } + return messages; + } + + @SuppressWarnings("unchecked") + private JSONObject convertToInt(JSONObject obj, String key) { + if (obj.containsKey(key)) { + obj.put(key, Integer.valueOf((String) obj.get(key))); + } + return obj; + } + + private void headerBlock(String name, StringBuilder sb) { + sb.append("(?<").append(name).append(">").append(HEADER_CAPTURE_PATTERN).append(")"); + } + + /** + * Maps string based severity in CEF format to integer. + * + * The strings are mapped according to the CEF 23 specification, taking the + * integer value as the value of the range buckets rounded up + * + * The valid string values are: Unknown, Low, Medium, High, and Very-High. + * The valid integer values are: 0-3=Low, 4-6=Medium, 7- 8=High, and + * 9-10=Very-High. + * + * @param severity + * String or Integer + * @return Integer value mapped from the string + */ + private Integer standardizeSeverity(String severity) { + if (severity.length() < 3) { + // should be a number + return Integer.valueOf(severity); + } else { + switch (severity) { + case "Low": + return 2; + case "Medium": + return 5; + case "High": + return 8; + case "Very-High": + return 10; + default: + return 0; + } + } + } + + @Override + public void configure(Map<String, Object> config) { + // TODO Auto-generated method stub + + } + + @SuppressWarnings("unchecked") + private JSONObject mutate(JSONObject json, String oldKey, String newKey) { + if (json.containsKey(oldKey)) { + json.put(newKey, json.remove(oldKey)); + } + return json; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9e15cb6e/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/utils/DateUtils.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/utils/DateUtils.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/utils/DateUtils.java new file mode 100644 index 0000000..888649a --- /dev/null +++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/utils/DateUtils.java @@ -0,0 +1,115 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.parsers.utils; + +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.time.ZonedDateTime; +import java.util.ArrayList; +import java.util.Calendar; +import java.util.Date; +import java.util.List; +import java.util.TimeZone; +import java.util.regex.Pattern; + +import org.apache.commons.lang.StringUtils; + +/** + * Various utilities for parsing and extracting dates + * + */ +public class DateUtils { + + public static List<SimpleDateFormat> DATE_FORMATS_CEF = new ArrayList<SimpleDateFormat>() { + { + // as per CEF Spec + add(new SimpleDateFormat("MMM dd HH:mm:ss.SSS zzz")); + add(new SimpleDateFormat("MMM dd HH:mm:ss.SSS")); + add(new SimpleDateFormat("MMM dd HH:mm:ss zzz")); + add(new SimpleDateFormat("MMM dd HH:mm:ss")); + add(new SimpleDateFormat("MMM dd yyyy HH:mm:ss.SSS zzz")); + add(new SimpleDateFormat("MMM dd yyyy HH:mm:ss.SSS")); + add(new SimpleDateFormat("MMM dd yyyy HH:mm:ss zzz")); + add(new SimpleDateFormat("MMM dd yyyy HH:mm:ss")); + // found in the wild + add(new SimpleDateFormat("dd MMMM yyyy HH:mm:ss")); + } + }; + + public static List<SimpleDateFormat> DATE_FORMATS_SYSLOG = new ArrayList<SimpleDateFormat>() { + { + // As specified in https://tools.ietf.org/html/rfc5424 + add(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSZ")); + + // common format per rsyslog defaults e.g. Mar 21 14:05:02 + add(new SimpleDateFormat("MMM dd HH:mm:ss")); + add(new SimpleDateFormat("MMM dd yyyy HH:mm:ss")); + + // additional formats found in the wild + add(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss")); + add(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ")); + add(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS")); + + } + }; + + Pattern NUMERIC = Pattern.compile("\\b\\d+\\b"); + + /** + * Parse the data according to a sequence of possible parse patterns. + * + * If the given date is entirely numeric, it is assumed to be a unix + * timestamp. + * + * If the year is not specified in the date string, use the current year. + * Assume that any date more than 4 days in the future is in the past as per + * SyslogUtils + * + * @param candidate + * The possible date. + * @param validPatterns + * A list of SimpleDateFormat instances to try parsing with. + * @return A java.util.Date based on the parse result + * @throws ParseException + */ + public static long parseMultiformat(String candidate, List<SimpleDateFormat> validPatterns) throws ParseException { + if (StringUtils.isNumeric(candidate)) { + return Long.valueOf(candidate); + } else { + for (SimpleDateFormat pattern : validPatterns) { + try { + Calendar cal = Calendar.getInstance(); + cal.setTime(pattern.parse(candidate)); + Calendar current = Calendar.getInstance(); + if (cal.get(Calendar.YEAR) == 1970) { + cal.set(Calendar.YEAR, current.get(Calendar.YEAR)); + } + current.add(Calendar.DAY_OF_MONTH, 4); + if (cal.after(current)) { + cal.add(Calendar.YEAR, -1); + } + return cal.getTimeInMillis(); + } catch (ParseException e) { + continue; + } + } + throw new ParseException("Failed to parse any of the given date formats", 0); + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9e15cb6e/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/cef/CEFParserTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/cef/CEFParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/cef/CEFParserTest.java new file mode 100644 index 0000000..88c0f0c --- /dev/null +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/cef/CEFParserTest.java @@ -0,0 +1,277 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.parsers.cef; + +import java.io.IOException; +import java.net.URL; +import java.nio.charset.Charset; +import java.text.SimpleDateFormat; +import java.util.Calendar; +import java.util.Date; +import java.util.List; +import java.util.Map; + +import org.json.simple.JSONObject; +import org.json.simple.parser.JSONParser; +import org.json.simple.parser.ParseException; +import org.junit.Assert; +import org.junit.Test; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.github.fge.jackson.JsonLoader; +import com.github.fge.jsonschema.core.report.ProcessingReport; +import com.github.fge.jsonschema.main.JsonSchemaFactory; +import com.github.fge.jsonschema.main.JsonValidator; +import com.google.common.io.Resources; + +import junit.framework.TestCase; + +public class CEFParserTest extends TestCase { + + private static final Charset UTF_8 = Charset.forName("utf-8"); + private CEFParser parser; + + @Override + public void setUp() { + parser = new CEFParser(); + parser.init(); + } + + @Test + public void testInvalid() { + List<JSONObject> obj = parse("test test test nonsense\n"); + assertEquals(0, obj.size()); + } + + @Test + public void testEscaping() { + for (JSONObject obj : parse( + "Sep 19 08:26:10 host CEF:0|security|threatmanager|1.0|100|detected a \\ in packet|10|src=10.0.0.1 act=blocked a \\ dst=1.1.1.1")) { + assertEquals("10.0.0.1", obj.get("ip_src_addr")); + assertEquals("blocked a \\", obj.get("deviceAction")); + assertEquals("1.1.1.1", obj.get("ip_dst_addr")); + } + } + + public void testBasicHeader() { + for (JSONObject obj : parse( + "CEF:0|Security|threatmanager|1.0|100|worm successfully stopped|10|src=10.0.0.1 dst=2.1.2.2 spt=1232")) { + assertEquals("Security", obj.get("DeviceVendor")); + assertEquals("threatmanager", obj.get("DeviceProduct")); + assertEquals("1.0", obj.get("DeviceVersion")); + assertEquals("100", obj.get("DeviceEvent")); + assertEquals("worm successfully stopped", obj.get("Name")); + assertEquals(10, obj.get("Severity")); + } + } + + public void testBasicExtensions() { + for (JSONObject obj : parse( + "CEF:0|Security|threatmanager|1.0|100|worm successfully stopped|10|src=10.0.0.1 dst=2.1.2.2 spt=1232")) { + assertEquals("10.0.0.1", obj.get("ip_src_addr")); + assertEquals("2.1.2.2", obj.get("ip_dst_addr")); + assertEquals(1232, obj.get("ip_src_port")); + } + } + + public void testCustomLabelWithSpace() { + for (JSONObject obj : parse( + "CEF:0|Security|threatmanager|1.0|100|worm successfully stopped|10|src=10.0.0.1 dst=2.1.2.2 spt=1232 custom=Text with space customLabel=Label with space")) { + assertEquals(true, obj.containsKey("Label with space")); + assertEquals("Text with space", obj.get("Label with space")); + } + } + + public void testTimestampPriority() throws java.text.ParseException { + long correctTime = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSz").parse("2016-05-01T09:29:11.356-0400") + .getTime(); + + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSz"); + + for (JSONObject obj : parse( + "CEF:0|Security|threatmanager|1.0|100|worm successfully stopped|10|src=10.0.0.1 rt=May 1 2016 09:29:11.356 -0400 dst=2.1.2.2 spt=1232")) { + assertEquals(new Date(correctTime), new Date((long) obj.get("timestamp"))); + assertEquals(correctTime, obj.get("timestamp")); + } + for (JSONObject obj : parse( + "2016-06-01T09:29:11.356-04:00 host CEF:0|Security|threatmanager|1.0|100|worm successfully stopped|10|src=10.0.0.1 rt=May 1 2016 09:29:11.356 -0400 dst=2.1.2.2 spt=1232")) { + assertEquals(new Date(correctTime), new Date((long) obj.get("timestamp"))); + assertEquals(correctTime, obj.get("timestamp")); + } + for (JSONObject obj : parse( + "2016-05-01T09:29:11.356-04:00 host CEF:0|Security|threatmanager|1.0|100|worm successfully stopped|10|src=10.0.0.1 dst=2.1.2.2 spt=1232")) { + assertEquals(new Date(correctTime), new Date((long) obj.get("timestamp"))); + assertEquals(correctTime, obj.get("timestamp")); + } + for (JSONObject obj : parse( + "CEF:0|Security|threatmanager|1.0|100|worm successfully stopped|10|src=10.0.0.1 dst=2.1.2.2 spt=1232")) { + assertNotNull(obj.get("timestamp")); + } + + } + + public void testRtValueAsEpochTimestamp() throws java.text.ParseException { + long correctTime = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSz").parse("2016-05-01T09:29:11.356-0400") + .getTime(); + for (JSONObject obj : parse("CEF:0|Security|threatmanager|1.0|100|worm successfully stopped|10|src=10.0.0.1 rt=" + + String.valueOf(correctTime) + " dst=2.1.2.2 spt=1232")) { + assertEquals(new Date(correctTime), new Date((long) obj.get("timestamp"))); + assertEquals(correctTime, obj.get("timestamp")); + } + } + + private void runMissingYear(Calendar expected, Calendar input) { + SimpleDateFormat sdf = new SimpleDateFormat("MMM dd HH:mm:ss.SSS"); + for (JSONObject obj : parse("CEF:0|Security|threatmanager|1.0|100|worm successfully stopped|10|src=10.0.0.1 rt=" + + sdf.format(input.getTime()) + " dst=2.1.2.2 spt=1232")) { + assertEquals(expected.getTimeInMillis(), obj.get("timestamp")); + assertEquals(expected.getTime(), new Date((long) obj.get("timestamp"))); + } + } + + public void testMissingYearFromDate() throws java.text.ParseException { + Calendar current = Calendar.getInstance(); + Calendar correct = Calendar.getInstance(); + + correct.setTimeInMillis(current.getTimeInMillis()); + + runMissingYear(correct, current); + } + + public void testFourDayFutureBecomesPast() { + Calendar current = Calendar.getInstance(); + Calendar correct = Calendar.getInstance(); + + current.add(Calendar.DAY_OF_MONTH, 5); + // correct.setTime(current.getTime()); + correct.setTimeInMillis(current.getTimeInMillis()); + correct.add(Calendar.YEAR, -1); + + runMissingYear(correct, current); + } + + public void testCEFParserAdallom() throws Exception { + runTest("adallom", Resources.readLines(Resources.getResource(getClass(), "adallom.cef"), UTF_8), + Resources.toString(Resources.getResource(getClass(), "adallom.schema"), UTF_8)); + } + + public void testCEFParserCyberArk() throws Exception { + runTest("cyberark", Resources.readLines(Resources.getResource(getClass(), "cyberark.cef"), UTF_8), + Resources.toString(Resources.getResource(getClass(), "cyberark.schema"), UTF_8), + Resources.toString(Resources.getResource(getClass(), "cyberark.json"), UTF_8)); + } + + public void testCEFParserWAF() throws Exception { + URL waf_url = Resources.getResource(getClass(), "waf.cef"); + runTest("waf", Resources.readLines(waf_url, UTF_8), + Resources.toString(Resources.getResource(getClass(), "waf.schema"), UTF_8)); + } + + private void runTest(String name, List<String> lines, String schema) throws Exception { + runTest(name, lines, schema, ""); + } + + private void runTest(String name, List<String> lines, String schema, String targetJson) throws Exception { + for (String inputString : lines) { + JSONObject parsed = parse(inputString).get(0); + assertNotNull(parsed); + assertNotNull(parsed.get("timestamp")); + assertTrue((long) parsed.get("timestamp") > 0); + + System.out.println(parsed); + JSONParser parser = new JSONParser(); + + Map<?, ?> json = null; + try { + json = (Map<?, ?>) parser.parse(parsed.toJSONString()); + Assert.assertEquals(true, validateJsonData(schema, json.toString())); + } catch (ParseException e) { + e.printStackTrace(); + } + + // test against an explicit json example + if (!targetJson.isEmpty()) { + + } + } + } + + /** + * Additional Sample from NiFi test Suite + * (https://github.com/apache/nifi/blob/rel/nifi-1.1.1/nifi-nar-bundles/nifi + * -standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/ + * processors/standard/TestParseCEF.java) + */ + private final static String sample = "CEF:0|TestVendor|TestProduct|TestVersion|TestEventClassID|TestName|Low|" + + // TimeStamp, String and Long + "rt=Feb 09 2015 00:27:43 UTC cn3Label=Test Long cn3=9223372036854775807 " + + // FloatPoint and MacAddress + "cfp1=1.234 cfp1Label=Test FP Number smac=00:00:0c:07:ac:00 " + + // IPv6 and String + "c6a3=2001:cdba::3257:9652 c6a3Label=Test IPv6 cs1Label=Test String cs1=test test test chocolate " + + // IPv4 + "destinationTranslatedAddress=123.123.123.123 " + + // Date without TZ + "deviceCustomDate1=Feb 06 2015 13:27:43 " + + // Integer and IP Address (from v4) + "dpt=1234 agt=123.123.0.124 dlat=40.366633 " + + // A JSON object inside one of CEF's custom Strings + "cs2Label=JSON payload " + + "cs2={\"test_test_test\": \"chocolate!\", \"what?!?\": \"Simple! test test test chocolate!\"}"; + + @Test + public void testSuccessfulWhenCEFContainsJSON() throws JsonProcessingException, IOException { + List<JSONObject> parse = parse(sample); + JSONObject obj = parse.get(0); + + assertEquals("TestVendor", obj.get("DeviceVendor")); + assertEquals(1423441663000L, obj.get("timestamp")); + assertEquals("9223372036854775807", obj.get("Test Long")); + assertEquals(obj.get("Test FP Number"), String.valueOf(1.234F)); + assertEquals("00:00:0c:07:ac:00", obj.get("smac")); + assertEquals("2001:cdba::3257:9652", obj.get("Test IPv6")); + assertEquals("test test test chocolate", obj.get("Test String")); + assertEquals("123.123.123.123", obj.get("destinationTranslatedAddress")); + + JsonNode inner = new ObjectMapper().readTree((String) obj.get("JSON payload")); + Assert.assertEquals("chocolate!", inner.get("test_test_test").asText()); + } + + protected boolean validateJsonData(final String jsonSchema, final String jsonData) throws Exception { + final JsonNode d = JsonLoader.fromString(jsonData); + final JsonNode s = JsonLoader.fromString(jsonSchema); + + final JsonSchemaFactory factory = JsonSchemaFactory.byDefault(); + JsonValidator v = factory.getValidator(); + + ProcessingReport report = v.validate(s, d); + System.out.println(report); + + return report.toString().contains("success"); + } + + private List<JSONObject> parse(String string) { + List<JSONObject> parse = parser.parse(string.getBytes(Charset.forName("utf-8"))); + assertNotNull(parse); + return parse; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9e15cb6e/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/adallom.cef ---------------------------------------------------------------------- diff --git a/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/adallom.cef b/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/adallom.cef new file mode 100644 index 0000000..a35f354 --- /dev/null +++ b/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/adallom.cef @@ -0,0 +1 @@ +2016-04-01T09:29:11.356-0400 CEF:0|Adallom|Adallom|1.0|56fe779ee4b0459f4e9a484a|ALERT_CABINET_EVENT_MATCH_AUDIT|0|msg=Activity policy 'User download/view file' was triggered by 'per...@example.com' suser=au...@example.com start=1459517280810 end=1459517280810 audits=["AVPR-4oIPeFmuZ3CKKrg","AVPR-wx80cd9PUpAu2aj","AVPR-6XGPeFmuZ3CKKvx","AVPSALn_qE4Kgs_8_yK9","AVPSASW3gw_f3aEvgEmi"] services=["APPID_SXC"] users=["anot...@example.com"] cs6=https://abcd-remote.console.arc.com/#/alerts/56fe779ee4b0459f4e9a484a cs6Label=consoleUrl \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9e15cb6e/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/adallom.schema ---------------------------------------------------------------------- diff --git a/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/adallom.schema b/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/adallom.schema new file mode 100644 index 0000000..a91cce0 --- /dev/null +++ b/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/adallom.schema @@ -0,0 +1,37 @@ +{ + "title": "Adallom Schema", + "type": "object", + "properties": { + "original_string": { + "type": "string" + }, + "timestamp": { + "type": "integer" + }, + "DeviceVendor": { + "type": "string" + }, + "DeviceProduct": { + "type": "string" + }, + "DeviceVersion": { + "type": "string" + }, + "DeviceEvent": { + "type": "string" + }, + "Name": { + "type": "string" + }, + "Severity": { + "type": "integer" + }, + "consoleUrl": { + "type": "string" + } + }, + "required": [ + "original_string", "timestamp", + "DeviceVendor", "DeviceProduct", "DeviceVersion", "Name", "Severity", + "consoleUrl"] +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9e15cb6e/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/cyberark.cef ---------------------------------------------------------------------- diff --git a/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/cyberark.cef b/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/cyberark.cef new file mode 100644 index 0000000..9d4fe6f --- /dev/null +++ b/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/cyberark.cef @@ -0,0 +1 @@ +Mar 21 14:05:02 HHHPVATN1 CEF:0|Cyber-Ark|Vault|7.20.0091|295|Retrieve password|5|act=Retrieve password suser=spilgrim fname=Root\ABC phobos3 - COMP dvc=120.99.70.3 shost=10.44.134.78 dhost= duser= externalId= app= reason= cs1Label="Affected User Name" cs1= cs2Label="Safe Name" cs2=Security Vulnerability Mgmt cs3Label="Device Type" cs3= cs4Label="Database" cs4= cs5Label="Other info" cs5=101.198.70.93 cn1Label="Request Id" cn1= cn2Label="Ticket Id" cn2=Needed to verify config files being pulled msg=Needed to verify config files being pulled \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9e15cb6e/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/cyberark.json ---------------------------------------------------------------------- diff --git a/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/cyberark.json b/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/cyberark.json new file mode 100644 index 0000000..e900a9a --- /dev/null +++ b/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/cyberark.json @@ -0,0 +1,21 @@ +{ + "\"Other info\"": "101.198.70.93", + "\"Safe Name\"": "Security Vulnerability Mgmt", + "\"Ticket Id\"": "Needed to verify config files being pulled ", + "deviceAction": "Retrieve password", + "deviceAddress": "120.99.70.3", + "device_product": "Vault", + "device_vendor": "Cyber-Ark", + "device_version": "7.20.0091", + "event_class_id": "295", + "event_name": "Retrieve password", + "fileName": "Root\\ABC phobos3 - COMP", + "header": "Mar 21 14:05:02 HHHPVATN1 CEF:0", + "message": "Needed to verify config files being pulled", + "original_string": "Mar 21 14:05:02 HHHPVATN1 CEF:0|Cyber-Ark|Vault|7.20.0091|295|Retrieve password|5|act=Retrieve password suser=spilgrim fname=Root\\ABC phobos3 - COMP dvc=120.99.70.3 shost=10.44.134.78 dhost= duser= externalId= app= reason= cs1Label=\"Affected User Name\" cs1= cs2Label=\"Safe Name\" cs2=Security Vulnerability Mgmt cs3Label=\"Device Type\" cs3= cs4Label=\"Database\" cs4= cs5Label=\"Other info\" cs5=101.198.70.93 cn1Label=\"Request Id\" cn1= cn2Label=\"Ticket Id\" cn2=Needed to verify config files being pulled msg=Needed to verify config files being pulled", + "severity": "5", + "source.type": "cyberark", + "src_hostname": "10.44.134.78", + "src_username": "spilgrim", + "timestamp": 1458569102000 +} http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9e15cb6e/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/cyberark.schema ---------------------------------------------------------------------- diff --git a/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/cyberark.schema b/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/cyberark.schema new file mode 100644 index 0000000..5bd1021 --- /dev/null +++ b/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/cyberark.schema @@ -0,0 +1,38 @@ +{ + "title": "CyberArk Schema", + "type": "object", + "properties": { + "ip_src_addr": { + "type": "string" + }, + "ip_dst_addr": { + "type": "string" + }, + "original_string": { + "type": "string" + }, + "timestamp": { + "type": "integer" + }, + "DeviceVendor": { + "type": "string" + }, + "DeviceProduct": { + "type": "string" + }, + "DeviceVersion": { + "type": "string" + }, + "DeviceEvent": { + "type": "string" + }, + "Name": { + "type": "string" + }, + "Severity": { + "type": "integer" + } + }, + "required": ["original_string", "timestamp", + "DeviceVendor", "DeviceProduct", "DeviceVersion", "Name", "Severity"] +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9e15cb6e/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/waf.cef ---------------------------------------------------------------------- diff --git a/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/waf.cef b/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/waf.cef new file mode 100644 index 0000000..86e1d6b --- /dev/null +++ b/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/waf.cef @@ -0,0 +1 @@ +<14>CEF:0|Imperva Inc.|SecureSphere|10.0.0.4_16|ABC - Secure Login.vm Page Rate Limit UK - Source IP||High|act=alert dst=17.43.200.42 dpt=88 duser=${Alert.username} src=10.31.45.69 spt=34435 proto=TCP rt=31 March 2016 13:04:55 cat=Alert cs1= cs1Label=Policy cs2=ABC-Secure cs2Label=ServerGroup cs3=servers_svc cs3Label=ServiceName cs4=server_app cs4Label=ApplicationName cs5=QA cs5Label=Description \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9e15cb6e/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/waf.schema ---------------------------------------------------------------------- diff --git a/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/waf.schema b/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/waf.schema new file mode 100644 index 0000000..b38485c --- /dev/null +++ b/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/waf.schema @@ -0,0 +1,67 @@ +{ + "title": "WAF CEF Schema", + "type": "object", + "properties": { + "ip_src_addr": { + "type": "string" + }, + "ip_src_port": { + "type": "integer" + }, + "ip_dst_addr": { + "type": "string" + }, + "ip_dst_port": { + "type": "integer" + }, + "original_string": { + "type": "string" + }, + "@version": { + "type": "string" + }, + "timestamp": { + "type": "integer" + }, + "type": { + "type": "string" + }, + "DeviceVendor": { + "type": "string" + }, + "DeviceProduct": { + "type": "string" + }, + "DeviceVersion": { + "type": "string" + }, + "DeviceEvent": { + "type": "string" + }, + "Name": { + "type": "string" + }, + "Severity": { + "type": "integer" + }, + "cat": { + "type": "string" + }, + "ServerGroup": { + "type": "string" + }, + "ServiceName": { + "type": "string" + }, + "ApplicationName": { + "type": "string" + }, + "Description": { + "type": "string" + } + }, + "required": ["ip_src_addr", "ip_dst_addr", "ip_src_port", "ip_dst_port", "original_string", "timestamp", + "DeviceVendor", "DeviceProduct", "DeviceVersion", "Name", "Severity", + "cat", + "ServerGroup", "ServiceName", "ApplicationName", "Description"] +} \ No newline at end of file
