[
https://issues.apache.org/jira/browse/METRON-157?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15863133#comment-15863133
]
ASF GitHub Bot commented on METRON-157:
---------------------------------------
Github user james-sirota commented on a diff in the pull request:
https://github.com/apache/incubator-metron/pull/451#discussion_r100716097
--- Diff:
metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/cef/CEFParserTest.java
---
@@ -0,0 +1,186 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.parsers.cef;
+
+import java.net.URL;
+import java.nio.charset.Charset;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
+
+import org.json.simple.JSONObject;
+import org.json.simple.parser.JSONParser;
+import org.json.simple.parser.ParseException;
+import org.junit.Assert;
+import org.junit.Test;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.github.fge.jackson.JsonLoader;
+import com.github.fge.jsonschema.core.report.ProcessingReport;
+import com.github.fge.jsonschema.main.JsonSchemaFactory;
+import com.github.fge.jsonschema.main.JsonValidator;
+import com.google.common.io.Resources;
+
+import junit.framework.TestCase;
+
+public class CEFParserTest extends TestCase {
+
+ private static final Charset UTF_8 = Charset.forName("utf-8");
+ private CEFParser parser;
+
+ @Override
+ public void setUp() {
+ parser = new CEFParser();
+ parser.init();
+ }
+
+ @Test
+ public void testEscaping() {
+ for (JSONObject obj : parse(
+ "Sep 19 08:26:10 host
CEF:0|security|threatmanager|1.0|100|detected a \\ in packet|10|src=10.0.0.1
act=blocked a \\ dst=1.1.1.1")) {
+ assertEquals("10.0.0.1", obj.get("ip_src_addr"));
+ assertEquals("blocked a \\", obj.get("deviceAction"));
+ assertEquals("1.1.1.1", obj.get("ip_dst_addr"));
+ }
+ }
+
+ public void testBasicHeader() {
+ for (JSONObject obj : parse(
+ "CEF:0|Security|threatmanager|1.0|100|worm
successfully stopped|10|src=10.0.0.1 dst=2.1.2.2 spt=1232")) {
+ assertEquals("Security", obj.get("DeviceVendor"));
+ assertEquals("threatmanager", obj.get("DeviceProduct"));
+ assertEquals("1.0", obj.get("DeviceVersion"));
+ assertEquals("100", obj.get("DeviceEvent"));
+ assertEquals("worm successfully stopped",
obj.get("Name"));
+ assertEquals(10, obj.get("Severity"));
+ }
+ }
+
+ public void testBasicExtensions() {
+ for (JSONObject obj : parse(
+ "CEF:0|Security|threatmanager|1.0|100|worm
successfully stopped|10|src=10.0.0.1 dst=2.1.2.2 spt=1232")) {
+ assertEquals("10.0.0.1", obj.get("ip_src_addr"));
+ assertEquals("2.1.2.2", obj.get("ip_dst_addr"));
+ assertEquals(1232, obj.get("ip_src_port"));
+ }
+ }
+
+ public void testCustomLabelWithSpace() {
+ for (JSONObject obj : parse(
+ "CEF:0|Security|threatmanager|1.0|100|worm
successfully stopped|10|src=10.0.0.1 dst=2.1.2.2 spt=1232 custom=Text with
space customLabel=Label with space")) {
+ assertEquals(true, obj.containsKey("Label with space"));
+ assertEquals("Text with space", obj.get("Label with
space"));
+ }
+ }
+
+ public void testTimestampPriority() throws java.text.ParseException {
+ long correctTime = new
SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSz").parse("2016-05-01T09:29:11.356-0400")
+ .getTime();
+
+ SimpleDateFormat sdf = new
SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSz");
+
+ for (JSONObject obj : parse(
+ "CEF:0|Security|threatmanager|1.0|100|worm
successfully stopped|10|src=10.0.0.1 rt=May 1 2016 09:29:11.356 -0400
dst=2.1.2.2 spt=1232")) {
+ assertEquals(new Date(correctTime), new Date((long)
obj.get("timestamp")));
+ assertEquals(correctTime, obj.get("timestamp"));
+ }
+ for (JSONObject obj : parse(
+ "2016-06-01T09:29:11.356-04:00 host
CEF:0|Security|threatmanager|1.0|100|worm successfully stopped|10|src=10.0.0.1
rt=May 1 2016 09:29:11.356 -0400 dst=2.1.2.2 spt=1232")) {
+ assertEquals(new Date(correctTime), new Date((long)
obj.get("timestamp")));
+ assertEquals(correctTime, obj.get("timestamp"));
+ }
+ for (JSONObject obj : parse(
+ "2016-05-01T09:29:11.356-04:00 host
CEF:0|Security|threatmanager|1.0|100|worm successfully stopped|10|src=10.0.0.1
dst=2.1.2.2 spt=1232")) {
+ assertEquals(new Date(correctTime), new Date((long)
obj.get("timestamp")));
+ assertEquals(correctTime, obj.get("timestamp"));
+ }
+ for (JSONObject obj : parse(
+ "CEF:0|Security|threatmanager|1.0|100|worm
successfully stopped|10|src=10.0.0.1 dst=2.1.2.2 spt=1232")) {
+ assertNotNull(obj.get("timestamp"));
+ }
+
+ }
+
+ public void testCEFParserAdallom() throws Exception {
+ runTest("adallom",
Resources.readLines(Resources.getResource(getClass(), "adallom.cef"), UTF_8),
+
Resources.toString(Resources.getResource(getClass(), "adallom.schema"), UTF_8));
+ }
+
+ public void testCEFParserCyberArk() throws Exception {
+ runTest("cyberark",
Resources.readLines(Resources.getResource(getClass(), "cyberark.cef"), UTF_8),
+
Resources.toString(Resources.getResource(getClass(), "cyberark.schema"), UTF_8),
+
Resources.toString(Resources.getResource(getClass(), "cyberark.json"), UTF_8));
+ }
+
+ public void testCEFParserWAF() throws Exception {
+ URL waf_url = Resources.getResource(getClass(), "waf.cef");
+ runTest("waf", Resources.readLines(waf_url, UTF_8),
+
Resources.toString(Resources.getResource(getClass(), "waf.schema"), UTF_8));
+ }
+
+ private void runTest(String name, List<String> lines, String schema)
throws Exception {
+ runTest(name, lines, schema, "");
+ }
+
+ private void runTest(String name, List<String> lines, String schema,
String targetJson) throws Exception {
+ for (String inputString : lines) {
+ JSONObject parsed = parse(inputString).get(0);
+ assertNotNull(parsed);
+ assertNotNull(parsed.get("timestamp"));
+ assertTrue((long) parsed.get("timestamp") > 0);
+
+ System.out.println(parsed);
+ JSONParser parser = new JSONParser();
+
+ Map<?, ?> json = null;
+ try {
+ json = (Map<?, ?>)
parser.parse(parsed.toJSONString());
+ Assert.assertEquals(true,
validateJsonData(schema, json.toString()));
+ } catch (ParseException e) {
+ e.printStackTrace();
+ }
+
+ // test against an explicit json example
+ if (!targetJson.isEmpty()) {
+
+ }
+ }
+ }
+
+ protected boolean validateJsonData(final String jsonSchema, final
String jsonData) throws Exception {
--- End diff --
I like this part. Would be nice to eventually make this global
> Create CEF Parser
> -----------------
>
> Key: METRON-157
> URL: https://issues.apache.org/jira/browse/METRON-157
> Project: Metron
> Issue Type: New Feature
> Reporter: Domenic Puzio
> Priority: Minor
> Labels: platform
>
> Create a parser for CEF (Common Event Format). CEF is a very common
> formatting for security data sources; it is used by FireEye, Adallom, Imperva
> WAF, CyberArk, and others. The parser should be flexible enough to work for
> any of these data sources. CEF uses shorthand field names, so field names
> should be changed to human-readable and Metron-friendly equivalents. CEF
> custom labels (cs1Label, flexString1Label, etc.) should be converted
> appropriately.
> Below are sample messages and their expected parsed output.
> Adallom CEF
> 2016-04-01T09:29:11.356-0400
> CEF:0|Adallom|Adallom|1.0|56fe779ee4b0459f4e9a484a|ALERT_CABINET_EVENT_MATCH_AUDIT|0|msg=Activity
> policy 'User download/view file' was triggered by '[email protected]'
> [email protected] start=1459517280810 end=1459517280810
> audits=["AVPR-4oIPeFmuZ3CKKrg","AVPR-wx80cd9PUpAu2aj","AVPR-6XGPeFmuZ3CKKvx","AVPSALn_qE4Kgs_8_yK9","AVPSASW3gw_f3aEvgEmi"]
> services=["APPID_SXC"] users=["[email protected]"]
> cs6=https://abcd-remote.console.arc.com/#/alerts/56fe779ee4b0459f4e9a484a
> cs6Label=consoleUrl
> ...
> {"source.type":"adallom","device_version":"1.0","severity":"0","device_product":"Adallom","services":"[\"APPID_SXC\"]","src_username":"[email protected]","message":"Activity
> policy 'User download\/view file' was triggered by
> '[email protected]'","users":"[\"[email protected]\"]","consoleUrl":"https:\/\/abcd-remote.console.arc.com\/#\/alerts\/56fe779ee4b0459f4e9a484a","event_class_id":"56fe779ee4b0459f4e9a484a","original_string":"2016-04-01T09:29:11.356-0400
>
> CEF:0|Adallom|Adallom|1.0|56fe779ee4b0459f4e9a484a|ALERT_CABINET_EVENT_MATCH_AUDIT|0|msg=Activity
> policy 'User download\/view file' was triggered by '[email protected]'
> [email protected] start=1459517280810 end=1459517280810
> audits=[\"AVPR-4oIPeFmuZ3CKKrg\",\"AVPR-wx80cd9PUpAu2aj\",\"AVPR-6XGPeFmuZ3CKKvx\",\"AVPSALn_qE4Kgs_8_yK9\",\"AVPSASW3gw_f3aEvgEmi\"]
> services=[\"APPID_SXC\"] users=[\"[email protected]\"]
> cs6=https:\/\/abcd-remote.console.arc.com\/#\/alerts\/56fe779ee4b0459f4e9a484a
> cs6Label=consoleUrl","header":"2016-04-01T09:29:11.356-0400
> CEF:0","event_name":"ALERT_CABINET_EVENT_MATCH_AUDIT","startTime":"1459517280810","device_vendor":"Adallom","endTime":"1459517280810","audits":"[\"AVPR-4oIPeFmuZ3CKKrg\",\"AVPR-wx80cd9PUpAu2aj\",\"AVPR-6XGPeFmuZ3CKKvx\",\"AVPSALn_qE4Kgs_8_yK9\",\"AVPSASW3gw_f3aEvgEmi\"]","timestamp":1459502951000}
> CyberArk CEF
> Mar 21 14:05:02 HHHPVATN1 CEF:0|Cyber-Ark|Vault|7.20.0091|295|Retrieve
> password|5|act=Retrieve password suser=spilgrim fname=Root\ABC phobos3 - COMP
> dvc=120.99.70.3 shost=10.44.134.78 dhost= duser= externalId= app= reason=
> cs1Label="Affected User Name" cs1= cs2Label="Safe Name" cs2=Security
> Vulnerability Mgmt cs3Label="Device Type" cs3= cs4Label="Database" cs4=
> cs5Label="Other info" cs5=101.198.70.93 cn1Label="Request Id" cn1=
> cn2Label="Ticket Id" cn2=Needed to verify config files being pulled
> msg=Needed to verify config files being pulled
> ...
> {"timestamp":1458569102000,"source.type":"cyberark","device_version":"7.20.0091","device_product":"Vault","fileName":"Root\\ABC
> phobos3 - COMP","src_username":"spilgrim","\"Other
> info\"":"101.198.70.93","\"Ticket Id\"":"Needed to verify config files being
> pulled
> ","deviceAddress":"120.99.70.3","severity":"5","deviceAction":"Retrieve
> password","message":"Needed to verify config files being
> pulled","event_class_id":"295","original_string":"Mar 21 14:05:02 HHHPVATN1
> CEF:0|Cyber-Ark|Vault|7.20.0091|295|Retrieve password|5|act=Retrieve password
> suser=spilgrim fname=Root\\ABC phobos3 - COMP dvc=120.99.70.3
> shost=10.44.134.78 dhost= duser= externalId= app= reason= cs1Label=\"Affected
> User Name\" cs1= cs2Label=\"Safe Name\" cs2=Security Vulnerability Mgmt
> cs3Label=\"Device Type\" cs3= cs4Label=\"Database\" cs4= cs5Label=\"Other
> info\" cs5=101.198.70.93 cn1Label=\"Request Id\" cn1= cn2Label=\"Ticket Id\"
> cn2=Needed to verify config files being pulled msg=Needed to verify config
> files being pulled","\"Safe Name\"":"Security Vulnerability
> Mgmt","header":"Mar 21 14:05:02 HHHPVATN1 CEF:0","event_name":"Retrieve
> password","device_vendor":"Cyber-Ark","src_hostname":"10.44.134.78"}
> WAF CEF
> <14>CEF:0|Imperva Inc.|SecureSphere|10.0.0.4_16|ABC - Secure Login.vm Page
> Rate Limit UK - Source IP||High|act=alert dst=17.43.200.42 dpt=88
> duser=${Alert.username} src=10.31.45.69 spt=34435 proto=TCP rt=31 March 2016
> 13:04:55 cat=Alert cs1= cs1Label=Policy cs2=ABC-Secure cs2Label=ServerGroup
> cs3=servers_svc cs3Label=ServiceName cs4=server_app cs4Label=ApplicationName
> cs5=QA cs5Label=Description
> ...
> {"source.type":"waf","device_version":"10.0.0.4_16","severity":"High","device_product":"SecureSphere","ServerGroup":"ABC-Secure","ApplicationName":"server_app","Description":"QA","deviceAction":"alert","ip_dst_port":"88","dst_username":"${Alert.username}","priority":"14","deviceEventCategory":"Alert","protocol":"TCP","ip_dst_addr":"17.43.200.42","ip_src_port":"34435","event_class_id":"ABC
> - Secure Login.vm Page Rate Limit UK - Source
> IP","ServiceName":"servers_svc","original_string":"<14>CEF:0|Imperva
> Inc.|SecureSphere|10.0.0.4_16|ABC - Secure Login.vm Page Rate Limit UK -
> Source IP||High|act=alert dst=17.43.200.42 dpt=88 duser=${Alert.username}
> src=10.31.45.69 spt=34435 proto=TCP rt=31 March 2016 13:04:55 cat=Alert cs1=
> cs1Label=Policy cs2=ABC-Secure cs2Label=ServerGroup cs3=servers_svc
> cs3Label=ServiceName cs4=server_app cs4Label=ApplicationName cs5=QA
> cs5Label=Description","header":"<14>CEF:0","device_vendor":"Imperva
> Inc.","ip_src_addr":"10.31.45.69","timestamp":1459429495000}
--
This message was sent by Atlassian JIRA
(v6.3.15#6346)
