Repository: metron Updated Branches: refs/heads/master 3467ffdb1 -> 747220f00
METRON-1811 Alert Search Fails When Sorting by Alert Status (merrimanr) closes apache/metron#1231 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/747220f0 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/747220f0 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/747220f0 Branch: refs/heads/master Commit: 747220f000188557bfa860b48f6eb4bc49758d4a Parents: 3467ffd Author: merrimanr <[email protected]> Authored: Tue Oct 9 15:33:39 2018 -0500 Committer: rmerriman <[email protected]> Committed: Tue Oct 9 15:33:39 2018 -0500 ---------------------------------------------------------------------- .../CURRENT/package/files/bro_index.template | 3 + .../CURRENT/package/files/snort_index.template | 3 + .../CURRENT/package/files/yaf_index.template | 3 + metron-platform/metron-elasticsearch/README.md | 20 +- .../ElasticsearchSearchIntegrationTest.java | 211 ++++++------------- .../indexing/dao/SearchIntegrationTest.java | 34 +-- .../src/main/config/schema/bro/schema.xml | 1 + .../src/main/config/schema/snort/schema.xml | 1 + .../src/main/config/schema/yaf/schema.xml | 1 + .../integration/SolrSearchIntegrationTest.java | 8 +- 10 files changed, 112 insertions(+), 173 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/747220f0/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template index 17ad4d2..48e76c7 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template @@ -133,6 +133,9 @@ "source:type": { "type": "keyword" }, + "alert_status": { + "type": "keyword" + }, /* * Widely-used Bro fields (potentially renamed during Metron ingest) */ http://git-wip-us.apache.org/repos/asf/metron/blob/747220f0/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/snort_index.template ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/snort_index.template b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/snort_index.template index f7c6e59..139dbc9 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/snort_index.template +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/snort_index.template @@ -123,6 +123,9 @@ "source:type": { "type": "keyword" }, + "alert_status": { + "type": "keyword" + }, "ip_dst_addr": { "type": "ip" }, http://git-wip-us.apache.org/repos/asf/metron/blob/747220f0/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/yaf_index.template ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/yaf_index.template b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/yaf_index.template index f4093ba..4a3e5f9 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/yaf_index.template +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/yaf_index.template @@ -123,6 +123,9 @@ "source:type": { "type": "keyword" }, + "alert_status": { + "type": "keyword" + }, "ip_dst_addr": { "type": "ip" }, http://git-wip-us.apache.org/repos/asf/metron/blob/747220f0/metron-platform/metron-elasticsearch/README.md ---------------------------------------------------------------------- diff --git a/metron-platform/metron-elasticsearch/README.md b/metron-platform/metron-elasticsearch/README.md index d889e27..177412e 100644 --- a/metron-platform/metron-elasticsearch/README.md +++ b/metron-platform/metron-elasticsearch/README.md @@ -269,9 +269,27 @@ Notes on other settings for types in ES * [https://www.elastic.co/guide/en/elasticsearch/reference/5.6/breaking_50_mapping_changes.html](https://www.elastic.co/guide/en/elasticsearch/reference/5.6/breaking_50_mapping_changes.html) * [https://www.elastic.co/blog/strings-are-dead-long-live-strings](https://www.elastic.co/blog/strings-are-dead-long-live-strings) +### Metron Properties + +Metron depends on some internal fields being defined in sensor templates. A field is defined in Elasticsearch by adding an entry to the `properties` section of the template: +``` +"properties": { + "metron_field": { + "type": "keyword" + } +} +``` + +The following is a list of properties that need to be defined along with their type: +* source:type - keyword +* alert_status - keyword +* metron_alert - nested + ## Using Metron with Elasticsearch 5.6.2 -There is a requirement that all sensors templates have a nested `metron_alert` field defined. This field is a dummy field. See [Ignoring Unmapped Fields](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-sort.html#_ignoring_unmapped_fields) for more information +Although infrequent sometimes an internal field is added in Metron and existing templates must be updated. The following steps outlines how to do this, using `metron_alert` as an example. + +With the addition of the meta alert feature, there is a requirement that all sensors templates have a nested `metron_alert` field defined. This field is a dummy field. See [Ignoring Unmapped Fields](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-sort.html#_ignoring_unmapped_fields) for more information Without this field, an error will be thrown during ALL searches (including from UIs, resulting in no alerts being found for any sensor). This error will be found in the REST service's logs. http://git-wip-us.apache.org/repos/asf/metron/blob/747220f0/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java index 8071e68..1d2d48e 100644 --- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java +++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java @@ -19,6 +19,7 @@ package org.apache.metron.elasticsearch.integration; import java.io.File; +import java.io.IOException; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -55,136 +56,12 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest { private static String indexDir = "target/elasticsearch_search"; private static String dateFormat = "yyyy.MM.dd.HH"; + private static String broTemplatePath = "../../metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template"; + private static String snortTemplatePath = "../../metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/snort_index.template"; private static final int MAX_RETRIES = 10; private static final int SLEEP_MS = 500; protected static IndexDao dao; - /** - * { - * "bro_doc": { - * "properties": { - * "source:type": { - * "type": "text", - * "fielddata" : "true" - * }, - * "guid" : { - * "type" : "keyword" - * }, - * "ip_src_addr": { - * "type": "ip" - * }, - * "ip_src_port": { - * "type": "integer" - * }, - * "long_field": { - * "type": "long" - * }, - * "timestamp": { - * "type": "date", - * "format": "epoch_millis" - * }, - * "latitude" : { - * "type": "float" - * }, - * "score": { - * "type": "double" - * }, - * "is_alert": { - * "type": "boolean" - * }, - * "location_point": { - * "type": "geo_point" - * }, - * "bro_field": { - * "type": "text", - * "fielddata" : "true" - * }, - * "ttl": { - * "type": "text", - * "fielddata" : "true" - * }, - * "alert": { - * "type": "nested" - * } - * } - * } - * } - */ - @Multiline - private static String broTypeMappings; - - /** - * { - * "snort_doc": { - * "properties": { - * "source:type": { - * "type": "text", - * "fielddata" : "true" - * }, - * "guid" : { - * "type" : "keyword" - * }, - * "ip_src_addr": { - * "type": "ip" - * }, - * "ip_src_port": { - * "type": "integer" - * }, - * "long_field": { - * "type": "long" - * }, - * "timestamp": { - * "type": "date", - * "format": "epoch_millis" - * }, - * "latitude" : { - * "type": "float" - * }, - * "score": { - * "type": "double" - * }, - * "is_alert": { - * "type": "boolean" - * }, - * "location_point": { - * "type": "geo_point" - * }, - * "snort_field": { - * "type": "integer" - * }, - * "ttl": { - * "type": "integer" - * }, - * "alert": { - * "type": "nested" - * }, - * "threat:triage:score": { - * "type": "float" - * } - * } - * } - * } - */ - @Multiline - private static String snortTypeMappings; - - /** - * { - * "bro_doc_default": { - * "dynamic_templates": [{ - * "strings": { - * "match_mapping_type": "string", - * "mapping": { - * "type": "text" - * } - * } - * }] - * } - * } - */ - @Multiline - private static String broDefaultStringMappings; - @BeforeClass public static void setup() throws Exception { indexComponent = startIndex(); @@ -220,13 +97,17 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest { return es; } - protected static void loadTestData() throws ParseException { + protected static void loadTestData() throws ParseException, IOException { ElasticSearchComponent es = (ElasticSearchComponent) indexComponent; + + JSONObject broTemplate = JSONUtils.INSTANCE.load(new File(broTemplatePath), JSONObject.class); + addTestFieldMappings(broTemplate, "bro_doc"); es.getClient().admin().indices().prepareCreate("bro_index_2017.01.01.01") - .addMapping("bro_doc", broTypeMappings) - .addMapping("bro_doc_default", broDefaultStringMappings).get(); + .addMapping("bro_doc", JSONUtils.INSTANCE.toJSON(broTemplate.get("mappings"), false)).get(); + JSONObject snortTemplate = JSONUtils.INSTANCE.load(new File(snortTemplatePath), JSONObject.class); + addTestFieldMappings(snortTemplate, "snort_doc"); es.getClient().admin().indices().prepareCreate("snort_index_2017.01.01.02") - .addMapping("snort_doc", snortTypeMappings).get(); + .addMapping("snort_doc", JSONUtils.INSTANCE.toJSON(snortTemplate.get("mappings"), false)).get(); BulkRequestBuilder bulkRequest = es.getClient().prepareBulk() .setRefreshPolicy(WriteRequest.RefreshPolicy.WAIT_UNTIL); @@ -258,6 +139,27 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest { } } + /** + * Add test fields to a template with defined types in case they are not defined in the sensor template shipped with Metron. + * This is useful for testing certain cases, for example faceting on fields of various types. + * @param template + * @param docType + */ + private static void addTestFieldMappings(JSONObject template, String docType) { + Map mappings = (Map) template.get("mappings"); + Map docTypeJSON = (Map) mappings.get(docType); + Map properties = (Map) docTypeJSON.get("properties"); + Map<String, String> longType = new HashMap<>(); + longType.put("type", "long"); + properties.put("long_field", longType); + Map<String, String> floatType = new HashMap<>(); + floatType.put("type", "float"); + properties.put("latitude", floatType); + Map<String, String> doubleType = new HashMap<>(); + doubleType.put("type", "double"); + properties.put("score", doubleType); + } + @Test public void bad_facet_query_throws_exception() throws Exception { thrown.expect(InvalidSearchException.class); @@ -273,11 +175,11 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest { // getColumnMetadata with only bro { Map<String, FieldType> fieldTypes = dao.getColumnMetadata(Collections.singletonList("bro")); - Assert.assertEquals(13, fieldTypes.size()); - Assert.assertEquals(FieldType.TEXT, fieldTypes.get("bro_field")); - Assert.assertEquals(FieldType.TEXT, fieldTypes.get("ttl")); + Assert.assertEquals(262, fieldTypes.size()); + Assert.assertEquals(FieldType.KEYWORD, fieldTypes.get("method")); + Assert.assertEquals(FieldType.KEYWORD, fieldTypes.get("ttl")); Assert.assertEquals(FieldType.KEYWORD, fieldTypes.get("guid")); - Assert.assertEquals(FieldType.TEXT, fieldTypes.get("source:type")); + Assert.assertEquals(FieldType.KEYWORD, fieldTypes.get("source:type")); Assert.assertEquals(FieldType.IP, fieldTypes.get("ip_src_addr")); Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("ip_src_port")); Assert.assertEquals(FieldType.LONG, fieldTypes.get("long_field")); @@ -285,19 +187,18 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest { Assert.assertEquals(FieldType.FLOAT, fieldTypes.get("latitude")); Assert.assertEquals(FieldType.DOUBLE, fieldTypes.get("score")); Assert.assertEquals(FieldType.BOOLEAN, fieldTypes.get("is_alert")); - Assert.assertEquals(FieldType.OTHER, fieldTypes.get("location_point")); - Assert.assertEquals(FieldType.TEXT, fieldTypes.get("bro_field")); - Assert.assertEquals(FieldType.TEXT, fieldTypes.get("ttl")); - Assert.assertEquals(FieldType.OTHER, fieldTypes.get("alert")); + Assert.assertEquals(FieldType.TEXT, fieldTypes.get("location_point")); + Assert.assertEquals(FieldType.KEYWORD, fieldTypes.get("ttl")); + Assert.assertEquals(FieldType.OTHER, fieldTypes.get("metron_alert")); } // getColumnMetadata with only snort { Map<String, FieldType> fieldTypes = dao.getColumnMetadata(Collections.singletonList("snort")); - Assert.assertEquals(14, fieldTypes.size()); - Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("snort_field")); + Assert.assertEquals(32, fieldTypes.size()); + Assert.assertEquals(FieldType.KEYWORD, fieldTypes.get("sig_generator")); Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("ttl")); Assert.assertEquals(FieldType.KEYWORD, fieldTypes.get("guid")); - Assert.assertEquals(FieldType.TEXT, fieldTypes.get("source:type")); + Assert.assertEquals(FieldType.KEYWORD, fieldTypes.get("source:type")); Assert.assertEquals(FieldType.IP, fieldTypes.get("ip_src_addr")); Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("ip_src_port")); Assert.assertEquals(FieldType.LONG, fieldTypes.get("long_field")); @@ -305,32 +206,40 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest { Assert.assertEquals(FieldType.FLOAT, fieldTypes.get("latitude")); Assert.assertEquals(FieldType.DOUBLE, fieldTypes.get("score")); Assert.assertEquals(FieldType.BOOLEAN, fieldTypes.get("is_alert")); - Assert.assertEquals(FieldType.OTHER, fieldTypes.get("location_point")); + Assert.assertEquals(FieldType.TEXT, fieldTypes.get("location_point")); Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("ttl")); - Assert.assertEquals(FieldType.OTHER, fieldTypes.get("alert")); + Assert.assertEquals(FieldType.OTHER, fieldTypes.get("metron_alert")); } } @Override public void returns_column_data_for_multiple_indices() throws Exception { Map<String, FieldType> fieldTypes = dao.getColumnMetadata(Arrays.asList("bro", "snort")); - Assert.assertEquals(15, fieldTypes.size()); + Assert.assertEquals(277, fieldTypes.size()); + + // Ensure internal Metron fields are properly defined Assert.assertEquals(FieldType.KEYWORD, fieldTypes.get("guid")); - Assert.assertEquals(FieldType.TEXT, fieldTypes.get("source:type")); + Assert.assertEquals(FieldType.KEYWORD, fieldTypes.get("source:type")); + Assert.assertEquals(FieldType.FLOAT, fieldTypes.get("threat:triage:score")); + Assert.assertEquals(FieldType.KEYWORD, fieldTypes.get("alert_status")); + Assert.assertEquals(FieldType.OTHER, fieldTypes.get("metron_alert")); + Assert.assertEquals(FieldType.IP, fieldTypes.get("ip_src_addr")); Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("ip_src_port")); Assert.assertEquals(FieldType.LONG, fieldTypes.get("long_field")); Assert.assertEquals(FieldType.DATE, fieldTypes.get("timestamp")); Assert.assertEquals(FieldType.FLOAT, fieldTypes.get("latitude")); Assert.assertEquals(FieldType.DOUBLE, fieldTypes.get("score")); + Assert.assertEquals(FieldType.DOUBLE, fieldTypes.get("suppress_for")); Assert.assertEquals(FieldType.BOOLEAN, fieldTypes.get("is_alert")); - Assert.assertEquals(FieldType.OTHER, fieldTypes.get("location_point")); - Assert.assertEquals(FieldType.TEXT, fieldTypes.get("bro_field")); - Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("snort_field")); - //NOTE: This is because the field is in both bro and snort and they have different types. + + // Ensure a field defined only in bro is included + Assert.assertEquals(FieldType.KEYWORD, fieldTypes.get("method")); + // Ensure a field defined only in snort is included + Assert.assertEquals(FieldType.KEYWORD, fieldTypes.get("sig_generator")); + // Ensure fields in both bro and snort have type OTHER because they have different types Assert.assertEquals(FieldType.OTHER, fieldTypes.get("ttl")); - Assert.assertEquals(FieldType.FLOAT, fieldTypes.get("threat:triage:score")); - Assert.assertEquals(FieldType.OTHER, fieldTypes.get("alert")); + Assert.assertEquals(FieldType.OTHER, fieldTypes.get("msg")); } @Test http://git-wip-us.apache.org/repos/asf/metron/blob/747220f0/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java index 2e1968a..cfe5752 100644 --- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java +++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java @@ -46,11 +46,11 @@ import org.junit.rules.ExpectedException; public abstract class SearchIntegrationTest { /** * [ - * {"source:type": "bro", "ip_src_addr":"192.168.1.1", "ip_src_port": 8010, "long_field": 10000, "timestamp":1, "latitude": 48.5839, "score": 10.0, "is_alert":true, "location_point": "48.5839,7.7455", "bro_field": "bro data 1", "ttl": "data 1", "guid":"bro_1"}, - * {"source:type": "bro", "ip_src_addr":"192.168.1.2", "ip_src_port": 8009, "long_field": 20000, "timestamp":2, "latitude": 48.0001, "score": 50.0, "is_alert":false, "location_point": "48.5839,7.7455", "bro_field": "bro data 2", "ttl": "data 2", "guid":"bro_2"}, - * {"source:type": "bro", "ip_src_addr":"192.168.1.3", "ip_src_port": 8008, "long_field": 10000, "timestamp":3, "latitude": 48.5839, "score": 20.0, "is_alert":true, "location_point": "50.0,7.7455", "bro_field": "bro data 3", "ttl": "data 3", "guid":"bro_3"}, - * {"source:type": "bro", "ip_src_addr":"192.168.1.4", "ip_src_port": 8007, "long_field": 10000, "timestamp":4, "latitude": 48.5839, "score": 10.0, "is_alert":true, "location_point": "48.5839,7.7455", "bro_field": "bro data 4", "ttl": "data 4", "guid":"bro_4"}, - * {"source:type": "bro", "ip_src_addr":"192.168.1.5", "ip_src_port": 8006, "long_field": 10000, "timestamp":5, "latitude": 48.5839, "score": 98.0, "is_alert":true, "location_point": "48.5839,7.7455", "bro_field": "bro data 5", "ttl": "data 5", "guid":"bro_5"} + * {"source:type": "bro", "ip_src_addr":"192.168.1.1", "ip_src_port": 8010, "long_field": 10000, "timestamp":1, "latitude": 48.5839, "score": 10.0, "is_alert":true, "location_point": "48.5839,7.7455", "method": "bro data 1", "ttl": "data 1", "guid":"bro_1"}, + * {"source:type": "bro", "ip_src_addr":"192.168.1.2", "ip_src_port": 8009, "long_field": 20000, "timestamp":2, "latitude": 48.0001, "score": 50.0, "is_alert":false, "location_point": "48.5839,7.7455", "method": "bro data 2", "ttl": "data 2", "guid":"bro_2"}, + * {"source:type": "bro", "ip_src_addr":"192.168.1.3", "ip_src_port": 8008, "long_field": 10000, "timestamp":3, "latitude": 48.5839, "score": 20.0, "is_alert":true, "location_point": "50.0,7.7455", "method": "bro data 3", "ttl": "data 3", "guid":"bro_3"}, + * {"source:type": "bro", "ip_src_addr":"192.168.1.4", "ip_src_port": 8007, "long_field": 10000, "timestamp":4, "latitude": 48.5839, "score": 10.0, "is_alert":true, "location_point": "48.5839,7.7455", "method": "bro data 4", "ttl": "data 4", "guid":"bro_4"}, + * {"source:type": "bro", "ip_src_addr":"192.168.1.5", "ip_src_port": 8006, "long_field": 10000, "timestamp":5, "latitude": 48.5839, "score": 98.0, "is_alert":true, "location_point": "48.5839,7.7455", "method": "bro data 5", "ttl": "data 5", "guid":"bro_5"} * ] */ @Multiline @@ -58,11 +58,11 @@ public abstract class SearchIntegrationTest { /** * [ - * {"source:type": "snort", "ip_src_addr":"192.168.1.6", "ip_src_port": 8005, "long_field": 10000, "timestamp":6, "latitude": 48.5839, "score": 50.0, "is_alert":false, "location_point": "50.0,7.7455", "snort_field": 10, "ttl": 1, "guid":"snort_1", "threat:triage:score":10.0}, - * {"source:type": "snort", "ip_src_addr":"192.168.1.1", "ip_src_port": 8004, "long_field": 10000, "timestamp":7, "latitude": 48.5839, "score": 10.0, "is_alert":true, "location_point": "48.5839,7.7455", "snort_field": 20, "ttl": 2, "guid":"snort_2", "threat:triage:score":20.0}, - * {"source:type": "snort", "ip_src_addr":"192.168.1.7", "ip_src_port": 8003, "long_field": 10000, "timestamp":8, "latitude": 48.5839, "score": 20.0, "is_alert":false, "location_point": "48.5839,7.7455", "snort_field": 30, "ttl": 3, "guid":"snort_3"}, - * {"source:type": "snort", "ip_src_addr":"192.168.1.1", "ip_src_port": 8002, "long_field": 20000, "timestamp":9, "latitude": 48.0001, "score": 50.0, "is_alert":true, "location_point": "48.5839,7.7455", "snort_field": 40, "ttl": 4, "guid":"snort_4"}, - * {"source:type": "snort", "ip_src_addr":"192.168.1.8", "ip_src_port": 8001, "long_field": 10000, "timestamp":10, "latitude": 48.5839, "score": 10.0, "is_alert":false, "location_point": "48.5839,7.7455", "snort_field": 50, "ttl": 5, "guid":"snort_5"} + * {"source:type": "snort", "ip_src_addr":"192.168.1.6", "ip_src_port": 8005, "long_field": 10000, "timestamp":6, "latitude": 48.5839, "score": 50.0, "is_alert":false, "location_point": "50.0,7.7455", "sig_generator": "sig_generator 1", "ttl": 1, "guid":"snort_1", "threat:triage:score":10.0}, + * {"source:type": "snort", "ip_src_addr":"192.168.1.1", "ip_src_port": 8004, "long_field": 10000, "timestamp":7, "latitude": 48.5839, "score": 10.0, "is_alert":true, "location_point": "48.5839,7.7455", "sig_generator": "sig_generator 2", "ttl": 2, "guid":"snort_2", "threat:triage:score":20.0}, + * {"source:type": "snort", "ip_src_addr":"192.168.1.7", "ip_src_port": 8003, "long_field": 10000, "timestamp":8, "latitude": 48.5839, "score": 20.0, "is_alert":false, "location_point": "48.5839,7.7455", "sig_generator": "sig_generator 3", "ttl": 3, "guid":"snort_3"}, + * {"source:type": "snort", "ip_src_addr":"192.168.1.1", "ip_src_port": 8002, "long_field": 20000, "timestamp":9, "latitude": 48.0001, "score": 50.0, "is_alert":true, "location_point": "48.5839,7.7455", "sig_generator": "sig_generator 4", "ttl": 4, "guid":"snort_4"}, + * {"source:type": "snort", "ip_src_addr":"192.168.1.8", "ip_src_port": 8001, "long_field": 10000, "timestamp":10, "latitude": 48.5839, "score": 10.0, "is_alert":false, "location_point": "48.5839,7.7455", "sig_generator": "sig_generator 5", "ttl": 5, "guid":"snort_5"} * ] */ @Multiline @@ -272,7 +272,7 @@ public abstract class SearchIntegrationTest { /** * { - * "facetFields": ["snort_field"], + * "facetFields": ["sig_generator"], * "indices": ["bro", "snort"], * "query": "*:*", * "from": 0, @@ -698,14 +698,14 @@ public abstract class SearchIntegrationTest { Map<String, Map<String, Long>> facetCounts = response.getFacetCounts(); Assert.assertEquals(1, facetCounts.size()); - Map<String, Long> snortFieldCounts = facetCounts.get("snort_field"); + Map<String, Long> snortFieldCounts = facetCounts.get("sig_generator"); Assert.assertEquals(5, snortFieldCounts.size()); - Assert.assertEquals(1L, snortFieldCounts.get("50").longValue()); - Assert.assertEquals(1L, snortFieldCounts.get("40").longValue()); - Assert.assertEquals(1L, snortFieldCounts.get("30").longValue()); - Assert.assertEquals(1L, snortFieldCounts.get("20").longValue()); - Assert.assertEquals(1L, snortFieldCounts.get("10").longValue()); + Assert.assertEquals(1L, snortFieldCounts.get("sig_generator 5").longValue()); + Assert.assertEquals(1L, snortFieldCounts.get("sig_generator 4").longValue()); + Assert.assertEquals(1L, snortFieldCounts.get("sig_generator 3").longValue()); + Assert.assertEquals(1L, snortFieldCounts.get("sig_generator 2").longValue()); + Assert.assertEquals(1L, snortFieldCounts.get("sig_generator 1").longValue()); response.getFacetCounts(); } http://git-wip-us.apache.org/repos/asf/metron/blob/747220f0/metron-platform/metron-solr/src/main/config/schema/bro/schema.xml ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/main/config/schema/bro/schema.xml b/metron-platform/metron-solr/src/main/config/schema/bro/schema.xml index ea9f6d3..6be76a0 100644 --- a/metron-platform/metron-solr/src/main/config/schema/bro/schema.xml +++ b/metron-platform/metron-solr/src/main/config/schema/bro/schema.xml @@ -34,6 +34,7 @@ * Metron-specific fields --> <field name="source.type" type="string" indexed="true" stored="true" /> + <field name="alert_status" type="string" indexed="true" stored="true" /> <field name="timestamp" type="timestamp" indexed="true" stored="true" /> <field name="guid" type="string" indexed="true" stored="true" required="true" multiValued="false" /> <uniqueKey>guid</uniqueKey> http://git-wip-us.apache.org/repos/asf/metron/blob/747220f0/metron-platform/metron-solr/src/main/config/schema/snort/schema.xml ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/main/config/schema/snort/schema.xml b/metron-platform/metron-solr/src/main/config/schema/snort/schema.xml index 84855df..3c57574 100644 --- a/metron-platform/metron-solr/src/main/config/schema/snort/schema.xml +++ b/metron-platform/metron-solr/src/main/config/schema/snort/schema.xml @@ -22,6 +22,7 @@ <!-- Metron specific fields --> <field name="timestamp" type="timestamp" indexed="true" stored="true" /> <field name="source.type" type="string" indexed="true" stored="true" /> + <field name="alert_status" type="string" indexed="true" stored="true" /> <field name="guid" type="string" indexed="true" stored="true" required="true" multiValued="false" /> <uniqueKey>guid</uniqueKey> http://git-wip-us.apache.org/repos/asf/metron/blob/747220f0/metron-platform/metron-solr/src/main/config/schema/yaf/schema.xml ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/main/config/schema/yaf/schema.xml b/metron-platform/metron-solr/src/main/config/schema/yaf/schema.xml index 5555a14..37e5f12 100644 --- a/metron-platform/metron-solr/src/main/config/schema/yaf/schema.xml +++ b/metron-platform/metron-solr/src/main/config/schema/yaf/schema.xml @@ -22,6 +22,7 @@ <!-- Metron specific fields --> <field name="timestamp" type="timestamp" indexed="true" stored="true" /> <field name="source.type" type="string" indexed="true" stored="true" /> + <field name="alert_status" type="string" indexed="true" stored="true" /> <field name="guid" type="string" indexed="true" stored="true" required="true" multiValued="false" /> <uniqueKey>guid</uniqueKey> http://git-wip-us.apache.org/repos/asf/metron/blob/747220f0/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrSearchIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrSearchIntegrationTest.java b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrSearchIntegrationTest.java index 4390fd1..84f2222 100644 --- a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrSearchIntegrationTest.java +++ b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrSearchIntegrationTest.java @@ -133,7 +133,7 @@ public class SolrSearchIntegrationTest extends SearchIntegrationTest { Assert.assertEquals(FieldType.OTHER, fieldTypes.get("timestamp")); // Bro only field in the dynamic catch all - Assert.assertEquals(FieldType.OTHER, fieldTypes.get("bro_field")); + Assert.assertEquals(FieldType.TEXT, fieldTypes.get("method")); // A field is in both bro and snort and they have different types. Assert.assertEquals(FieldType.TEXT, fieldTypes.get("ttl")); @@ -169,7 +169,7 @@ public class SolrSearchIntegrationTest extends SearchIntegrationTest { Assert.assertEquals(FieldType.OTHER, fieldTypes.get("timestamp")); // Snort only field in the dynamic catch all - Assert.assertEquals(FieldType.OTHER, fieldTypes.get("snort_field")); + Assert.assertEquals(FieldType.TEXT, fieldTypes.get("sig_generator")); // A field is in both bro and snort and they have different types. Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("ttl")); @@ -211,10 +211,10 @@ public class SolrSearchIntegrationTest extends SearchIntegrationTest { Assert.assertEquals(FieldType.OTHER, fieldTypes.get("timestamp")); // Bro only field in the dynamic catch all - Assert.assertEquals(FieldType.OTHER, fieldTypes.get("bro_field")); + Assert.assertEquals(FieldType.TEXT, fieldTypes.get("method")); // Snort only field in the dynamic catch all - Assert.assertEquals(FieldType.OTHER, fieldTypes.get("snort_field")); + Assert.assertEquals(FieldType.TEXT, fieldTypes.get("sig_generator")); // A field is in both bro and snort and they have different types. Assert.assertEquals(FieldType.OTHER, fieldTypes.get("ttl"));
