Repository: metron Updated Branches: refs/heads/master 4a089900a -> 995b20170
METRON-1321 Metaalert Threat Score Type Does Not Match Sensor Indices (nickwallen) closes apache/metron#845 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/995b2017 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/995b2017 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/995b2017 Branch: refs/heads/master Commit: 995b20170f460df4fc121fec0f3946857d748426 Parents: 4a08990 Author: nickwallen <[email protected]> Authored: Mon Nov 20 17:39:00 2017 -0500 Committer: nickallen <[email protected]> Committed: Mon Nov 20 17:39:00 2017 -0500 ---------------------------------------------------------------------- .../package/files/metaalert_index.template | 9 ++++ .../dao/ElasticsearchMetaAlertDao.java | 28 +++++++++++-- .../dao/ElasticsearchMetaAlertDaoTest.java | 43 +++++++++++++++----- .../indexing/dao/metaalert/MetaScores.java | 3 +- 4 files changed, 68 insertions(+), 15 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/995b2017/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/metaalert_index.template ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/metaalert_index.template b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/metaalert_index.template index 964a480..5af9251 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/metaalert_index.template +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/metaalert_index.template @@ -15,6 +15,15 @@ "index": "not_analyzed" } } + }, + { + "threat_triage_score": { + "mapping": { + "type": "float" + }, + "match": "threat:triage:*score", + "match_mapping_type": "*" + } } ], "properties": { http://git-wip-us.apache.org/repos/asf/metron/blob/995b2017/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java index 90d5410..f8fb145 100644 --- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java +++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java @@ -61,6 +61,7 @@ import org.apache.metron.indexing.dao.search.SearchResult; import org.apache.metron.indexing.dao.update.Document; import org.apache.metron.indexing.dao.update.OriginalNotFoundException; import org.apache.metron.indexing.dao.update.PatchRequest; +import org.apache.metron.stellar.common.utils.ConversionUtils; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; @@ -77,6 +78,14 @@ public class ElasticsearchMetaAlertDao implements MetaAlertDao { private ElasticsearchDao elasticsearchDao; private String index = METAALERTS_INDEX; private String threatTriageField = THREAT_FIELD_DEFAULT; + + /** + * Defines which summary aggregation is used to represent the overall threat triage score for + * the metaalert. The summary aggregation is applied to the threat triage score of all child alerts. + * + * This overall score is primarily used for sorting; hence it is called the 'threatSort'. This + * can be either max, min, average, count, median, or sum. + */ private String threatSort = THREAT_SORT_DEFAULT; private int pageSize = 500; @@ -92,9 +101,11 @@ public class ElasticsearchMetaAlertDao implements MetaAlertDao { * Wraps an {@link org.apache.metron.indexing.dao.IndexDao} to handle meta alerts. * @param indexDao The Dao to wrap * @param triageLevelField The field name to use as the threat scoring field + * @param threatSort The summary aggregation of all child threat triage scores used + * as the overall threat triage score for the metaalert. This + * can be either max, min, average, count, median, or sum. */ - public ElasticsearchMetaAlertDao(IndexDao indexDao, String index, String triageLevelField, - String threatSort) { + public ElasticsearchMetaAlertDao(IndexDao indexDao, String index, String triageLevelField, String threatSort) { init(indexDao, Optional.of(threatSort)); this.index = index; this.threatTriageField = triageLevelField; @@ -108,7 +119,9 @@ public class ElasticsearchMetaAlertDao implements MetaAlertDao { * Initializes this implementation by setting the supplied IndexDao and also setting a separate ElasticsearchDao. * This is needed for some specific Elasticsearch functions (looking up an index from a GUID for example). * @param indexDao The DAO to wrap for our queries - * @param threatSort The aggregation to use as the threat field. E.g. "sum", "median", etc. + * @param threatSort The summary aggregation of the child threat triage scores used + * as the overall threat triage score for the metaalert. This + * can be either max, min, average, count, median, or sum. */ @Override public void init(IndexDao indexDao, Optional<String> threatSort) { @@ -608,8 +621,15 @@ public class ElasticsearchMetaAlertDao implements MetaAlertDao { } metaScores = new MetaScores(scores); } + + // add a summary (max, min, avg, ...) of all the threat scores from the child alerts metaAlert.getDocument().putAll(metaScores.getMetaScores()); - metaAlert.getDocument().put(threatTriageField, metaScores.getMetaScores().get(threatSort)); + + // add the overall threat score for the metaalert; one of the summary aggregations as defined by `threatSort` + Object threatScore = metaScores.getMetaScores().get(threatSort); + + // add the threat score as a float; type needs to match the threat score field from each of the sensor indices + metaAlert.getDocument().put(threatTriageField, ConversionUtils.convert(threatScore, Float.class)); } private Double parseThreatField(Object threatRaw) { http://git-wip-us.apache.org/repos/asf/metron/blob/995b2017/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDaoTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDaoTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDaoTest.java index ffafe52..1bfa9d6 100644 --- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDaoTest.java +++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDaoTest.java @@ -20,6 +20,7 @@ package org.apache.metron.elasticsearch.dao; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; import java.io.IOException; import java.util.ArrayList; @@ -50,6 +51,8 @@ import org.junit.Test; public class ElasticsearchMetaAlertDaoTest { + + @Test(expected = IllegalArgumentException.class) public void testInvalidInit() { IndexDao dao = new IndexDao() { @@ -207,20 +210,40 @@ public class ElasticsearchMetaAlertDaoTest { @Test public void testCalculateMetaScoresList() { + final double delta = 0.001; List<Map<String, Object>> alertList = new ArrayList<>(); - Map<String, Object> alertMap = new HashMap<>(); - alertMap.put(MetaAlertDao.THREAT_FIELD_DEFAULT, 10.0d); - alertList.add(alertMap); + + // add an alert with a threat score + alertList.add( Collections.singletonMap(MetaAlertDao.THREAT_FIELD_DEFAULT, 10.0f)); + + // add a second alert with a threat score + alertList.add( Collections.singletonMap(MetaAlertDao.THREAT_FIELD_DEFAULT, 20.0f)); + + // add a third alert with NO threat score + alertList.add( Collections.singletonMap("alert3", "has no threat score")); + + // create the metaalert Map<String, Object> docMap = new HashMap<>(); docMap.put(MetaAlertDao.ALERT_FIELD, alertList); + Document metaalert = new Document(docMap, "guid", MetaAlertDao.METAALERT_TYPE, 0L); - Document doc = new Document(docMap, "guid", MetaAlertDao.METAALERT_TYPE, 0L); - + // calculate the threat score for the metaalert ElasticsearchMetaAlertDao metaAlertDao = new ElasticsearchMetaAlertDao(); - metaAlertDao.calculateMetaScores(doc); - assertEquals(1L, doc.getDocument().get("count")); - assertEquals(10.0d, - doc.getDocument().get(ElasticsearchMetaAlertDao.THREAT_FIELD_DEFAULT) - ); + metaAlertDao.calculateMetaScores(metaalert); + Object threatScore = metaalert.getDocument().get(ElasticsearchMetaAlertDao.THREAT_FIELD_DEFAULT); + + // the metaalert must contain a summary of all child threat scores + assertEquals(20D, (Double) metaalert.getDocument().get("max"), delta); + assertEquals(10D, (Double) metaalert.getDocument().get("min"), delta); + assertEquals(15D, (Double) metaalert.getDocument().get("average"), delta); + assertEquals(2L, metaalert.getDocument().get("count")); + assertEquals(30D, (Double) metaalert.getDocument().get("sum"), delta); + assertEquals(15D, (Double) metaalert.getDocument().get("median"), delta); + + // it must contain an overall threat score; a float to match the type of the threat score of the other sensor indices + assertTrue(threatScore instanceof Float); + + // by default, the overall threat score is the sum of all child threat scores + assertEquals(30.0F, threatScore); } } http://git-wip-us.apache.org/repos/asf/metron/blob/995b2017/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaScores.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaScores.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaScores.java index 632cfd2..07285d6 100644 --- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaScores.java +++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaScores.java @@ -18,11 +18,12 @@ package org.apache.metron.indexing.dao.metaalert; +import org.apache.commons.math3.stat.descriptive.rank.Median; + import java.util.DoubleSummaryStatistics; import java.util.HashMap; import java.util.List; import java.util.Map; -import org.apache.commons.math3.stat.descriptive.rank.Median; public class MetaScores {
