METRON-1608 Add configuration for threat.triage.field name (merrimanr) closes apache/metron#1055
Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/32d132b3 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/32d132b3 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/32d132b3 Branch: refs/heads/feature/METRON-1554-pcap-query-panel Commit: 32d132b3f8ac7d915180882934d38504263d86ae Parents: 8202cd2 Author: merrimanr <merrim...@gmail.com> Authored: Mon Jun 11 08:31:51 2018 -0500 Committer: merrimanr <merrim...@apache.org> Committed: Mon Jun 11 08:31:51 2018 -0500 ---------------------------------------------------------------------- .../rest/service/impl/SearchServiceImpl.java | 10 +-- .../service/impl/SearchServiceImplTest.java | 3 +- .../org/apache/metron/common/Constants.java | 1 + .../configuration/ConfigurationsUtils.java | 7 ++ .../elasticsearch/dao/ElasticsearchDao.java | 8 ++ .../dao/ElasticsearchMetaAlertDao.java | 78 ++++++++++---------- .../dao/ElasticsearchMetaAlertDaoTest.java | 33 ++++++++- .../ElasticsearchMetaAlertIntegrationTest.java | 64 ++++++++++++++-- .../metron/indexing/dao/MetaAlertDao.java | 3 +- 9 files changed, 150 insertions(+), 57 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/32d132b3/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SearchServiceImpl.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SearchServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SearchServiceImpl.java index 21d158f..1c92fcb 100644 --- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SearchServiceImpl.java +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SearchServiceImpl.java @@ -18,6 +18,7 @@ package org.apache.metron.rest.service.impl; import static org.apache.metron.common.Constants.ERROR_TYPE; +import static org.apache.metron.common.Constants.SENSOR_TYPE_FIELD_PROPERTY; import static org.apache.metron.indexing.dao.MetaAlertDao.METAALERT_TYPE; import static org.apache.metron.rest.MetronRestConstants.INDEX_WRITER_NAME; import static org.apache.metron.rest.MetronRestConstants.SEARCH_FACET_FIELDS_SPRING_PROPERTY; @@ -31,6 +32,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; import org.apache.metron.common.Constants; +import org.apache.metron.common.configuration.ConfigurationsUtils; import org.apache.metron.indexing.dao.IndexDao; import org.apache.metron.indexing.dao.search.FieldType; import org.apache.metron.indexing.dao.search.GetRequest; @@ -146,13 +148,9 @@ public class SearchServiceImpl implements SearchService { if (!alertUserSettings.isPresent() || alertUserSettings.get().getFacetFields() == null) { String facetFieldsProperty = environment .getProperty(SEARCH_FACET_FIELDS_SPRING_PROPERTY, String.class, ""); - - Map<String, Object> globalConfig = globalConfigService.get(); - String sourceTypeField = Constants.SENSOR_TYPE.replace('.', ':'); + String sourceTypeField = ConfigurationsUtils.getFieldName(globalConfigService.get(), SENSOR_TYPE_FIELD_PROPERTY, + Constants.SENSOR_TYPE.replace('.', ':')); List<String> facetFields = new ArrayList<>(); - if (globalConfig != null) { - sourceTypeField = (String) globalConfig.getOrDefault("source.type.field", sourceTypeField); - } facetFields.add(sourceTypeField); if (facetFieldsProperty != null) { facetFields.addAll(Arrays.asList(facetFieldsProperty.split(","))); http://git-wip-us.apache.org/repos/asf/metron/blob/32d132b3/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SearchServiceImplTest.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SearchServiceImplTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SearchServiceImplTest.java index 4c63c6d..82e7221 100644 --- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SearchServiceImplTest.java +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SearchServiceImplTest.java @@ -17,6 +17,7 @@ */ package org.apache.metron.rest.service.impl; +import static org.apache.metron.common.Constants.SENSOR_TYPE_FIELD_PROPERTY; import static org.apache.metron.rest.MetronRestConstants.INDEX_WRITER_NAME; import static org.apache.metron.rest.MetronRestConstants.SEARCH_FACET_FIELDS_SPRING_PROPERTY; import static org.junit.Assert.assertEquals; @@ -183,7 +184,7 @@ public class SearchServiceImplTest { when(environment.getProperty(SEARCH_FACET_FIELDS_SPRING_PROPERTY, String.class, "")) .thenReturn("ip_src_addr"); Map<String, Object> globalConfig = new HashMap<>(); - globalConfig.put("source.type.field", "source.type"); + globalConfig.put(SENSOR_TYPE_FIELD_PROPERTY, "source.type"); when(globalConfigService.get()).thenReturn(globalConfig); when(alertsUIService.getAlertsUIUserSettings()).thenReturn(Optional.empty()); List<String> defaultFields = searchService.getDefaultFacetFields(); http://git-wip-us.apache.org/repos/asf/metron/blob/32d132b3/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java index 12b541c..f74660c 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java @@ -27,6 +27,7 @@ public class Constants { public static final String ZOOKEEPER_TOPOLOGY_ROOT = ZOOKEEPER_ROOT + "/topology"; public static final long DEFAULT_CONFIGURED_BOLT_TIMEOUT = 5000; public static final String SENSOR_TYPE = "source.type"; + public static final String SENSOR_TYPE_FIELD_PROPERTY = "source.type.field"; public static final String ENRICHMENT_TOPIC = "enrichments"; public static final String INDEXING_TOPIC = "indexing"; public static final String ERROR_STREAM = "error"; http://git-wip-us.apache.org/repos/asf/metron/blob/32d132b3/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationsUtils.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationsUtils.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationsUtils.java index c7b39f0..4550a41 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationsUtils.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationsUtils.java @@ -799,5 +799,12 @@ public class ConfigurationsUtils { out.println(type + " Config: " + name + System.lineSeparator() + data); }, configType, configName); } + + public static String getFieldName(Map<String, Object> globalConfig, String globalConfigKey, String defaultFieldName) { + if (globalConfig == null) { + return defaultFieldName; + } + return (String) globalConfig.getOrDefault(globalConfigKey, defaultFieldName); + } } http://git-wip-us.apache.org/repos/asf/metron/blob/32d132b3/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java index cb5bb58..181cb87 100644 --- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java +++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java @@ -125,6 +125,14 @@ public class ElasticsearchDao implements IndexDao { //uninitialized. } + public AccessConfig getAccessConfig() { + return accessConfig; + } + + public void setAccessConfig(AccessConfig accessConfig) { + this.accessConfig = accessConfig; + } + private static Map<String, FieldType> elasticsearchSearchTypeMap; static { http://git-wip-us.apache.org/repos/asf/metron/blob/32d132b3/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java index d12e40c..ee3ca89 100644 --- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java +++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java @@ -18,21 +18,9 @@ package org.apache.metron.elasticsearch.dao; -import static org.apache.metron.common.Constants.GUID; -import static org.elasticsearch.index.query.QueryBuilders.boolQuery; -import static org.elasticsearch.index.query.QueryBuilders.constantScoreQuery; -import static org.elasticsearch.index.query.QueryBuilders.existsQuery; -import static org.elasticsearch.index.query.QueryBuilders.nestedQuery; -import static org.elasticsearch.index.query.QueryBuilders.termQuery; - -import com.fasterxml.jackson.databind.JsonNode; -import java.io.IOException; -import java.util.*; -import java.util.Map.Entry; -import java.util.stream.Collectors; -import org.apache.commons.collections4.SetUtils; import org.apache.lucene.search.join.ScoreMode; import org.apache.metron.common.Constants; +import org.apache.metron.common.configuration.ConfigurationsUtils; import org.apache.metron.indexing.dao.AccessConfig; import org.apache.metron.indexing.dao.IndexDao; import org.apache.metron.indexing.dao.MetaAlertDao; @@ -51,40 +39,48 @@ import org.apache.metron.indexing.dao.search.SearchRequest; import org.apache.metron.indexing.dao.search.SearchResponse; import org.apache.metron.indexing.dao.search.SearchResult; import org.apache.metron.indexing.dao.update.Document; -import org.elasticsearch.action.get.GetResponse; -import org.elasticsearch.action.get.MultiGetItemResponse; -import org.elasticsearch.action.get.MultiGetRequest.Item; -import org.elasticsearch.action.get.MultiGetRequestBuilder; -import org.elasticsearch.action.get.MultiGetResponse; -import org.elasticsearch.action.index.IndexRequest; -import org.elasticsearch.action.search.SearchRequestBuilder; -import org.elasticsearch.action.support.replication.ReplicationResponse.ShardInfo; -import org.elasticsearch.action.update.UpdateRequest; -import org.elasticsearch.action.update.UpdateResponse; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.index.query.InnerHitBuilder; -import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.index.query.QueryStringQueryBuilder; -import org.elasticsearch.search.SearchHit; import org.apache.metron.indexing.dao.update.OriginalNotFoundException; import org.apache.metron.indexing.dao.update.PatchRequest; import org.apache.metron.stellar.common.utils.ConversionUtils; import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.index.query.InnerHitBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.query.QueryStringQueryBuilder; +import org.elasticsearch.search.SearchHit; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; +import java.util.stream.Collectors; + +import static org.apache.metron.common.Constants.GUID; +import static org.apache.metron.common.Constants.SENSOR_TYPE_FIELD_PROPERTY; +import static org.elasticsearch.index.query.QueryBuilders.boolQuery; +import static org.elasticsearch.index.query.QueryBuilders.constantScoreQuery; +import static org.elasticsearch.index.query.QueryBuilders.existsQuery; +import static org.elasticsearch.index.query.QueryBuilders.nestedQuery; +import static org.elasticsearch.index.query.QueryBuilders.termQuery; public class ElasticsearchMetaAlertDao implements MetaAlertDao { public static final String SOURCE_TYPE = Constants.SENSOR_TYPE.replace('.', ':'); + public static final String THREAT_TRIAGE_FIELD = THREAT_FIELD_DEFAULT.replace('.', ':'); private static final String STATUS_PATH = "/status"; private static final String ALERT_PATH = "/alert"; private IndexDao indexDao; private ElasticsearchDao elasticsearchDao; private String index = METAALERTS_INDEX; - private String threatTriageField = THREAT_FIELD_DEFAULT; /** * Defines which summary aggregation is used to represent the overall threat triage score for @@ -101,21 +97,19 @@ public class ElasticsearchMetaAlertDao implements MetaAlertDao { * @param indexDao The Dao to wrap */ public ElasticsearchMetaAlertDao(IndexDao indexDao) { - this(indexDao, METAALERTS_INDEX, THREAT_FIELD_DEFAULT, THREAT_SORT_DEFAULT); + this(indexDao, METAALERTS_INDEX, THREAT_SORT_DEFAULT); } /** * Wraps an {@link org.apache.metron.indexing.dao.IndexDao} to handle meta alerts. * @param indexDao The Dao to wrap - * @param triageLevelField The field name to use as the threat scoring field * @param threatSort The summary aggregation of all child threat triage scores used * as the overall threat triage score for the metaalert. This * can be either max, min, average, count, median, or sum. */ - public ElasticsearchMetaAlertDao(IndexDao indexDao, String index, String triageLevelField, String threatSort) { + public ElasticsearchMetaAlertDao(IndexDao indexDao, String index, String threatSort) { init(indexDao, Optional.of(threatSort)); this.index = index; - this.threatTriageField = triageLevelField; } public ElasticsearchMetaAlertDao() { @@ -196,7 +190,7 @@ public class ElasticsearchMetaAlertDao implements MetaAlertDao { Document metaAlert = buildCreateDocument(alerts, request.getGroups()); calculateMetaScores(metaAlert); // Add source type to be consistent with other sources and allow filtering - metaAlert.getDocument().put(SOURCE_TYPE, MetaAlertDao.METAALERT_TYPE); + metaAlert.getDocument().put(getFieldName(SENSOR_TYPE_FIELD_PROPERTY, SOURCE_TYPE), MetaAlertDao.METAALERT_TYPE); // Start a list of updates / inserts we need to run Map<Document, Optional<String>> updates = new HashMap<>(); @@ -353,7 +347,7 @@ public class ElasticsearchMetaAlertDao implements MetaAlertDao { List<Map<String, Object>> currentAlerts = (List<Map<String, Object>>) metaAlert.getDocument() .get(MetaAlertDao.ALERT_FIELD); currentAlerts.stream().forEach(currentAlert -> { - getRequests.add(new GetRequest((String) currentAlert.get(GUID), (String) currentAlert.get(SOURCE_TYPE))); + getRequests.add(new GetRequest((String) currentAlert.get(GUID), (String) currentAlert.get(getFieldName(SENSOR_TYPE_FIELD_PROPERTY, SOURCE_TYPE)))); }); Iterable<Document> alerts = indexDao.getAllLatest(getRequests); List<Map<String, Object>> updatedAlerts = new ArrayList<>(); @@ -685,7 +679,7 @@ public class ElasticsearchMetaAlertDao implements MetaAlertDao { ArrayList<Double> scores = new ArrayList<>(); for (Object alertRaw : alertsRaw) { Map<String, Object> alert = (Map<String, Object>) alertRaw; - Double scoreNum = parseThreatField(alert.get(threatTriageField)); + Double scoreNum = parseThreatField(alert.get(getFieldName(THREAT_FIELD_PROPERTY, THREAT_TRIAGE_FIELD))); if (scoreNum != null) { scores.add(scoreNum); } @@ -700,7 +694,7 @@ public class ElasticsearchMetaAlertDao implements MetaAlertDao { Object threatScore = metaScores.getMetaScores().get(threatSort); // add the threat score as a float; type needs to match the threat score field from each of the sensor indices - metaAlert.getDocument().put(threatTriageField, ConversionUtils.convert(threatScore, Float.class)); + metaAlert.getDocument().put(getFieldName(THREAT_FIELD_PROPERTY, THREAT_TRIAGE_FIELD), ConversionUtils.convert(threatScore, Float.class)); } private Double parseThreatField(Object threatRaw) { @@ -720,4 +714,12 @@ public class ElasticsearchMetaAlertDao implements MetaAlertDao { public void setPageSize(int pageSize) { this.pageSize = pageSize; } + + private String getFieldName(String globalConfigKey, String defaultFieldName) { + if (this.elasticsearchDao == null || this.elasticsearchDao.getAccessConfig() == null) { + return defaultFieldName; + } + Map<String, Object> globalConfig = this.elasticsearchDao.getAccessConfig().getGlobalConfigSupplier().get(); + return ConfigurationsUtils.getFieldName(globalConfig, globalConfigKey, defaultFieldName); + } } http://git-wip-us.apache.org/repos/asf/metron/blob/32d132b3/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDaoTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDaoTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDaoTest.java index 1bfa9d6..44defb3 100644 --- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDaoTest.java +++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDaoTest.java @@ -214,10 +214,10 @@ public class ElasticsearchMetaAlertDaoTest { List<Map<String, Object>> alertList = new ArrayList<>(); // add an alert with a threat score - alertList.add( Collections.singletonMap(MetaAlertDao.THREAT_FIELD_DEFAULT, 10.0f)); + alertList.add( Collections.singletonMap(ElasticsearchMetaAlertDao.THREAT_TRIAGE_FIELD, 10.0f)); // add a second alert with a threat score - alertList.add( Collections.singletonMap(MetaAlertDao.THREAT_FIELD_DEFAULT, 20.0f)); + alertList.add( Collections.singletonMap(ElasticsearchMetaAlertDao.THREAT_TRIAGE_FIELD, 20.0f)); // add a third alert with NO threat score alertList.add( Collections.singletonMap("alert3", "has no threat score")); @@ -230,7 +230,7 @@ public class ElasticsearchMetaAlertDaoTest { // calculate the threat score for the metaalert ElasticsearchMetaAlertDao metaAlertDao = new ElasticsearchMetaAlertDao(); metaAlertDao.calculateMetaScores(metaalert); - Object threatScore = metaalert.getDocument().get(ElasticsearchMetaAlertDao.THREAT_FIELD_DEFAULT); + Object threatScore = metaalert.getDocument().get(ElasticsearchMetaAlertDao.THREAT_TRIAGE_FIELD); // the metaalert must contain a summary of all child threat scores assertEquals(20D, (Double) metaalert.getDocument().get("max"), delta); @@ -246,4 +246,31 @@ public class ElasticsearchMetaAlertDaoTest { // by default, the overall threat score is the sum of all child threat scores assertEquals(30.0F, threatScore); } + + @Test + public void testCalculateMetaScoresWithDifferentFieldName() { + List<Map<String, Object>> alertList = new ArrayList<>(); + + // add an alert with a threat score + alertList.add( Collections.singletonMap(MetaAlertDao.THREAT_FIELD_DEFAULT, 10.0f)); + + // create the metaalert + Map<String, Object> docMap = new HashMap<>(); + docMap.put(MetaAlertDao.ALERT_FIELD, alertList); + Document metaalert = new Document(docMap, "guid", MetaAlertDao.METAALERT_TYPE, 0L); + + // Configure a different threat triage score field name + AccessConfig accessConfig = new AccessConfig(); + accessConfig.setGlobalConfigSupplier(() -> new HashMap<String, Object>() {{ + put(MetaAlertDao.THREAT_FIELD_PROPERTY, MetaAlertDao.THREAT_FIELD_DEFAULT); + }}); + ElasticsearchDao elasticsearchDao = new ElasticsearchDao(); + elasticsearchDao.setAccessConfig(accessConfig); + + // calculate the threat score for the metaalert + ElasticsearchMetaAlertDao metaAlertDao = new ElasticsearchMetaAlertDao(); + metaAlertDao.init(elasticsearchDao); + metaAlertDao.calculateMetaScores(metaalert); + assertNotNull(metaalert.getDocument().get(MetaAlertDao.THREAT_FIELD_DEFAULT)); + } } http://git-wip-us.apache.org/repos/asf/metron/blob/32d132b3/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java index b001050..adc1a27 100644 --- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java +++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java @@ -18,11 +18,14 @@ package org.apache.metron.elasticsearch.integration; +import static org.apache.metron.common.Constants.SENSOR_TYPE; +import static org.apache.metron.common.Constants.SENSOR_TYPE_FIELD_PROPERTY; import static org.apache.metron.indexing.dao.MetaAlertDao.ALERT_FIELD; import static org.apache.metron.indexing.dao.MetaAlertDao.METAALERTS_INDEX; import static org.apache.metron.indexing.dao.MetaAlertDao.METAALERT_FIELD; import static org.apache.metron.indexing.dao.MetaAlertDao.METAALERT_TYPE; import static org.apache.metron.indexing.dao.MetaAlertDao.STATUS_FIELD; +import static org.apache.metron.indexing.dao.MetaAlertDao.THREAT_FIELD_PROPERTY; import com.fasterxml.jackson.core.JsonProcessingException; import com.google.common.base.Joiner; @@ -85,7 +88,7 @@ public class ElasticsearchMetaAlertIntegrationTest { private static final String NEW_FIELD = "new-field"; private static final String NAME_FIELD = "name"; - private static IndexDao esDao; + private static ElasticsearchDao esDao; private static MetaAlertDao metaDao; private static ElasticSearchComponent es; @@ -193,6 +196,12 @@ public class ElasticsearchMetaAlertIntegrationTest { .withIndexDir(new File(INDEX_DIR)) .build(); es.start(); + } + + @Before + public void setup() throws IOException { + es.createIndexWithMapping(METAALERTS_INDEX, MetaAlertDao.METAALERT_DOC, template.replace("%MAPPING_NAME%", "metaalert")); + es.createIndexWithMapping(INDEX, "index_doc", template.replace("%MAPPING_NAME%", "index")); AccessConfig accessConfig = new AccessConfig(); Map<String, Object> globalConfig = new HashMap<String, Object>() { @@ -212,12 +221,6 @@ public class ElasticsearchMetaAlertIntegrationTest { metaDao = new ElasticsearchMetaAlertDao(esDao); } - @Before - public void setup() throws IOException { - es.createIndexWithMapping(METAALERTS_INDEX, MetaAlertDao.METAALERT_DOC, template.replace("%MAPPING_NAME%", "metaalert")); - es.createIndexWithMapping(INDEX, "index_doc", template.replace("%MAPPING_NAME%", "index")); - } - @AfterClass public static void teardown() { if (es != null) { @@ -320,6 +323,12 @@ public class ElasticsearchMetaAlertIntegrationTest { findCreatedDoc(metaAlertCreateResponse.getGuid(), MetaAlertDao.METAALERT_TYPE); } { + // Verify metaalert has the default field names + Document metaAlert = metaDao.getLatest(metaAlertCreateResponse.getGuid(), MetaAlertDao.METAALERT_TYPE); + Assert.assertTrue(metaAlert.getDocument().containsKey(ElasticsearchMetaAlertDao.SOURCE_TYPE)); + Assert.assertTrue(metaAlert.getDocument().containsKey(ElasticsearchMetaAlertDao.THREAT_TRIAGE_FIELD)); + } + { // Verify alert 0 was not updated with metaalert field Document alert = metaDao.getLatest("message_0", SENSOR_NAME); Assert.assertEquals(4, alert.getDocument().size()); @@ -343,6 +352,45 @@ public class ElasticsearchMetaAlertIntegrationTest { } @Test + public void shouldCreateMetaAlertWithConfiguredFieldNames() throws Exception { + // Configure field names + AccessConfig accessConfig = esDao.getAccessConfig(); + accessConfig.setGlobalConfigSupplier(() -> new HashMap<String, Object>() {{ + put("es.date.format", DATE_FORMAT); + put(SENSOR_TYPE_FIELD_PROPERTY, SENSOR_TYPE); + put(THREAT_FIELD_PROPERTY, MetaAlertDao.THREAT_FIELD_DEFAULT); + }}); + + // Load alerts + List<Map<String, Object>> alerts = buildAlerts(1); + elasticsearchAdd(alerts, INDEX, SENSOR_NAME); + + // Verify load was successful + findCreatedDocs(Collections.singletonList( + new GetRequest("message_0", SENSOR_NAME))); + + { + MetaAlertCreateRequest metaAlertCreateRequest = new MetaAlertCreateRequest() {{ + setAlerts(new ArrayList<GetRequest>() {{ + add(new GetRequest("message_0", SENSOR_NAME)); + }}); + setGroups(Collections.singletonList("group")); + }}; + MetaAlertCreateResponse metaAlertCreateResponse = metaDao.createMetaAlert(metaAlertCreateRequest); + { + // Verify metaAlert was created + findCreatedDoc(metaAlertCreateResponse.getGuid(), MetaAlertDao.METAALERT_TYPE); + } + { + // Verify alert 0 was not updated with metaalert field + Document metaAlert = metaDao.getLatest(metaAlertCreateResponse.getGuid(), MetaAlertDao.METAALERT_TYPE); + Assert.assertTrue(metaAlert.getDocument().containsKey(SENSOR_TYPE)); + Assert.assertTrue(metaAlert.getDocument().containsKey(MetaAlertDao.THREAT_FIELD_DEFAULT)); + } + } + } + + @Test public void shouldAddAlertsToMetaAlert() throws Exception { // Load alerts List<Map<String, Object>> alerts = buildAlerts(4); @@ -1173,7 +1221,7 @@ public class ElasticsearchMetaAlertIntegrationTest { Map<String, Object> alerts = new HashMap<>(); alerts.put(Constants.GUID, guid); alerts.put("source:type", SENSOR_NAME); - alerts.put(MetaAlertDao.THREAT_FIELD_DEFAULT, i); + alerts.put(ElasticsearchMetaAlertDao.THREAT_TRIAGE_FIELD, i); alerts.put("timestamp", System.currentTimeMillis()); inputData.add(alerts); } http://git-wip-us.apache.org/repos/asf/metron/blob/32d132b3/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MetaAlertDao.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MetaAlertDao.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MetaAlertDao.java index 8807849..93c791b 100644 --- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MetaAlertDao.java +++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MetaAlertDao.java @@ -68,7 +68,8 @@ public interface MetaAlertDao extends IndexDao { String METAALERT_TYPE = "metaalert"; String METAALERT_FIELD = "metaalerts"; String METAALERT_DOC = METAALERT_TYPE + "_doc"; - String THREAT_FIELD_DEFAULT = "threat:triage:score"; + String THREAT_FIELD_DEFAULT = "threat.triage.score"; + String THREAT_FIELD_PROPERTY = "threat.triage.score.field"; String THREAT_SORT_DEFAULT = "sum"; String ALERT_FIELD = "metron_alert"; String STATUS_FIELD = "status";
