Repository: metron Updated Branches: refs/heads/master fef8833c1 -> 73623ec74
METRON-1255 MetaAlert search is not filtering on status (merrimanr) closes apache/metron#802 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/73623ec7 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/73623ec7 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/73623ec7 Branch: refs/heads/master Commit: 73623ec74232765849f40480add9117da67c11d5 Parents: fef8833 Author: merrimanr <[email protected]> Authored: Wed Oct 18 16:21:37 2017 -0500 Committer: merrimanr <[email protected]> Committed: Wed Oct 18 16:21:37 2017 -0500 ---------------------------------------------------------------------- .../dao/ElasticsearchMetaAlertDao.java | 13 +- .../ElasticsearchMetaAlertIntegrationTest.java | 163 ++++++++++++++++++- .../metron/indexing/dao/MetaAlertDao.java | 2 +- 3 files changed, 168 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/73623ec7/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java index a64f5ad..44c983e 100644 --- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java +++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java @@ -21,6 +21,7 @@ package org.apache.metron.elasticsearch.dao; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.index.query.QueryBuilders.boolQuery; import static org.elasticsearch.index.query.QueryBuilders.constantScoreQuery; +import static org.elasticsearch.index.query.QueryBuilders.existsQuery; import static org.elasticsearch.index.query.QueryBuilders.nestedQuery; import static org.elasticsearch.index.query.QueryBuilders.termQuery; @@ -177,15 +178,19 @@ public class ElasticsearchMetaAlertDao implements MetaAlertDao { public SearchResponse search(SearchRequest searchRequest) throws InvalidSearchException { // Wrap the query to also get any meta-alerts. QueryBuilder qb = constantScoreQuery(boolQuery() - .should(new QueryStringQueryBuilder(searchRequest.getQuery())) - .should(boolQuery() - .must(termQuery(MetaAlertDao.STATUS_FIELD, MetaAlertStatus.ACTIVE.getStatusString())) - .must(nestedQuery( + .must(boolQuery() + .should(new QueryStringQueryBuilder(searchRequest.getQuery())) + .should(nestedQuery( ALERT_FIELD, new QueryStringQueryBuilder(searchRequest.getQuery()) ) ) ) + // Ensures that it's a meta alert with active status or that it's an alert (signified by having no status field) + .must(boolQuery() + .should(termQuery(MetaAlertDao.STATUS_FIELD, MetaAlertStatus.ACTIVE.getStatusString())) + .should(boolQuery().mustNot(existsQuery(MetaAlertDao.STATUS_FIELD))) + ) ); return elasticsearchDao.search(searchRequest, qb); } http://git-wip-us.apache.org/repos/asf/metron/blob/73623ec7/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java index c562e0b..12ac888 100644 --- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java +++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java @@ -21,16 +21,20 @@ package org.apache.metron.elasticsearch.integration; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; import java.io.File; import java.io.IOException; import java.text.SimpleDateFormat; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.stream.Collectors; +import org.adrianwalker.multilinestring.Multiline; import org.apache.metron.common.Constants; import org.apache.metron.common.utils.JSONUtils; import org.apache.metron.elasticsearch.dao.ElasticsearchDao; @@ -40,6 +44,9 @@ import org.apache.metron.elasticsearch.integration.components.ElasticSearchCompo import org.apache.metron.indexing.dao.AccessConfig; import org.apache.metron.indexing.dao.IndexDao; import org.apache.metron.indexing.dao.MetaAlertDao; +import org.apache.metron.indexing.dao.search.SearchRequest; +import org.apache.metron.indexing.dao.search.SearchResponse; +import org.apache.metron.indexing.dao.search.SortField; import org.apache.metron.indexing.dao.update.Document; import org.apache.metron.indexing.dao.update.ReplaceRequest; import org.junit.AfterClass; @@ -83,6 +90,7 @@ public class ElasticsearchMetaAlertIntegrationTest { put("es.date.format", DATE_FORMAT); } }; + accessConfig.setMaxSearchResults(1000); accessConfig.setGlobalConfigSupplier(() -> globalConfig); esDao = new ElasticsearchDao(); @@ -198,7 +206,7 @@ public class ElasticsearchMetaAlertIntegrationTest { { //ensure alerts in ES are up-to-date - boolean found = findUpdatedDoc(message0, guid); + boolean found = findUpdatedDoc(message0, guid, SENSOR_NAME); Assert.assertTrue("Unable to find updated document", found); long cnt = 0; for (int t = 0; t < MAX_RETRIES && cnt == 0; ++t, Thread.sleep(SLEEP_MS)) { @@ -257,7 +265,7 @@ public class ElasticsearchMetaAlertIntegrationTest { } }, Optional.empty()); - boolean found = findUpdatedDoc(message0, guid); + boolean found = findUpdatedDoc(message0, guid, SENSOR_NAME); Assert.assertTrue("Unable to find updated document", found); { //ensure ES is up-to-date @@ -302,12 +310,157 @@ public class ElasticsearchMetaAlertIntegrationTest { } } - protected boolean findUpdatedDoc(Map<String, Object> message0, String guid) + /** + { + "guid": "active_metaalert", + "source:type": "metaalert", + "alert": [], + "status": "active" + } + */ + @Multiline + public static String activeMetaAlert; + + /** + { + "guid": "inactive_metaalert", + "source:type": "metaalert", + "alert": [], + "status": "inactive" + } + */ + @Multiline + public static String inactiveMetaAlert; + + @Test + public void shouldSearchByStatus() throws Exception { + List<Map<String, Object>> metaInputData = new ArrayList<>(); + Map<String, Object> activeMetaAlertJSON = JSONUtils.INSTANCE.load(activeMetaAlert, new TypeReference<Map<String, Object>>() {}); + metaInputData.add(activeMetaAlertJSON); + Map<String, Object> inactiveMetaAlertJSON = JSONUtils.INSTANCE.load(inactiveMetaAlert, new TypeReference<Map<String, Object>>() {}); + metaInputData.add(inactiveMetaAlertJSON); + + // We pass MetaAlertDao.METAALERT_TYPE, because the "_doc" gets appended automatically. + elasticsearchAdd(metaInputData, MetaAlertDao.METAALERTS_INDEX, MetaAlertDao.METAALERT_TYPE); + // Wait for updates to persist + findUpdatedDoc(inactiveMetaAlertJSON, "inactive_metaalert", MetaAlertDao.METAALERT_TYPE); + + SearchResponse searchResponse = metaDao.search(new SearchRequest() { + { + setQuery("*"); + setIndices(Collections.singletonList(MetaAlertDao.METAALERT_TYPE)); + setFrom(0); + setSize(5); + setSort(Collections.singletonList(new SortField(){{ setField(Constants.GUID); }})); + } + }); + Assert.assertEquals(1, searchResponse.getTotal()); + Assert.assertEquals(MetaAlertStatus.ACTIVE.getStatusString(), searchResponse.getResults().get(0).getSource().get(MetaAlertDao.STATUS_FIELD)); + } + + /** + { + "guid": "search_by_nested_alert_active_0", + "source:type": "test", + "ip_src_addr": "192.168.1.1", + "ip_src_port": 8010 + } + */ + @Multiline + public static String searchByNestedAlertActive0; + + /** + { + "guid": "search_by_nested_alert_inactive_1", + "source:type": "test", + "ip_src_addr": "192.168.1.2", + "ip_src_port": 8009 + } + */ + @Multiline + public static String searchByNestedAlertActive1; + + /** + { + "guid": "search_by_nested_alert_inactive_0", + "source:type": "test", + "ip_src_addr": "192.168.1.3", + "ip_src_port": 8008 + } + */ + @Multiline + public static String searchByNestedAlertInactive0; + + /** + { + "guid": "search_by_nested_alert_inactive_1", + "source:type": "test", + "ip_src_addr": "192.168.1.4", + "ip_src_port": 8007 + } + */ + @Multiline + public static String searchByNestedAlertInactive1; + + @Test + public void shouldSearchByNestedAlert() throws Exception { + // Create alerts + List<Map<String, Object>> alerts = new ArrayList<>(); + Map<String, Object> searchByNestedAlertActive0JSON = JSONUtils.INSTANCE.load(searchByNestedAlertActive0, new TypeReference<Map<String, Object>>() {}); + alerts.add(searchByNestedAlertActive0JSON); + Map<String, Object> searchByNestedAlertActive1JSON = JSONUtils.INSTANCE.load(searchByNestedAlertActive1, new TypeReference<Map<String, Object>>() {}); + alerts.add(searchByNestedAlertActive1JSON); + Map<String, Object> searchByNestedAlertInactive0JSON = JSONUtils.INSTANCE.load(searchByNestedAlertInactive0, new TypeReference<Map<String, Object>>() {}); + alerts.add(searchByNestedAlertInactive0JSON); + Map<String, Object> searchByNestedAlertInactive1JSON = JSONUtils.INSTANCE.load(searchByNestedAlertInactive1, new TypeReference<Map<String, Object>>() {}); + alerts.add(searchByNestedAlertInactive1JSON); + elasticsearchAdd(alerts, INDEX, SENSOR_NAME); + // Wait for updates to persist + findUpdatedDoc(searchByNestedAlertInactive1JSON, "search_by_nested_alert_inactive_1", SENSOR_NAME); + + // Create metaalerts + Map<String, Object> activeMetaAlertJSON = JSONUtils.INSTANCE.load(activeMetaAlert, new TypeReference<Map<String, Object>>() {}); + activeMetaAlertJSON.put("alert", Arrays.asList(searchByNestedAlertActive0JSON, searchByNestedAlertActive1JSON)); + Map<String, Object> inactiveMetaAlertJSON = JSONUtils.INSTANCE.load(inactiveMetaAlert, new TypeReference<Map<String, Object>>() {}); + inactiveMetaAlertJSON.put("alert", Arrays.asList(searchByNestedAlertInactive0JSON, searchByNestedAlertInactive1JSON)); + + // We pass MetaAlertDao.METAALERT_TYPE, because the "_doc" gets appended automatically. + elasticsearchAdd(Arrays.asList(activeMetaAlertJSON, inactiveMetaAlertJSON), MetaAlertDao.METAALERTS_INDEX, MetaAlertDao.METAALERT_TYPE); + // Wait for updates to persist + findUpdatedDoc(activeMetaAlertJSON, "active_metaalert", MetaAlertDao.METAALERT_TYPE); + + SearchResponse searchResponse = metaDao.search(new SearchRequest() { + { + setQuery("(ip_src_addr:192.168.1.1 AND ip_src_port:8009) OR (alert.ip_src_addr:192.168.1.1 AND alert.ip_src_port:8009)"); + setIndices(Collections.singletonList(MetaAlertDao.METAALERT_TYPE)); + setFrom(0); + setSize(5); + setSort(Collections.singletonList(new SortField(){{ setField(Constants.GUID); }})); + } + }); + // Should not have results because nested alerts shouldn't be flattened + Assert.assertEquals(0, searchResponse.getTotal()); + + searchResponse = metaDao.search(new SearchRequest() { + { + setQuery("(ip_src_addr:192.168.1.1 AND ip_src_port:8010) OR (alert.ip_src_addr:192.168.1.1 AND alert.ip_src_port:8010)"); + setIndices(Collections.singletonList(MetaAlertDao.METAALERT_TYPE)); + setFrom(0); + setSize(5); + setSort(Collections.singletonList(new SortField(){{ setField(Constants.GUID); }})); + } + }); + // Nested query should match a nested alert + Assert.assertEquals(1, searchResponse.getTotal()); + Assert.assertEquals("active_metaalert", searchResponse.getResults().get(0).getSource().get("guid")); + } + + protected boolean findUpdatedDoc(Map<String, Object> message0, String guid, String sensorType) throws InterruptedException, IOException { boolean found = false; for (int t = 0; t < MAX_RETRIES && !found; ++t, Thread.sleep(SLEEP_MS)) { - Document doc = metaDao.getLatest(guid, SENSOR_NAME); - if (message0.equals(doc.getDocument())) { + Document doc = metaDao.getLatest(guid, sensorType); + if (doc != null && message0.equals(doc.getDocument())) { found = true; } } http://git-wip-us.apache.org/repos/asf/metron/blob/73623ec7/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MetaAlertDao.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MetaAlertDao.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MetaAlertDao.java index 05746c4..e9f047b 100644 --- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MetaAlertDao.java +++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MetaAlertDao.java @@ -27,7 +27,7 @@ import org.apache.metron.indexing.dao.search.SearchResponse; public interface MetaAlertDao extends IndexDao { - String METAALERTS_INDEX = "metaalert"; + String METAALERTS_INDEX = "metaalert_index"; String METAALERT_TYPE = "metaalert"; String METAALERT_DOC = METAALERT_TYPE + "_doc"; String THREAT_FIELD_DEFAULT = "threat:triage:score";
