Repository: metron Updated Branches: refs/heads/master 243ff1f72 -> 32a0a9da8
METRON-1262 Unable to add comment for a alert in a meta-alert (merrimanr) closes apache/metron#806 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/32a0a9da Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/32a0a9da Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/32a0a9da Branch: refs/heads/master Commit: 32a0a9da89f7480fdc6a15a93eebf8df05fb0ecf Parents: 243ff1f Author: merrimanr <[email protected]> Authored: Fri Oct 20 13:46:39 2017 -0500 Committer: merrimanr <[email protected]> Committed: Fri Oct 20 13:46:39 2017 -0500 ---------------------------------------------------------------------- metron-platform/metron-elasticsearch/pom.xml | 10 -- .../dao/ElasticsearchMetaAlertDao.java | 16 ++- .../dao/ElasticsearchMetaAlertDaoTest.java | 63 +++------- .../ElasticsearchMetaAlertIntegrationTest.java | 119 ++++++++++++++++++- .../dao/metaalert/MetaAlertCreateResponse.java | 9 ++ 5 files changed, 148 insertions(+), 69 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/32a0a9da/metron-platform/metron-elasticsearch/pom.xml ---------------------------------------------------------------------- diff --git a/metron-platform/metron-elasticsearch/pom.xml b/metron-platform/metron-elasticsearch/pom.xml index bdd37f9..f9f05ea 100644 --- a/metron-platform/metron-elasticsearch/pom.xml +++ b/metron-platform/metron-elasticsearch/pom.xml @@ -164,16 +164,6 @@ <version>3.10.5.Final</version> </dependency> <dependency> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-databind</artifactId> - <version>2.2.3</version> - </dependency> - <dependency> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-annotations</artifactId> - <version>2.2.3</version> - </dependency> - <dependency> <groupId>org.apache.metron</groupId> <artifactId>metron-integration-test</artifactId> <version>${project.parent.version}</version> http://git-wip-us.apache.org/repos/asf/metron/blob/32a0a9da/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java index 44c983e..3409973 100644 --- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java +++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java @@ -66,7 +66,6 @@ import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryStringQueryBuilder; import org.elasticsearch.index.query.support.QueryInnerHitBuilder; import org.elasticsearch.search.SearchHit; -import org.elasticsearch.search.SearchHits; public class ElasticsearchMetaAlertDao implements MetaAlertDao { @@ -168,6 +167,7 @@ public class ElasticsearchMetaAlertDao implements MetaAlertDao { handleMetaUpdate(createDoc, Optional.of(METAALERTS_INDEX)); MetaAlertCreateResponse createResponse = new MetaAlertCreateResponse(); createResponse.setCreated(true); + createResponse.setGuid(createDoc.getGuid()); return createResponse; } catch (IOException ioe) { throw new InvalidCreateException("Unable to create meta alert", ioe); @@ -404,16 +404,14 @@ public class ElasticsearchMetaAlertDao implements MetaAlertDao { // Run through the nested alerts of the meta alert and either use the new or old versions builder.startArray(ALERT_FIELD); - Map<String, SearchHits> innerHits = hit.getInnerHits(); + Map<String, Object> hitAlerts = hit.sourceAsMap(); - SearchHits alertHits = innerHits.get(ALERT_FIELD); - for (SearchHit alertHit : alertHits.getHits()) { - Map<String, Object> docMap; - // If we're at the update use it, otherwise use the original - if (alertHit.sourceAsMap().get(Constants.GUID).equals(update.getGuid())) { + List<Map<String, Object>> alertHits = (List<Map<String, Object>>) hitAlerts.get(ALERT_FIELD); + for (Map<String, Object> alertHit : alertHits) { + Map<String, Object> docMap = alertHit; + // If we're at the update use it instead of the original + if (alertHit.get(Constants.GUID).equals(update.getGuid())) { docMap = update.getDocument(); - } else { - docMap = alertHit.getSource(); } builder.map(docMap); http://git-wip-us.apache.org/repos/asf/metron/blob/32a0a9da/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDaoTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDaoTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDaoTest.java index ad2e7f3..5d6f4e0 100644 --- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDaoTest.java +++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDaoTest.java @@ -27,6 +27,8 @@ import static org.mockito.Mockito.when; import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.Iterator; import java.util.List; @@ -54,14 +56,11 @@ import org.elasticsearch.action.get.MultiGetItemResponse; import org.elasticsearch.action.get.MultiGetResponse; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.search.SearchHit; -import org.elasticsearch.search.SearchHitField; -import org.elasticsearch.search.SearchHits; import org.json.simple.JSONArray; import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; import org.json.simple.parser.ParseException; import org.junit.Test; -import org.mockito.Mock; public class ElasticsearchMetaAlertDaoTest { @@ -96,22 +95,13 @@ public class ElasticsearchMetaAlertDaoTest { when(metaHit.getSource()).thenReturn(metaSource); // Construct the inner alert - SearchHit innerAlertHit = mock(SearchHit.class); HashMap<String, Object> innerAlertSource = new HashMap<>(); innerAlertSource.put(Constants.GUID, "a1"); - when(innerAlertHit.sourceAsMap()).thenReturn(innerAlertSource); - SearchHitField field = mock(SearchHitField.class); - when(field.getValue()).thenReturn(10d); - when(innerAlertHit.field(MetaAlertDao.THREAT_FIELD_DEFAULT)).thenReturn(field); - SearchHit[] innerHitArray = new SearchHit[1]; - innerHitArray[0] = innerAlertHit; - - // Construct the inner hits that contains the alert - SearchHits searchHits = mock(SearchHits.class); - when(searchHits.getHits()).thenReturn(innerHitArray); - Map<String, SearchHits> innerHits = new HashMap<>(); - innerHits.put(MetaAlertDao.ALERT_FIELD, searchHits); - when(metaHit.getInnerHits()).thenReturn(innerHits); + innerAlertSource.put(MetaAlertDao.THREAT_FIELD_DEFAULT, 10d); + + Map<String, Object> innerHits = new HashMap<>(); + innerHits.put(MetaAlertDao.ALERT_FIELD, Collections.singletonList(innerAlertSource)); + when(metaHit.sourceAsMap()).thenReturn(innerHits); // Construct the updated Document Map<String, Object> updateMap = new HashMap<>(); @@ -169,43 +159,18 @@ public class ElasticsearchMetaAlertDaoTest { when(metaHit.getSource()).thenReturn(metaSource); // Construct the inner alerts - SearchHit innerAlertHitOne = mock(SearchHit.class); HashMap<String, Object> innerAlertSourceOne = new HashMap<>(); String guidOne = "a1"; innerAlertSourceOne.put(Constants.GUID, guidOne); - when(innerAlertHitOne.sourceAsMap()).thenReturn(innerAlertSourceOne); - when(innerAlertHitOne.getId()).thenReturn(guidOne); - SearchHitField triageOne = mock(SearchHitField.class); - when(triageOne.getValue()).thenReturn(threatValueOne); - Map<String, Object> innerAlertHitOneSource = new HashMap<>(); - innerAlertHitOneSource.put(MetaAlertDao.THREAT_FIELD_DEFAULT, threatValueTwo); - innerAlertHitOneSource.put(Constants.GUID, guidOne); - when(innerAlertHitOne.getSource()).thenReturn(innerAlertHitOneSource); - when(innerAlertHitOne.field(MetaAlertDao.THREAT_FIELD_DEFAULT)).thenReturn(triageOne); - - SearchHit innerAlertHitTwo = mock(SearchHit.class); + innerAlertSourceOne.put(MetaAlertDao.THREAT_FIELD_DEFAULT, threatValueTwo); + HashMap<String, Object> innerAlertSourceTwo = new HashMap<>(); innerAlertSourceTwo.put(Constants.GUID, guidTwo); - when(innerAlertHitTwo.sourceAsMap()).thenReturn(innerAlertSourceTwo); - when(innerAlertHitOne.getId()).thenReturn(guidTwo); - SearchHitField triageTwo = mock(SearchHitField.class); - when(triageTwo.getValue()).thenReturn(threatValueTwo); - Map<String, Object> innerAlertHitTwoSource = new HashMap<>(); - innerAlertHitTwoSource.put(MetaAlertDao.THREAT_FIELD_DEFAULT, threatValueTwo); - innerAlertHitTwoSource.put(Constants.GUID, guidTwo); - when(innerAlertHitTwo.getSource()).thenReturn(innerAlertHitTwoSource); - when(innerAlertHitTwo.field(MetaAlertDao.THREAT_FIELD_DEFAULT)).thenReturn(triageTwo); - - SearchHit[] innerHitArray = new SearchHit[2]; - innerHitArray[0] = innerAlertHitOne; - innerHitArray[1] = innerAlertHitTwo; - - // Construct the inner hits that contains the alert - SearchHits searchHits = mock(SearchHits.class); - when(searchHits.getHits()).thenReturn(innerHitArray); - Map<String, SearchHits> innerHits = new HashMap<>(); - innerHits.put(MetaAlertDao.ALERT_FIELD, searchHits); - when(metaHit.getInnerHits()).thenReturn(innerHits); + innerAlertSourceTwo.put(MetaAlertDao.THREAT_FIELD_DEFAULT, threatValueTwo); + + Map<String, Object> innerHits = new HashMap<>(); + innerHits.put(MetaAlertDao.ALERT_FIELD, Arrays.asList(innerAlertSourceOne, innerAlertSourceTwo)); + when(metaHit.sourceAsMap()).thenReturn(innerHits); // Construct the updated Document Map<String, Object> updateMap = new HashMap<>(); http://git-wip-us.apache.org/repos/asf/metron/blob/32a0a9da/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java index 12ac888..b13032f 100644 --- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java +++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java @@ -44,10 +44,13 @@ import org.apache.metron.elasticsearch.integration.components.ElasticSearchCompo import org.apache.metron.indexing.dao.AccessConfig; import org.apache.metron.indexing.dao.IndexDao; import org.apache.metron.indexing.dao.MetaAlertDao; +import org.apache.metron.indexing.dao.metaalert.MetaAlertCreateRequest; +import org.apache.metron.indexing.dao.metaalert.MetaAlertCreateResponse; import org.apache.metron.indexing.dao.search.SearchRequest; import org.apache.metron.indexing.dao.search.SearchResponse; import org.apache.metron.indexing.dao.search.SortField; import org.apache.metron.indexing.dao.update.Document; +import org.apache.metron.indexing.dao.update.PatchRequest; import org.apache.metron.indexing.dao.update.ReplaceRequest; import org.junit.AfterClass; import org.junit.Assert; @@ -66,7 +69,7 @@ public class ElasticsearchMetaAlertIntegrationTest { private static final String NEW_FIELD = "new-field"; private static IndexDao esDao; - private static IndexDao metaDao; + private static MetaAlertDao metaDao; private static ElasticSearchComponent es; @BeforeClass @@ -455,6 +458,108 @@ public class ElasticsearchMetaAlertIntegrationTest { Assert.assertEquals("active_metaalert", searchResponse.getResults().get(0).getSource().get("guid")); } + /** + { + "guid": "update_metaalert_alert_0", + "source:type": "test", + "field": "value 0" + } + */ + @Multiline + public static String updateMetaAlertAlert0; + + /** + { + "guid": "update_metaalert_alert_1", + "source:type": "test", + "field":"value 1" + } + */ + @Multiline + public static String updateMetaAlertAlert1; + + /** + { + "guid": "update_metaalert_alert_0", + "patch": [ + { + "op": "add", + "path": "/field", + "value": "patched value 0" + } + ], + "sensorType": "test" + } + */ + @Multiline + public static String updateMetaAlertPatchRequest; + + /** + { + "guid": "update_metaalert_alert_0", + "replacement": { + "guid": "update_metaalert_alert_0", + "source:type": "test", + "field": "replaced value 0" + }, + "sensorType": "test" + } + */ + @Multiline + public static String updateMetaAlertReplaceRequest; + + @Test + public void shouldUpdateMetaAlertOnAlertPatchOrReplace() throws Exception { + List<Map<String, Object>> inputData = new ArrayList<>(); + Map<String, Object> updateMetaAlertAlert0JSON = JSONUtils.INSTANCE.load(updateMetaAlertAlert0, new TypeReference<Map<String, Object>>() {}); + inputData.add(updateMetaAlertAlert0JSON); + Map<String, Object> updateMetaAlertAlert1JSON = JSONUtils.INSTANCE.load(updateMetaAlertAlert1, new TypeReference<Map<String, Object>>() {}); + inputData.add(updateMetaAlertAlert1JSON); + elasticsearchAdd(inputData, INDEX, SENSOR_NAME); + // Wait for updates to persist + findUpdatedDoc(updateMetaAlertAlert1JSON, "update_metaalert_alert_1", SENSOR_NAME); + + MetaAlertCreateResponse metaAlertCreateResponse = metaDao.createMetaAlert(new MetaAlertCreateRequest() {{ + setGuidToIndices(new HashMap<String, String>() {{ + put("update_metaalert_alert_0", INDEX); + put("update_metaalert_alert_1", INDEX); + }}); + setGroups(Collections.singletonList("group")); + }}); + // Wait for updates to persist + findCreatedDoc(metaAlertCreateResponse.getGuid(), MetaAlertDao.METAALERT_TYPE); + + // Patch alert + metaDao.patch(JSONUtils.INSTANCE.load(updateMetaAlertPatchRequest, PatchRequest.class), Optional.empty()); + + // Wait for updates to persist + updateMetaAlertAlert0JSON.put("field", "patched value 0"); + findUpdatedDoc(updateMetaAlertAlert0JSON, "update_metaalert_alert_0", SENSOR_NAME); + + Map<String, Object> metaalert = metaDao.getLatest(metaAlertCreateResponse.getGuid(), MetaAlertDao.METAALERT_TYPE).getDocument(); + List<Map<String, Object>> alerts = (List<Map<String, Object>>) metaalert.get("alert"); + Assert.assertEquals(2, alerts.size()); + Assert.assertEquals("update_metaalert_alert_1", alerts.get(0).get("guid")); + Assert.assertEquals("value 1", alerts.get(0).get("field")); + Assert.assertEquals("update_metaalert_alert_0", alerts.get(1).get("guid")); + Assert.assertEquals("patched value 0", alerts.get(1).get("field")); + + // Replace alert + metaDao.replace(JSONUtils.INSTANCE.load(updateMetaAlertReplaceRequest, ReplaceRequest.class), Optional.empty()); + + // Wait for updates to persist + updateMetaAlertAlert0JSON.put("field", "replaced value 0"); + findUpdatedDoc(updateMetaAlertAlert0JSON, "update_metaalert_alert_0", SENSOR_NAME); + + metaalert = metaDao.getLatest(metaAlertCreateResponse.getGuid(), MetaAlertDao.METAALERT_TYPE).getDocument(); + alerts = (List<Map<String, Object>>) metaalert.get("alert"); + Assert.assertEquals(2, alerts.size()); + Assert.assertEquals("update_metaalert_alert_1", alerts.get(0).get("guid")); + Assert.assertEquals("value 1", alerts.get(0).get("field")); + Assert.assertEquals("update_metaalert_alert_0", alerts.get(1).get("guid")); + Assert.assertEquals("replaced value 0", alerts.get(1).get("field")); + } + protected boolean findUpdatedDoc(Map<String, Object> message0, String guid, String sensorType) throws InterruptedException, IOException { boolean found = false; @@ -467,6 +572,18 @@ public class ElasticsearchMetaAlertIntegrationTest { return found; } + protected boolean findCreatedDoc(String guid, String sensorType) + throws InterruptedException, IOException { + boolean found = false; + for (int t = 0; t < MAX_RETRIES && !found; ++t, Thread.sleep(SLEEP_MS)) { + Document doc = metaDao.getLatest(guid, sensorType); + if (doc != null) { + found = true; + } + } + return found; + } + protected void elasticsearchAdd(List<Map<String, Object>> inputData, String index, String docType) throws IOException { es.add(index, docType, inputData.stream().map(m -> { http://git-wip-us.apache.org/repos/asf/metron/blob/32a0a9da/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertCreateResponse.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertCreateResponse.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertCreateResponse.java index e84286e..0bdf332 100644 --- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertCreateResponse.java +++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertCreateResponse.java @@ -20,6 +20,7 @@ package org.apache.metron.indexing.dao.metaalert; public class MetaAlertCreateResponse { private boolean created; + private String guid; public boolean isCreated() { return created; @@ -28,4 +29,12 @@ public class MetaAlertCreateResponse { public void setCreated(boolean created) { this.created = created; } + + public String getGuid() { + return guid; + } + + public void setGuid(String guid) { + this.guid = guid; + } }
