http://git-wip-us.apache.org/repos/asf/metron/blob/49f851e0/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java index 56406f4..7fca764 100644 --- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java +++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java @@ -39,8 +39,6 @@ import org.apache.metron.indexing.dao.update.Document; import org.apache.metron.integration.InMemoryComponent; import org.junit.AfterClass; import org.junit.Assert; -import org.junit.Before; -import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; @@ -472,25 +470,15 @@ public abstract class SearchIntegrationTest { @Multiline public static String differentTypeFilterQuery; - protected static IndexDao dao; protected static InMemoryComponent indexComponent; - @Before - public synchronized void setup() throws Exception { - if(dao == null && indexComponent == null) { - indexComponent = startIndex(); - loadTestData(); - dao = createDao(); - } - } - @Rule public ExpectedException thrown = ExpectedException.none(); @Test public void all_query_returns_all_results() throws Exception { SearchRequest request = JSONUtils.INSTANCE.load(allQuery, SearchRequest.class); - SearchResponse response = dao.search(request); + SearchResponse response = getIndexDao().search(request); Assert.assertEquals(10, response.getTotal()); List<SearchResult> results = response.getResults(); Assert.assertEquals(10, results.size()); @@ -507,7 +495,7 @@ public abstract class SearchIntegrationTest { @Test public void find_one_guid() throws Exception { GetRequest request = JSONUtils.INSTANCE.load(findOneGuidQuery, GetRequest.class); - Optional<Map<String, Object>> response = dao.getLatestResult(request); + Optional<Map<String, Object>> response = getIndexDao().getLatestResult(request); Assert.assertTrue(response.isPresent()); Map<String, Object> doc = response.get(); Assert.assertEquals("bro", doc.get(getSourceTypeField())); @@ -519,7 +507,7 @@ public abstract class SearchIntegrationTest { List<GetRequest> request = JSONUtils.INSTANCE.load(getAllLatestQuery, new JSONUtils.ReferenceSupplier<List<GetRequest>>(){}); Map<String, Document> docs = new HashMap<>(); - for(Document doc : dao.getAllLatest(request)) { + for(Document doc : getIndexDao().getAllLatest(request)) { docs.put(doc.getGuid(), doc); } Assert.assertEquals(2, docs.size()); @@ -532,7 +520,7 @@ public abstract class SearchIntegrationTest { @Test public void filter_query_filters_results() throws Exception { SearchRequest request = JSONUtils.INSTANCE.load(filterQuery, SearchRequest.class); - SearchResponse response = dao.search(request); + SearchResponse response = getIndexDao().search(request); Assert.assertEquals(3, response.getTotal()); List<SearchResult> results = response.getResults(); Assert.assertEquals("snort", results.get(0).getSource().get(getSourceTypeField())); @@ -546,7 +534,7 @@ public abstract class SearchIntegrationTest { @Test public void sort_query_sorts_results_ascending() throws Exception { SearchRequest request = JSONUtils.INSTANCE.load(sortQuery, SearchRequest.class); - SearchResponse response = dao.search(request); + SearchResponse response = getIndexDao().search(request); Assert.assertEquals(10, response.getTotal()); List<SearchResult> results = response.getResults(); for (int i = 8001; i < 8011; ++i) { @@ -557,7 +545,7 @@ public abstract class SearchIntegrationTest { @Test public void sort_ascending_with_missing_fields() throws Exception { SearchRequest request = JSONUtils.INSTANCE.load(sortAscendingWithMissingFields, SearchRequest.class); - SearchResponse response = dao.search(request); + SearchResponse response = getIndexDao().search(request); Assert.assertEquals(10, response.getTotal()); List<SearchResult> results = response.getResults(); Assert.assertEquals(10, results.size()); @@ -575,7 +563,7 @@ public abstract class SearchIntegrationTest { @Test public void sort_descending_with_missing_fields() throws Exception { SearchRequest request = JSONUtils.INSTANCE.load(sortDescendingWithMissingFields, SearchRequest.class); - SearchResponse response = dao.search(request); + SearchResponse response = getIndexDao().search(request); Assert.assertEquals(10, response.getTotal()); List<SearchResult> results = response.getResults(); Assert.assertEquals(10, results.size()); @@ -593,7 +581,7 @@ public abstract class SearchIntegrationTest { @Test public void results_are_paginated() throws Exception { SearchRequest request = JSONUtils.INSTANCE.load(paginationQuery, SearchRequest.class); - SearchResponse response = dao.search(request); + SearchResponse response = getIndexDao().search(request); Assert.assertEquals(10, response.getTotal()); List<SearchResult> results = response.getResults(); Assert.assertEquals(3, results.size()); @@ -608,7 +596,7 @@ public abstract class SearchIntegrationTest { @Test public void returns_results_only_for_specified_indices() throws Exception { SearchRequest request = JSONUtils.INSTANCE.load(indexQuery, SearchRequest.class); - SearchResponse response = dao.search(request); + SearchResponse response = getIndexDao().search(request); Assert.assertEquals(5, response.getTotal()); List<SearchResult> results = response.getResults(); for (int i = 5, j = 0; i > 0; i--, j++) { @@ -621,7 +609,7 @@ public abstract class SearchIntegrationTest { public void facet_query_yields_field_types() throws Exception { String facetQuery = facetQueryRaw.replace("source:type", getSourceTypeField()); SearchRequest request = JSONUtils.INSTANCE.load(facetQuery, SearchRequest.class); - SearchResponse response = dao.search(request); + SearchResponse response = getIndexDao().search(request); Assert.assertEquals(10, response.getTotal()); Map<String, Map<String, Long>> facetCounts = response.getFacetCounts(); Assert.assertEquals(8, facetCounts.size()); @@ -696,14 +684,14 @@ public abstract class SearchIntegrationTest { @Test public void disabled_facet_query_returns_null_count() throws Exception { SearchRequest request = JSONUtils.INSTANCE.load(disabledFacetQuery, SearchRequest.class); - SearchResponse response = dao.search(request); + SearchResponse response = getIndexDao().search(request); Assert.assertNull(response.getFacetCounts()); } @Test public void missing_type_facet_query() throws Exception { SearchRequest request = JSONUtils.INSTANCE.load(missingTypeFacetQuery, SearchRequest.class); - SearchResponse response = dao.search(request); + SearchResponse response = getIndexDao().search(request); Assert.assertEquals(10, response.getTotal()); Map<String, Map<String, Long>> facetCounts = response.getFacetCounts(); @@ -723,7 +711,7 @@ public abstract class SearchIntegrationTest { public void different_type_facet_query() throws Exception { thrown.expect(Exception.class); SearchRequest request = JSONUtils.INSTANCE.load(differentTypeFacetQuery, SearchRequest.class); - SearchResponse response = dao.search(request); + SearchResponse response = getIndexDao().search(request); Assert.assertEquals(3, response.getTotal()); } @@ -732,14 +720,14 @@ public abstract class SearchIntegrationTest { thrown.expect(InvalidSearchException.class); thrown.expectMessage("Search result size must be less than 100"); SearchRequest request = JSONUtils.INSTANCE.load(exceededMaxResultsQuery, SearchRequest.class); - dao.search(request); + getIndexDao().search(request); } @Test public void column_metadata_for_missing_index() throws Exception { // getColumnMetadata with an index that doesn't exist { - Map<String, FieldType> fieldTypes = dao.getColumnMetadata(Collections.singletonList("someindex")); + Map<String, FieldType> fieldTypes = getIndexDao().getColumnMetadata(Collections.singletonList("someindex")); Assert.assertEquals(0, fieldTypes.size()); } } @@ -747,14 +735,14 @@ public abstract class SearchIntegrationTest { @Test public void no_results_returned_when_query_does_not_match() throws Exception { SearchRequest request = JSONUtils.INSTANCE.load(noResultsFieldsQuery, SearchRequest.class); - SearchResponse response = dao.search(request); + SearchResponse response = getIndexDao().search(request); Assert.assertEquals(0, response.getTotal()); } @Test public void group_by_ip_query() throws Exception { GroupRequest request = JSONUtils.INSTANCE.load(groupByIpQuery, GroupRequest.class); - GroupResponse response = dao.group(request); + GroupResponse response = getIndexDao().group(request); // expect only 1 group for 'ip_src_addr' Assert.assertEquals("ip_src_addr", response.getGroupedBy()); @@ -778,7 +766,7 @@ public abstract class SearchIntegrationTest { public void group_by_returns_results_in_groups() throws Exception { // Group by test case, default order is count descending GroupRequest request = JSONUtils.INSTANCE.load(groupByQuery, GroupRequest.class); - GroupResponse response = dao.group(request); + GroupResponse response = getIndexDao().group(request); Assert.assertEquals("is_alert", response.getGroupedBy()); List<GroupResult> isAlertGroups = response.getGroupResults(); Assert.assertEquals(2, isAlertGroups.size()); @@ -830,7 +818,7 @@ public abstract class SearchIntegrationTest { public void group_by_returns_results_in_sorted_groups() throws Exception { // Group by with sorting test case where is_alert is sorted by count ascending and ip_src_addr is sorted by term descending GroupRequest request = JSONUtils.INSTANCE.load(sortedGroupByQuery, GroupRequest.class); - GroupResponse response = dao.group(request); + GroupResponse response = getIndexDao().group(request); Assert.assertEquals("is_alert", response.getGroupedBy()); List<GroupResult> isAlertGroups = response.getGroupResults(); Assert.assertEquals(2, isAlertGroups.size()); @@ -909,7 +897,7 @@ public abstract class SearchIntegrationTest { @Test public void queries_fields() throws Exception { SearchRequest request = JSONUtils.INSTANCE.load(fieldsQuery, SearchRequest.class); - SearchResponse response = dao.search(request); + SearchResponse response = getIndexDao().search(request); Assert.assertEquals(10, response.getTotal()); List<SearchResult> results = response.getResults(); for (int i = 0; i < 5; ++i) { @@ -927,7 +915,7 @@ public abstract class SearchIntegrationTest { @Test public void sort_by_guid() throws Exception { SearchRequest request = JSONUtils.INSTANCE.load(sortByGuidQuery, SearchRequest.class); - SearchResponse response = dao.search(request); + SearchResponse response = getIndexDao().search(request); Assert.assertEquals(5, response.getTotal()); List<SearchResult> results = response.getResults(); for (int i = 0; i < 5; ++i) { @@ -938,7 +926,7 @@ public abstract class SearchIntegrationTest { } @AfterClass - public static void stop() throws Exception { + public static void stop() { indexComponent.stop(); } @@ -949,9 +937,7 @@ public abstract class SearchIntegrationTest { @Test public abstract void different_type_filter_query() throws Exception; + protected abstract IndexDao getIndexDao(); - protected abstract IndexDao createDao() throws Exception; - protected abstract InMemoryComponent startIndex() throws Exception; - protected abstract void loadTestData() throws Exception; protected abstract String getSourceTypeField(); }
http://git-wip-us.apache.org/repos/asf/metron/blob/49f851e0/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/UpdateIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/UpdateIntegrationTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/UpdateIntegrationTest.java index 471acf6..eebf0bb 100644 --- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/UpdateIntegrationTest.java +++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/UpdateIntegrationTest.java @@ -20,86 +20,43 @@ import java.util.List; import java.util.Map; import java.util.NavigableMap; import java.util.Optional; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.Result; import org.apache.metron.common.Constants; import org.apache.metron.common.utils.JSONUtils; -import org.apache.metron.hbase.mock.MockHBaseTableProvider; import org.apache.metron.hbase.mock.MockHTable; import org.apache.metron.indexing.dao.update.Document; import org.apache.metron.indexing.dao.update.ReplaceRequest; -import org.apache.metron.integration.InMemoryComponent; -import org.junit.After; -import org.junit.AfterClass; import org.junit.Assert; -import org.junit.Before; import org.junit.Test; -import static org.junit.Assert.assertEquals; - public abstract class UpdateIntegrationTest { private static final int MAX_RETRIES = 10; private static final int SLEEP_MS = 500; - protected static final String SENSOR_NAME= "test"; - private static final String TABLE_NAME = "modifications"; + protected static final String SENSOR_NAME = "test"; private static final String CF = "p"; - private static String index; - private static MockHTable table; - private static IndexDao hbaseDao; - private static AccessConfig accessConfig; protected static MultiIndexDao dao; - protected static InMemoryComponent indexComponent; - - @Before - public void setup() throws Exception { - if(dao == null && indexComponent == null) { - index = getIndexName(); - indexComponent = startIndex(); - loadTestData(); - Configuration config = HBaseConfiguration.create(); - MockHBaseTableProvider tableProvider = new MockHBaseTableProvider(); - tableProvider.addToCache(TABLE_NAME, CF); - table = (MockHTable)tableProvider.getTable(config, TABLE_NAME); - - hbaseDao = new HBaseDao(); - accessConfig = new AccessConfig(); - accessConfig.setTableProvider(tableProvider); - Map<String, Object> globalConfig = createGlobalConfig(); - globalConfig.put(HBaseDao.HBASE_TABLE, TABLE_NAME); - globalConfig.put(HBaseDao.HBASE_CF, CF); - accessConfig.setGlobalConfigSupplier(() -> globalConfig); - } - } - - protected AccessConfig getAccessConfig() { - return accessConfig; - } @Test public void test() throws Exception { - dao = new MultiIndexDao(hbaseDao, createDao()); - dao.init(getAccessConfig()); - List<Map<String, Object>> inputData = new ArrayList<>(); for(int i = 0; i < 10;++i) { final String name = "message" + i; inputData.add( new HashMap<String, Object>() {{ - put("source:type", SENSOR_NAME); + put("source.type", SENSOR_NAME); put("name" , name); put("timestamp", System.currentTimeMillis()); put(Constants.GUID, name); }} ); } - addTestData(index, SENSOR_NAME, inputData); + addTestData(getIndexName(), SENSOR_NAME, inputData); List<Map<String,Object>> docs = null; for(int t = 0;t < MAX_RETRIES;++t, Thread.sleep(SLEEP_MS)) { - docs = getIndexedTestData(index, SENSOR_NAME); + docs = getIndexedTestData(getIndexName(), SENSOR_NAME); if(docs.size() >= 10) { break; } @@ -115,16 +72,16 @@ public abstract class UpdateIntegrationTest { setReplacement(message0); setGuid(guid); setSensorType(SENSOR_NAME); - setIndex(index); + setIndex(getIndexName()); }}, Optional.empty()); - Assert.assertEquals(1, table.size()); + Assert.assertEquals(1, getMockHTable().size()); Document doc = dao.getLatest(guid, SENSOR_NAME); Assert.assertEquals(message0, doc.getDocument()); { //ensure hbase is up to date Get g = new Get(HBaseDao.Key.toBytes(new HBaseDao.Key(guid, SENSOR_NAME))); - Result r = table.get(g); + Result r = getMockHTable().get(g); NavigableMap<byte[], byte[]> columns = r.getFamilyMap(CF.getBytes()); Assert.assertEquals(1, columns.size()); Assert.assertEquals(message0 @@ -136,7 +93,7 @@ public abstract class UpdateIntegrationTest { //ensure ES is up-to-date long cnt = 0; for (int t = 0; t < MAX_RETRIES && cnt == 0; ++t, Thread.sleep(SLEEP_MS)) { - docs = getIndexedTestData(index, SENSOR_NAME); + docs = getIndexedTestData(getIndexName(), SENSOR_NAME); cnt = docs .stream() .filter(d -> message0.get("new-field").equals(d.get("new-field"))) @@ -155,15 +112,15 @@ public abstract class UpdateIntegrationTest { setReplacement(message0); setGuid(guid); setSensorType(SENSOR_NAME); - setIndex(index); + setIndex(getIndexName()); }}, Optional.empty()); - Assert.assertEquals(1, table.size()); + Assert.assertEquals(1, getMockHTable().size()); Document doc = dao.getLatest(guid, SENSOR_NAME); Assert.assertEquals(message0, doc.getDocument()); { //ensure hbase is up to date Get g = new Get(HBaseDao.Key.toBytes(new HBaseDao.Key(guid, SENSOR_NAME))); - Result r = table.get(g); + Result r = getMockHTable().get(g); NavigableMap<byte[], byte[]> columns = r.getFamilyMap(CF.getBytes()); Assert.assertEquals(2, columns.size()); Assert.assertEquals(message0, JSONUtils.INSTANCE.load(new String(columns.lastEntry().getValue()) @@ -177,36 +134,20 @@ public abstract class UpdateIntegrationTest { //ensure ES is up-to-date long cnt = 0; for (int t = 0; t < MAX_RETRIES && cnt == 0; ++t,Thread.sleep(SLEEP_MS)) { - docs = getIndexedTestData(index, SENSOR_NAME); + docs = getIndexedTestData(getIndexName(), SENSOR_NAME); cnt = docs .stream() .filter(d -> message0.get("new-field").equals(d.get("new-field"))) .count(); } - Assert.assertNotEquals("Elasticsearch is not updated!", cnt, 0); + Assert.assertNotEquals("Index is not updated!", cnt, 0); } } } - @After - public void reset() throws Exception { - indexComponent.reset(); - } - - @AfterClass - public static void teardown() { - if(indexComponent != null) { - indexComponent.stop(); - } - } - protected abstract String getIndexName(); - protected abstract Map<String, Object> createGlobalConfig() throws Exception; - protected abstract IndexDao createDao() throws Exception; - protected abstract InMemoryComponent startIndex() throws Exception; - protected abstract void loadTestData() throws Exception; + protected abstract MockHTable getMockHTable(); protected abstract void addTestData(String indexName, String sensorType, List<Map<String,Object>> docs) throws Exception; protected abstract List<Map<String,Object>> getIndexedTestData(String indexName, String sensorType) throws Exception; - } http://git-wip-us.apache.org/repos/asf/metron/blob/49f851e0/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/metaalert/MetaAlertIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/metaalert/MetaAlertIntegrationTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/metaalert/MetaAlertIntegrationTest.java new file mode 100644 index 0000000..b4f7d38 --- /dev/null +++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/metaalert/MetaAlertIntegrationTest.java @@ -0,0 +1,1012 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.indexing.dao.metaalert; + +import static org.apache.metron.indexing.dao.metaalert.MetaAlertConstants.ALERT_FIELD; +import static org.apache.metron.indexing.dao.metaalert.MetaAlertConstants.METAALERT_FIELD; +import static org.apache.metron.indexing.dao.metaalert.MetaAlertConstants.METAALERT_TYPE; +import static org.apache.metron.indexing.dao.metaalert.MetaAlertConstants.STATUS_FIELD; +import static org.apache.metron.indexing.dao.metaalert.MetaAlertConstants.THREAT_FIELD_DEFAULT; + +import com.google.common.base.Joiner; +import com.google.common.collect.Iterables; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import org.adrianwalker.multilinestring.Multiline; +import org.apache.metron.common.Constants; +import org.apache.metron.common.utils.JSONUtils; +import org.apache.metron.indexing.dao.search.GetRequest; +import org.apache.metron.indexing.dao.search.Group; +import org.apache.metron.indexing.dao.search.GroupRequest; +import org.apache.metron.indexing.dao.search.GroupResponse; +import org.apache.metron.indexing.dao.search.GroupResult; +import org.apache.metron.indexing.dao.search.InvalidSearchException; +import org.apache.metron.indexing.dao.search.SearchRequest; +import org.apache.metron.indexing.dao.search.SearchResponse; +import org.apache.metron.indexing.dao.search.SearchResult; +import org.apache.metron.indexing.dao.search.SortField; +import org.apache.metron.indexing.dao.update.Document; +import org.apache.metron.indexing.dao.update.OriginalNotFoundException; +import org.apache.metron.indexing.dao.update.PatchRequest; +import org.junit.Assert; +import org.junit.Test; + +public abstract class MetaAlertIntegrationTest { + + private static final String META_INDEX_FLAG = "%META_INDEX%"; + // To change back after testing + protected static int MAX_RETRIES = 10; + protected static final int SLEEP_MS = 500; + protected static final String SENSOR_NAME = "test"; + + protected static final String NEW_FIELD = "new-field"; + protected static final String NAME_FIELD = "name"; + protected static final String DATE_FORMAT = "yyyy.MM.dd.HH"; + + // Separate the raw indices from the query indices. ES for example, modifies the indices to + // have a separator + protected ArrayList<String> allIndices = new ArrayList<String>() { + { + add(getTestIndexName()); + add(getMetaAlertIndex()); + } + }; + + protected ArrayList<String> queryIndices = allIndices; + + protected static MetaAlertDao metaDao; + + /** + { + "guid": "meta_alert", + "index": "%META_INDEX%", + "patch": [ + { + "op": "add", + "path": "/name", + "value": "New Meta Alert" + } + ], + "sensorType": "metaalert" + } + */ + @Multiline + public static String namePatchRequest; + + /** + { + "guid": "meta_alert", + "index": "%META_INDEX%", + "patch": [ + { + "op": "add", + "path": "/name", + "value": "New Meta Alert" + }, + { + "op": "add", + "path": "/alert", + "value": [] + } + ], + "sensorType": "metaalert" + } + */ + @Multiline + public static String alertPatchRequest; + + /** + { + "guid": "meta_alert", + "index": "%META_INDEX%", + "patch": [ + { + "op": "add", + "path": "/status", + "value": "inactive" + }, + { + "op": "add", + "path": "/name", + "value": "New Meta Alert" + } + ], + "sensorType": "metaalert" + } + */ + @Multiline + public static String statusPatchRequest; + + + @Test + public void shouldGetAllMetaAlertsForAlert() throws Exception { + // Load alerts + List<Map<String, Object>> alerts = buildAlerts(3); + addRecords(alerts, getTestIndexFullName(), SENSOR_NAME); + + // Load metaAlerts + List<Map<String, Object>> metaAlerts = buildMetaAlerts(12, MetaAlertStatus.ACTIVE, + Optional.of(Collections.singletonList(alerts.get(0)))); + metaAlerts.add(buildMetaAlert("meta_active_12", MetaAlertStatus.ACTIVE, + Optional.of(Arrays.asList(alerts.get(0), alerts.get(2))))); + metaAlerts.add(buildMetaAlert("meta_inactive", MetaAlertStatus.INACTIVE, + Optional.of(Arrays.asList(alerts.get(0), alerts.get(2))))); + // We pass MetaAlertDao.METAALERT_TYPE, because the "_doc" gets appended automatically. + addRecords(metaAlerts, getMetaAlertIndex(), METAALERT_TYPE); + + // Verify load was successful + List<GetRequest> createdDocs = metaAlerts.stream().map(metaAlert -> + new GetRequest((String) metaAlert.get(Constants.GUID), METAALERT_TYPE)) + .collect(Collectors.toList()); + createdDocs.addAll(alerts.stream().map(alert -> + new GetRequest((String) alert.get(Constants.GUID), SENSOR_NAME)) + .collect(Collectors.toList())); + findCreatedDocs(createdDocs); + + { + // Verify searches successfully return more than 10 results + SearchResponse searchResponse0 = metaDao.getAllMetaAlertsForAlert("message_0"); + List<SearchResult> searchResults0 = searchResponse0.getResults(); + Assert.assertEquals(13, searchResults0.size()); + Set<Map<String, Object>> resultSet = new HashSet<>(); + Iterables.addAll(resultSet, Iterables.transform(searchResults0, r -> r.getSource())); + StringBuffer reason = new StringBuffer("Unable to find " + metaAlerts.get(0) + "\n"); + reason.append(Joiner.on("\n").join(resultSet)); + Assert.assertTrue(reason.toString(), resultSet.contains(metaAlerts.get(0))); + + // Verify no meta alerts are returned because message_1 was not added to any + SearchResponse searchResponse1 = metaDao.getAllMetaAlertsForAlert("message_1"); + List<SearchResult> searchResults1 = searchResponse1.getResults(); + Assert.assertEquals(0, searchResults1.size()); + + // Verify only the meta alert message_2 was added to is returned + SearchResponse searchResponse2 = metaDao.getAllMetaAlertsForAlert("message_2"); + List<SearchResult> searchResults2 = searchResponse2.getResults(); + Assert.assertEquals(1, searchResults2.size()); + Assert.assertEquals(metaAlerts.get(12), searchResults2.get(0).getSource()); + } + } + + @Test + public void getAllMetaAlertsForAlertShouldThrowExceptionForEmptyGuid() throws Exception { + try { + metaDao.getAllMetaAlertsForAlert(""); + Assert.fail("An exception should be thrown for empty guid"); + } catch (InvalidSearchException ise) { + Assert.assertEquals("Guid cannot be empty", ise.getMessage()); + } + } + + @Test + public void shouldCreateMetaAlert() throws Exception { + // Load alerts + List<Map<String, Object>> alerts = buildAlerts(3); + addRecords(alerts, getTestIndexFullName(), SENSOR_NAME); + + // Verify load was successful + findCreatedDocs(Arrays.asList( + new GetRequest("message_0", SENSOR_NAME), + new GetRequest("message_1", SENSOR_NAME), + new GetRequest("message_2", SENSOR_NAME))); + + { + MetaAlertCreateRequest metaAlertCreateRequest = new MetaAlertCreateRequest() {{ + setAlerts(new ArrayList<GetRequest>() {{ + add(new GetRequest("message_1", SENSOR_NAME)); + add(new GetRequest("message_2", SENSOR_NAME, getTestIndexFullName())); + }}); + setGroups(Collections.singletonList("group")); + }}; + MetaAlertCreateResponse metaAlertCreateResponse = metaDao + .createMetaAlert(metaAlertCreateRequest); + { + // Verify metaAlert was created + findCreatedDoc(metaAlertCreateResponse.getGuid(), METAALERT_TYPE); + } + { + // Verify alert 0 was not updated with metaalert field + Document alert = metaDao.getLatest("message_0", SENSOR_NAME); + Assert.assertEquals(4, alert.getDocument().size()); + Assert.assertNull(alert.getDocument().get(METAALERT_FIELD)); + } + { + // Verify alert 1 was properly updated with metaalert field + Map<String, Object> expectedAlert = new HashMap<>(alerts.get(1)); + expectedAlert + .put(METAALERT_FIELD, Collections.singletonList(metaAlertCreateResponse.getGuid())); + findUpdatedDoc(expectedAlert, "message_1", SENSOR_NAME); + } + { + // Verify alert 2 was properly updated with metaalert field + Map<String, Object> expectedAlert = new HashMap<>(alerts.get(2)); + expectedAlert + .put(METAALERT_FIELD, Collections.singletonList(metaAlertCreateResponse.getGuid())); + findUpdatedDoc(expectedAlert, "message_2", SENSOR_NAME); + } + } + } + + @Test + public void shouldAddAlertsToMetaAlert() throws Exception { + // Load alerts + List<Map<String, Object>> alerts = buildAlerts(4); + alerts.get(0).put(METAALERT_FIELD, Collections.singletonList("meta_alert")); + addRecords(alerts, getTestIndexFullName(), SENSOR_NAME); + + // Load metaAlert + Map<String, Object> metaAlert = buildMetaAlert("meta_alert", MetaAlertStatus.ACTIVE, + Optional.of(Collections.singletonList(alerts.get(0)))); + addRecords(Collections.singletonList(metaAlert), getMetaAlertIndex(), METAALERT_TYPE); + + // Verify load was successful + findCreatedDocs(Arrays.asList( + new GetRequest("message_0", SENSOR_NAME), + new GetRequest("message_1", SENSOR_NAME), + new GetRequest("message_2", SENSOR_NAME), + new GetRequest("message_3", SENSOR_NAME), + new GetRequest("meta_alert", METAALERT_TYPE) + )); + + // Build expected metaAlert after alerts are added + Map<String, Object> expectedMetaAlert = new HashMap<>(metaAlert); + + // Verify the proper alerts were added + @SuppressWarnings("unchecked") + List<Map<String, Object>> metaAlertAlerts = new ArrayList<>( + (List<Map<String, Object>>) expectedMetaAlert.get(ALERT_FIELD)); + // Alert 0 is already in the metaalert. Add alerts 1 and 2. + Map<String, Object> expectedAlert1 = alerts.get(1); + expectedAlert1.put(METAALERT_FIELD, Collections.singletonList("meta_alert")); + metaAlertAlerts.add(expectedAlert1); + Map<String, Object> expectedAlert2 = alerts.get(2); + expectedAlert2.put(METAALERT_FIELD, Collections.singletonList("meta_alert")); + metaAlertAlerts.add(expectedAlert2); + expectedMetaAlert.put(ALERT_FIELD, metaAlertAlerts); + + // Verify the counts were properly updated + expectedMetaAlert.put("average", 1.0d); + expectedMetaAlert.put("min", 0.0d); + expectedMetaAlert.put("median", 1.0d); + expectedMetaAlert.put("max", 2.0d); + expectedMetaAlert.put("count", 3); + expectedMetaAlert.put("sum", 3.0d); + expectedMetaAlert.put(getThreatTriageField(), 3.0d); + + { + // Verify alerts were successfully added to the meta alert + Assert.assertTrue(metaDao.addAlertsToMetaAlert("meta_alert", Arrays + .asList(new GetRequest("message_1", SENSOR_NAME), + new GetRequest("message_2", SENSOR_NAME)))); + findUpdatedDoc(expectedMetaAlert, "meta_alert", METAALERT_TYPE); + } + + { + // Verify False when alerts are already in a meta alert and no new alerts are added + Assert.assertFalse(metaDao.addAlertsToMetaAlert("meta_alert", Arrays + .asList(new GetRequest("message_0", SENSOR_NAME), + new GetRequest("message_1", SENSOR_NAME)))); + findUpdatedDoc(expectedMetaAlert, "meta_alert", METAALERT_TYPE); + } + + { + // Verify only 1 alert is added when a list of alerts only contains 1 alert that is not in the meta alert + metaAlertAlerts = (List<Map<String, Object>>) expectedMetaAlert.get(ALERT_FIELD); + Map<String, Object> expectedAlert3 = alerts.get(3); + expectedAlert3.put(METAALERT_FIELD, Collections.singletonList("meta_alert")); + metaAlertAlerts.add(expectedAlert3); + expectedMetaAlert.put(ALERT_FIELD, metaAlertAlerts); + + expectedMetaAlert.put("average", 1.5d); + expectedMetaAlert.put("min", 0.0d); + expectedMetaAlert.put("median", 1.5d); + expectedMetaAlert.put("max", 3.0d); + expectedMetaAlert.put("count", 4); + expectedMetaAlert.put("sum", 6.0d); + expectedMetaAlert.put(getThreatTriageField(), 6.0d); + + Assert.assertTrue(metaDao.addAlertsToMetaAlert("meta_alert", Arrays + .asList(new GetRequest("message_2", SENSOR_NAME), + new GetRequest("message_3", SENSOR_NAME)))); + findUpdatedDoc(expectedMetaAlert, "meta_alert", METAALERT_TYPE); + } + } + + @Test + @SuppressWarnings("unchecked") + public void shouldRemoveAlertsFromMetaAlert() throws Exception { + // Load alerts + List<Map<String, Object>> alerts = buildAlerts(4); + alerts.get(0).put(METAALERT_FIELD, Collections.singletonList("meta_alert")); + alerts.get(1).put(METAALERT_FIELD, Collections.singletonList("meta_alert")); + alerts.get(2).put(METAALERT_FIELD, Collections.singletonList("meta_alert")); + alerts.get(3).put(METAALERT_FIELD, Collections.singletonList("meta_alert")); + addRecords(alerts, getTestIndexFullName(), SENSOR_NAME); + + // Load metaAlert + Map<String, Object> metaAlert = buildMetaAlert("meta_alert", MetaAlertStatus.ACTIVE, + Optional.of(Arrays.asList(alerts.get(0), alerts.get(1), alerts.get(2), alerts.get(3)))); + addRecords(Collections.singletonList(metaAlert), getMetaAlertIndex(), METAALERT_TYPE); + + // Verify load was successful + findCreatedDocs(Arrays.asList( + new GetRequest("message_0", SENSOR_NAME), + new GetRequest("message_1", SENSOR_NAME), + new GetRequest("message_2", SENSOR_NAME), + new GetRequest("message_3", SENSOR_NAME), + new GetRequest("meta_alert", METAALERT_TYPE))); + + // Build expected metaAlert after alerts are added + Map<String, Object> expectedMetaAlert = new HashMap<>(metaAlert); + + // Verify the proper alerts were added + List<Map<String, Object>> metaAlertAlerts = new ArrayList<>( + (List<Map<String, Object>>) expectedMetaAlert.get(ALERT_FIELD)); + metaAlertAlerts.remove(0); + metaAlertAlerts.remove(0); + expectedMetaAlert.put(ALERT_FIELD, metaAlertAlerts); + + // Verify the counts were properly updated + expectedMetaAlert.put("average", 2.5d); + expectedMetaAlert.put("min", 2.0d); + expectedMetaAlert.put("median", 2.5d); + expectedMetaAlert.put("max", 3.0d); + expectedMetaAlert.put("count", 2); + expectedMetaAlert.put("sum", 5.0d); + expectedMetaAlert.put(getThreatTriageField(), 5.0d); + + { + // Verify a list of alerts are removed from a meta alert + Assert.assertTrue(metaDao.removeAlertsFromMetaAlert("meta_alert", Arrays + .asList(new GetRequest("message_0", SENSOR_NAME), + new GetRequest("message_1", SENSOR_NAME)))); + findUpdatedDoc(expectedMetaAlert, "meta_alert", METAALERT_TYPE); + } + + { + // Verify False when alerts are not present in a meta alert and no alerts are removed + Assert.assertFalse(metaDao.removeAlertsFromMetaAlert("meta_alert", Arrays + .asList(new GetRequest("message_0", SENSOR_NAME), + new GetRequest("message_1", SENSOR_NAME)))); + findUpdatedDoc(expectedMetaAlert, "meta_alert", METAALERT_TYPE); + } + + { + // Verify only 1 alert is removed when a list of alerts only contains 1 alert that is in the meta alert + metaAlertAlerts = new ArrayList<>( + (List<Map<String, Object>>) expectedMetaAlert.get(ALERT_FIELD)); + metaAlertAlerts.remove(0); + expectedMetaAlert.put(ALERT_FIELD, metaAlertAlerts); + + expectedMetaAlert.put("average", 3.0d); + expectedMetaAlert.put("min", 3.0d); + expectedMetaAlert.put("median", 3.0d); + expectedMetaAlert.put("max", 3.0d); + expectedMetaAlert.put("count", 1); + expectedMetaAlert.put("sum", 3.0d); + expectedMetaAlert.put(getThreatTriageField(), 3.0d); + + Assert.assertTrue(metaDao.removeAlertsFromMetaAlert("meta_alert", Arrays + .asList(new GetRequest("message_0", SENSOR_NAME), + new GetRequest("message_2", SENSOR_NAME)))); + findUpdatedDoc(expectedMetaAlert, "meta_alert", METAALERT_TYPE); + } + + { + // Verify all alerts are removed from a metaAlert + metaAlertAlerts = new ArrayList<>( + (List<Map<String, Object>>) expectedMetaAlert.get(ALERT_FIELD)); + metaAlertAlerts.remove(0); + if (isEmptyMetaAlertList()) { + expectedMetaAlert.put(ALERT_FIELD, metaAlertAlerts); + } else { + expectedMetaAlert.remove(ALERT_FIELD); + } + + expectedMetaAlert.put("average", 0.0d); + expectedMetaAlert.put("count", 0); + expectedMetaAlert.put("sum", 0.0d); + expectedMetaAlert.put(getThreatTriageField(), 0.0d); + + // Handle the cases with non-finite Double values on a per store basis + if (isFiniteDoubleOnly()) { + expectedMetaAlert.put("min", String.valueOf(Double.POSITIVE_INFINITY)); + expectedMetaAlert.put("median", String.valueOf(Double.NaN)); + expectedMetaAlert.put("max", String.valueOf(Double.NEGATIVE_INFINITY)); + } else { + expectedMetaAlert.put("min", Double.POSITIVE_INFINITY); + expectedMetaAlert.put("median", Double.NaN); + expectedMetaAlert.put("max", Double.NEGATIVE_INFINITY); + } + + // Verify removing alerts cannot result in an empty meta alert + try { + metaDao.removeAlertsFromMetaAlert("meta_alert", + Collections.singletonList(new GetRequest("message_3", SENSOR_NAME))); + Assert.fail("Removing these alerts will result in an empty meta alert. Empty meta alerts are not allowed."); + } catch (IllegalStateException ise) { + Assert.assertEquals("Removing these alerts will result in an empty meta alert. Empty meta alerts are not allowed.", + ise.getMessage()); + } + } + } + + @Test + public void addRemoveAlertsShouldThrowExceptionForInactiveMetaAlert() throws Exception { + // Load alerts + List<Map<String, Object>> alerts = buildAlerts(2); + alerts.get(0).put(METAALERT_FIELD, Collections.singletonList("meta_alert")); + addRecords(alerts, getTestIndexFullName(), SENSOR_NAME); + + // Load metaAlert + Map<String, Object> metaAlert = buildMetaAlert("meta_alert", MetaAlertStatus.INACTIVE, + Optional.of(Collections.singletonList(alerts.get(0)))); + addRecords(Collections.singletonList(metaAlert), getMetaAlertIndex(), METAALERT_TYPE); + + // Verify load was successful + findCreatedDocs(Arrays.asList( + new GetRequest("message_0", SENSOR_NAME), + new GetRequest("message_1", SENSOR_NAME), + new GetRequest("meta_alert", METAALERT_TYPE))); + + { + // Verify alerts cannot be added to an INACTIVE meta alert + try { + metaDao.addAlertsToMetaAlert("meta_alert", + Collections.singletonList(new GetRequest("message_1", SENSOR_NAME))); + Assert.fail("Adding alerts to an inactive meta alert should throw an exception"); + } catch (IllegalStateException ise) { + Assert.assertEquals("Adding alerts to an INACTIVE meta alert is not allowed", + ise.getMessage()); + } + } + + { + // Verify alerts cannot be removed from an INACTIVE meta alert + try { + metaDao.removeAlertsFromMetaAlert("meta_alert", + Collections.singletonList(new GetRequest("message_0", SENSOR_NAME))); + Assert.fail("Removing alerts from an inactive meta alert should throw an exception"); + } catch (IllegalStateException ise) { + Assert.assertEquals("Removing alerts from an INACTIVE meta alert is not allowed", + ise.getMessage()); + } + } + } + + @Test + public void shouldUpdateMetaAlertStatus() throws Exception { + int numChildAlerts = 25; + int numUnrelatedAlerts = 25; + int totalAlerts = numChildAlerts + numUnrelatedAlerts; + + // Load alerts + List<Map<String, Object>> alerts = buildAlerts(totalAlerts); + List<Map<String, Object>> childAlerts = alerts.subList(0, numChildAlerts); + List<Map<String, Object>> unrelatedAlerts = alerts.subList(numChildAlerts, totalAlerts); + for (Map<String, Object> alert : childAlerts) { + alert.put(METAALERT_FIELD, Collections.singletonList("meta_alert")); + } + addRecords(alerts, getTestIndexFullName(), SENSOR_NAME); + + // Load metaAlerts + Map<String, Object> metaAlert = buildMetaAlert("meta_alert", MetaAlertStatus.ACTIVE, + Optional.of(childAlerts)); + // We pass MetaAlertDao.METAALERT_TYPE, because the "_doc" gets appended automatically. + addRecords(Collections.singletonList(metaAlert), getMetaAlertIndex(), + METAALERT_TYPE); + + List<GetRequest> requests = new ArrayList<>(); + for (int i = 0; i < numChildAlerts; ++i) { + requests.add(new GetRequest("message_" + i, SENSOR_NAME)); + } + requests.add(new GetRequest("meta_alert", METAALERT_TYPE)); + + // Verify load was successful + findCreatedDocs(requests); + + { + // Verify status changed to inactive and child alerts are updated + Assert.assertTrue(metaDao.updateMetaAlertStatus("meta_alert", MetaAlertStatus.INACTIVE)); + + Map<String, Object> expectedMetaAlert = new HashMap<>(metaAlert); + expectedMetaAlert.put(STATUS_FIELD, MetaAlertStatus.INACTIVE.getStatusString()); + + findUpdatedDoc(expectedMetaAlert, "meta_alert", METAALERT_TYPE); + + for (int i = 0; i < numChildAlerts; ++i) { + Map<String, Object> expectedAlert = new HashMap<>(childAlerts.get(i)); + setEmptiedMetaAlertField(expectedAlert); + findUpdatedDoc(expectedAlert, "message_" + i, SENSOR_NAME); + } + + // Ensure unrelated alerts are unaffected + for (int i = 0; i < numUnrelatedAlerts; ++i) { + Map<String, Object> expectedAlert = new HashMap<>(unrelatedAlerts.get(i)); + // Make sure to handle the guid offset from creation + findUpdatedDoc(expectedAlert, "message_" + (i + numChildAlerts), SENSOR_NAME); + } + } + + { + // Verify status changed to active and child alerts are updated + Assert.assertTrue(metaDao.updateMetaAlertStatus("meta_alert", MetaAlertStatus.ACTIVE)); + + Map<String, Object> expectedMetaAlert = new HashMap<>(metaAlert); + expectedMetaAlert.put(STATUS_FIELD, MetaAlertStatus.ACTIVE.getStatusString()); + + findUpdatedDoc(expectedMetaAlert, "meta_alert", METAALERT_TYPE); + + for (int i = 0; i < numChildAlerts; ++i) { + Map<String, Object> expectedAlert = new HashMap<>(alerts.get(i)); + expectedAlert.put("metaalerts", Collections.singletonList("meta_alert")); + findUpdatedDoc(expectedAlert, "message_" + i, SENSOR_NAME); + } + + // Ensure unrelated alerts are unaffected + for (int i = 0; i < numUnrelatedAlerts; ++i) { + Map<String, Object> expectedAlert = new HashMap<>(unrelatedAlerts.get(i)); + // Make sure to handle the guid offset from creation + findUpdatedDoc(expectedAlert, "message_" + (i + numChildAlerts), SENSOR_NAME); + } + + { + // Verify status changed to current status has no effect + Assert.assertFalse(metaDao.updateMetaAlertStatus("meta_alert", MetaAlertStatus.ACTIVE)); + + findUpdatedDoc(expectedMetaAlert, "meta_alert", METAALERT_TYPE); + + for (int i = 0; i < numChildAlerts; ++i) { + Map<String, Object> expectedAlert = new HashMap<>(alerts.get(i)); + expectedAlert.put("metaalerts", Collections.singletonList("meta_alert")); + findUpdatedDoc(expectedAlert, "message_" + i, SENSOR_NAME); + } + + // Ensure unrelated alerts are unaffected + for (int i = 0; i < numUnrelatedAlerts; ++i) { + Map<String, Object> expectedAlert = new HashMap<>(unrelatedAlerts.get(i)); + // Make sure to handle the guid offset from creation + findUpdatedDoc(expectedAlert, "message_" + (i + numChildAlerts), SENSOR_NAME); + } + } + } + } + + @Test + public void shouldSearchByStatus() throws Exception { + // Load alert + List<Map<String, Object>> alerts = buildAlerts(1); + alerts.get(0).put(METAALERT_FIELD, Collections.singletonList("meta_active")); + alerts.get(0).put("ip_src_addr", "192.168.1.1"); + alerts.get(0).put("ip_src_port", 8010); + + // Load metaAlerts + Map<String, Object> activeMetaAlert = buildMetaAlert("meta_active", MetaAlertStatus.ACTIVE, + Optional.of(Collections.singletonList(alerts.get(0)))); + Map<String, Object> inactiveMetaAlert = buildMetaAlert("meta_inactive", + MetaAlertStatus.INACTIVE, + Optional.empty()); + + // We pass MetaAlertDao.METAALERT_TYPE, because the "_doc" gets appended automatically. + addRecords(Arrays.asList(activeMetaAlert, inactiveMetaAlert), getMetaAlertIndex(), + METAALERT_TYPE); + + // Verify load was successful + findCreatedDocs(Arrays.asList( + new GetRequest("meta_active", METAALERT_TYPE), + new GetRequest("meta_inactive", METAALERT_TYPE))); + + SearchResponse searchResponse = metaDao.search(new SearchRequest() { + { + setQuery("*:*"); + setIndices(Collections.singletonList(METAALERT_TYPE)); + setFrom(0); + setSize(5); + setSort(Collections.singletonList(new SortField() {{ + setField(Constants.GUID); + }})); + } + }); + + // Verify only active meta alerts are returned + Assert.assertEquals(1, searchResponse.getTotal()); + Assert.assertEquals(MetaAlertStatus.ACTIVE.getStatusString(), + searchResponse.getResults().get(0).getSource().get(STATUS_FIELD)); + } + + + @Test + public void shouldHidesAlertsOnGroup() throws Exception { + // Load alerts + List<Map<String, Object>> alerts = buildAlerts(2); + alerts.get(0).put(METAALERT_FIELD, Collections.singletonList("meta_active")); + alerts.get(0).put("ip_src_addr", "192.168.1.1"); + alerts.get(0).put("score", 1); + alerts.get(1).put("ip_src_addr", "192.168.1.1"); + alerts.get(1).put("score", 10); + addRecords(alerts, getTestIndexFullName(), SENSOR_NAME); + + // Put the nested type into the test index, so that it'll match appropriately + setupTypings(); + + // Don't need any meta alerts to actually exist, since we've populated the field on the alerts. + + // Verify load was successful + findCreatedDocs(Arrays.asList( + new GetRequest("message_0", SENSOR_NAME), + new GetRequest("message_1", SENSOR_NAME))); + + // Build our group request + Group searchGroup = new Group(); + searchGroup.setField("ip_src_addr"); + List<Group> groupList = new ArrayList<>(); + groupList.add(searchGroup); + GroupResponse groupResponse = metaDao.group(new GroupRequest() { + { + setQuery("ip_src_addr:192.168.1.1"); + setIndices(queryIndices); + setScoreField("score"); + setGroups(groupList); + } + }); + + // Should only return the standalone alert in the group + GroupResult result = groupResponse.getGroupResults().get(0); + Assert.assertEquals(1, result.getTotal()); + Assert.assertEquals("192.168.1.1", result.getKey()); + // No delta, since no ops happen + Assert.assertEquals(10.0d, result.getScore(), 0.0d); + } + + // This test is important enough that everyone should implement it, but is pretty specific to + // implementation + @Test + public abstract void shouldSearchByNestedAlert() throws Exception; + + @SuppressWarnings("unchecked") + @Test + public void shouldUpdateMetaAlertOnAlertUpdate() throws Exception { + // Load alerts + List<Map<String, Object>> alerts = buildAlerts(2); + alerts.get(0).put(METAALERT_FIELD, Arrays.asList("meta_active", "meta_inactive")); + addRecords(alerts, getTestIndexFullName(), SENSOR_NAME); + + // Load metaAlerts + Map<String, Object> activeMetaAlert = buildMetaAlert("meta_active", MetaAlertStatus.ACTIVE, + Optional.of(Collections.singletonList(alerts.get(0)))); + Map<String, Object> inactiveMetaAlert = buildMetaAlert("meta_inactive", + MetaAlertStatus.INACTIVE, + Optional.of(Collections.singletonList(alerts.get(0)))); + // We pass MetaAlertDao.METAALERT_TYPE, because the "_doc" gets appended automatically. + addRecords(Arrays.asList(activeMetaAlert, inactiveMetaAlert), getMetaAlertIndex(), + METAALERT_TYPE); + + // Verify load was successful + findCreatedDocs(Arrays.asList( + new GetRequest("message_0", SENSOR_NAME), + new GetRequest("message_1", SENSOR_NAME), + new GetRequest("meta_active", METAALERT_TYPE), + new GetRequest("meta_inactive", METAALERT_TYPE))); + + { + // Modify the first message and add a new field + Map<String, Object> message0 = new HashMap<String, Object>(alerts.get(0)) { + { + put(NEW_FIELD, "metron"); + put(THREAT_FIELD_DEFAULT, 10.0d); + } + }; + String guid = "" + message0.get(Constants.GUID); + metaDao.update(new Document(message0, guid, SENSOR_NAME, null), + Optional.of(getTestIndexFullName())); + + { + // Verify alerts are up-to-date + findUpdatedDoc(message0, guid, SENSOR_NAME); + long cnt = getMatchingAlertCount(NEW_FIELD, message0.get(NEW_FIELD)); + if (cnt == 0) { + Assert.fail("Alert not updated!"); + } + } + + { + // Verify meta alerts are up-to-date + long cnt = getMatchingMetaAlertCount(NEW_FIELD, "metron"); + if (cnt == 0) { + Assert.fail("Active metaalert was not updated!"); + } + if (cnt != 1) { + Assert.fail("Metaalerts not updated correctly!"); + } + } + } + //modify the same message and modify the new field + { + Map<String, Object> message0 = new HashMap<String, Object>(alerts.get(0)) { + { + put(NEW_FIELD, "metron2"); + } + }; + String guid = "" + message0.get(Constants.GUID); + metaDao.update(new Document(message0, guid, SENSOR_NAME, null), Optional.empty()); + + { + // Verify index is up-to-date + findUpdatedDoc(message0, guid, SENSOR_NAME); + long cnt = getMatchingAlertCount(NEW_FIELD, message0.get(NEW_FIELD)); + if (cnt == 0) { + Assert.fail("Alert not updated!"); + } + } + { + // Verify meta alerts are up-to-date + long cnt = getMatchingMetaAlertCount(NEW_FIELD, "metron2"); + if (cnt == 0) { + Assert.fail("Active metaalert was not updated!"); + } + if (cnt != 1) { + Assert.fail("Metaalerts not updated correctly!"); + } + } + } + } + + @Test + public void shouldThrowExceptionOnMetaAlertUpdate() throws Exception { + Document metaAlert = new Document(new HashMap<>(), "meta_alert", METAALERT_TYPE, 0L); + try { + // Verify a meta alert cannot be updated in the meta alert dao + metaDao.update(metaAlert, Optional.empty()); + Assert.fail("Direct meta alert update should throw an exception"); + } catch (UnsupportedOperationException uoe) { + Assert.assertEquals("Meta alerts cannot be directly updated", uoe.getMessage()); + } + } + + @Test + public void shouldPatchAllowedMetaAlerts() throws Exception { + // Load alerts + List<Map<String, Object>> alerts = buildAlerts(2); + alerts.get(0).put(METAALERT_FIELD, Collections.singletonList("meta_active")); + alerts.get(1).put(METAALERT_FIELD, Collections.singletonList("meta_active")); + addRecords(alerts, getTestIndexFullName(), SENSOR_NAME); + + // Put the nested type into the test index, so that it'll match appropriately + setupTypings(); + + // Load metaAlerts + Map<String, Object> metaAlert = buildMetaAlert("meta_alert", MetaAlertStatus.ACTIVE, + Optional.of(Arrays.asList(alerts.get(0), alerts.get(1)))); + // We pass MetaAlertDao.METAALERT_TYPE, because the "_doc" gets appended automatically. + addRecords(Collections.singletonList(metaAlert), getMetaAlertIndex(), METAALERT_TYPE); + + // Verify load was successful + findCreatedDocs(Arrays.asList( + new GetRequest("message_0", SENSOR_NAME), + new GetRequest("message_1", SENSOR_NAME), + new GetRequest("meta_alert", METAALERT_TYPE))); + + Map<String, Object> expectedMetaAlert = new HashMap<>(metaAlert); + expectedMetaAlert.put(NAME_FIELD, "New Meta Alert"); + { + // Verify a patch to a field other than "status" or "alert" can be patched + String namePatch = namePatchRequest.replace(META_INDEX_FLAG, getMetaAlertIndex()); + PatchRequest patchRequest = JSONUtils.INSTANCE.load(namePatch, PatchRequest.class); + metaDao.patch(metaDao, patchRequest, Optional.of(System.currentTimeMillis())); + + findUpdatedDoc(expectedMetaAlert, "meta_alert", METAALERT_TYPE); + } + + { + // Verify a patch to an alert field should throw an exception + try { + String alertPatch = alertPatchRequest.replace(META_INDEX_FLAG, getMetaAlertIndex()); + PatchRequest patchRequest = JSONUtils.INSTANCE.load(alertPatch, PatchRequest.class); + metaDao.patch(metaDao, patchRequest, Optional.of(System.currentTimeMillis())); + + Assert.fail("A patch on the alert field should throw an exception"); + } catch (IllegalArgumentException iae) { + Assert.assertEquals("Meta alert patches are not allowed for /alert or /status paths. " + + "Please use the add/remove alert or update status functions instead.", + iae.getMessage()); + } + + // Verify the metaAlert was not updated + findUpdatedDoc(expectedMetaAlert, "meta_alert", METAALERT_TYPE); + } + + { + // Verify a patch to a status field should throw an exception + try { + String statusPatch = statusPatchRequest + .replace(META_INDEX_FLAG, getMetaAlertIndex()); + PatchRequest patchRequest = JSONUtils.INSTANCE.load(statusPatch, PatchRequest.class); + metaDao.patch(metaDao, patchRequest, Optional.of(System.currentTimeMillis())); + + Assert.fail("A patch on the status field should throw an exception"); + } catch (IllegalArgumentException iae) { + Assert.assertEquals("Meta alert patches are not allowed for /alert or /status paths. " + + "Please use the add/remove alert or update status functions instead.", + iae.getMessage()); + } + + // Verify the metaAlert was not updated + findUpdatedDoc(expectedMetaAlert, "meta_alert", METAALERT_TYPE); + } + } + + protected void findUpdatedDoc(Map<String, Object> message0, String guid, String sensorType) + throws InterruptedException, IOException, OriginalNotFoundException { + commit(); + for (int t = 0; t < MAX_RETRIES; ++t, Thread.sleep(SLEEP_MS)) { + Document doc = metaDao.getLatest(guid, sensorType); + // Change the underlying document alerts lists to sets to avoid ordering issues. + convertAlertsFieldToSet(doc.getDocument()); + convertAlertsFieldToSet(message0); + + if (doc.getDocument() != null && message0.equals(doc.getDocument())) { + convertAlertsFieldToList(doc.getDocument()); + convertAlertsFieldToList(message0); + return; + } + } + + throw new OriginalNotFoundException( + "Count not find " + guid + " after " + MAX_RETRIES + " tries"); + } + + protected void convertAlertsFieldToSet(Map<String, Object> document) { + if (document.get(ALERT_FIELD) instanceof List) { + @SuppressWarnings("unchecked") + List<Map<String, Object>> message0AlertField = (List<Map<String, Object>>) document + .get(ALERT_FIELD); + Set<Map<String, Object>> message0AlertSet = new HashSet<>(message0AlertField); + document.put(ALERT_FIELD, message0AlertSet); + } + } + + protected void convertAlertsFieldToList(Map<String, Object> document) { + if (document.get(ALERT_FIELD) instanceof Set) { + @SuppressWarnings("unchecked") + Set<Map<String, Object>> message0AlertField = (Set<Map<String, Object>>) document + .get(ALERT_FIELD); + List<Map<String, Object>> message0AlertList = new ArrayList<>(message0AlertField); + message0AlertList.sort(Comparator.comparing(o -> ((String) o.get(Constants.GUID)))); + document.put(ALERT_FIELD, message0AlertList); + } + } + + protected boolean findCreatedDoc(String guid, String sensorType) + throws InterruptedException, IOException, OriginalNotFoundException { + for (int t = 0; t < MAX_RETRIES; ++t, Thread.sleep(SLEEP_MS)) { + Document doc = metaDao.getLatest(guid, sensorType); + if (doc != null) { + return true; + } + } + throw new OriginalNotFoundException( + "Count not find " + guid + " after " + MAX_RETRIES + "tries"); + } + + protected boolean findCreatedDocs(List<GetRequest> getRequests) + throws InterruptedException, IOException, OriginalNotFoundException { + for (int t = 0; t < MAX_RETRIES; ++t, Thread.sleep(SLEEP_MS)) { + Iterable<Document> docs = metaDao.getAllLatest(getRequests); + if (docs != null) { + int docCount = 0; + for (Document doc : docs) { + docCount++; + } + if (getRequests.size() == docCount) { + return true; + } + } + } + throw new OriginalNotFoundException("Count not find guids after " + MAX_RETRIES + "tries"); + } + + protected List<Map<String, Object>> buildAlerts(int count) { + List<Map<String, Object>> inputData = new ArrayList<>(); + for (int i = 0; i < count; ++i) { + final String guid = "message_" + i; + Map<String, Object> alerts = new HashMap<>(); + alerts.put(Constants.GUID, guid); + alerts.put(getSourceTypeField(), SENSOR_NAME); + alerts.put(THREAT_FIELD_DEFAULT, (double) i); + alerts.put("timestamp", System.currentTimeMillis()); + inputData.add(alerts); + } + return inputData; + } + + protected List<Map<String, Object>> buildMetaAlerts(int count, MetaAlertStatus status, + Optional<List<Map<String, Object>>> alerts) { + List<Map<String, Object>> inputData = new ArrayList<>(); + for (int i = 0; i < count; ++i) { + final String guid = "meta_" + status.getStatusString() + "_" + i; + inputData.add(buildMetaAlert(guid, status, alerts)); + } + return inputData; + } + + protected Map<String, Object> buildMetaAlert(String guid, MetaAlertStatus status, + Optional<List<Map<String, Object>>> alerts) { + Map<String, Object> metaAlert = new HashMap<>(); + metaAlert.put(Constants.GUID, guid); + metaAlert.put(getSourceTypeField(), METAALERT_TYPE); + metaAlert.put(STATUS_FIELD, status.getStatusString()); + if (alerts.isPresent()) { + List<Map<String, Object>> alertsList = alerts.get(); + metaAlert.put(ALERT_FIELD, alertsList); + } + return metaAlert; + } + + protected abstract long getMatchingAlertCount(String fieldName, Object fieldValue) + throws IOException, InterruptedException; + + protected abstract void addRecords(List<Map<String, Object>> inputData, String index, + String docType) throws IOException; + + protected abstract long getMatchingMetaAlertCount(String fieldName, String fieldValue) + throws IOException, InterruptedException; + + protected abstract void setupTypings(); + + // Get the base index name without any adjustments (e.g. without ES's "_index") + protected abstract String getTestIndexName(); + + // Get the full name of the test index. E.g. Elasticsearch appends "_index" + protected String getTestIndexFullName() { + return getTestIndexName(); + } + + protected abstract String getMetaAlertIndex(); + + protected abstract String getSourceTypeField(); + + protected String getThreatTriageField() { + return THREAT_FIELD_DEFAULT; + } + + // Allow for impls to do any commit they need to do. + protected void commit() throws IOException { + } + + // Different stores can have different representations of empty metaalerts field. + // E.g. Solr expects the field to not be present, ES expects it to be empty. + protected abstract void setEmptiedMetaAlertField(Map<String, Object> docMap); + + // Different stores may choose to store non finite double values as Strings. + // E.g. NaN may be a string, not a double value. + protected abstract boolean isFiniteDoubleOnly(); + + // Different stores may choose to return empty alerts lists differently. + // E.g. It may be missing completely, or may be an empty list + protected abstract boolean isEmptyMetaAlertList(); +} http://git-wip-us.apache.org/repos/asf/metron/blob/49f851e0/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/metaalert/MetaScoresTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/metaalert/MetaScoresTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/metaalert/MetaScoresTest.java new file mode 100644 index 0000000..1359ba9 --- /dev/null +++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/metaalert/MetaScoresTest.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.indexing.dao.metaalert; + +import static org.apache.metron.indexing.dao.metaalert.MetaAlertConstants.ALERT_FIELD; +import static org.apache.metron.indexing.dao.metaalert.MetaAlertConstants.METAALERT_TYPE; +import static org.apache.metron.indexing.dao.metaalert.MetaAlertConstants.THREAT_FIELD_DEFAULT; +import static org.apache.metron.indexing.dao.metaalert.MetaAlertConstants.THREAT_SORT_DEFAULT; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.metron.indexing.dao.update.Document; +import org.junit.Test; + +public class MetaScoresTest { + @Test + public void testCalculateMetaScoresList() { + final double delta = 0.001; + List<Map<String, Object>> alertList = new ArrayList<>(); + + // add an alert with a threat score + alertList.add(Collections.singletonMap(THREAT_FIELD_DEFAULT, 10.0f)); + + // add a second alert with a threat score + alertList.add(Collections.singletonMap(THREAT_FIELD_DEFAULT, 20.0f)); + + // add a third alert with NO threat score + alertList.add(Collections.singletonMap("alert3", "has no threat score")); + + // create the metaalert + Map<String, Object> docMap = new HashMap<>(); + docMap.put(ALERT_FIELD, alertList); + Document metaalert = new Document(docMap, "guid", METAALERT_TYPE, 0L); + + // calculate the threat score for the metaalert + MetaScores.calculateMetaScores(metaalert, THREAT_FIELD_DEFAULT, THREAT_SORT_DEFAULT); + + // the metaalert must contain a summary of all child threat scores + assertEquals(20D, (Double) metaalert.getDocument().get("max"), delta); + assertEquals(10D, (Double) metaalert.getDocument().get("min"), delta); + assertEquals(15D, (Double) metaalert.getDocument().get("average"), delta); + assertEquals(2L, metaalert.getDocument().get("count")); + assertEquals(30D, (Double) metaalert.getDocument().get("sum"), delta); + assertEquals(15D, (Double) metaalert.getDocument().get("median"), delta); + + // it must contain an overall threat score; a float to match the type of the threat score of + // the other sensor indices + Object threatScore = metaalert.getDocument().get(THREAT_FIELD_DEFAULT); + assertTrue(threatScore instanceof Float); + + // by default, the overall threat score is the sum of all child threat scores + assertEquals(30.0F, threatScore); + } +}