Repository: metron Updated Branches: refs/heads/master 309d3757d -> 40c93527e
http://git-wip-us.apache.org/repos/asf/metron/blob/40c93527/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDaoTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDaoTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDaoTest.java new file mode 100644 index 0000000..02ea795 --- /dev/null +++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDaoTest.java @@ -0,0 +1,427 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.elasticsearch.dao; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; +import org.apache.metron.common.Constants; +import org.apache.metron.common.Constants.Fields; +import org.apache.metron.indexing.dao.AccessConfig; +import org.apache.metron.indexing.dao.IndexDao; +import org.apache.metron.indexing.dao.MetaAlertDao; +import org.apache.metron.indexing.dao.MultiIndexDao; +import org.apache.metron.indexing.dao.metaalert.MetaAlertCreateRequest; +import org.apache.metron.indexing.dao.metaalert.MetaScores; +import org.apache.metron.indexing.dao.search.FieldType; +import org.apache.metron.indexing.dao.search.GroupRequest; +import org.apache.metron.indexing.dao.search.GroupResponse; +import org.apache.metron.indexing.dao.search.InvalidCreateException; +import org.apache.metron.indexing.dao.search.InvalidSearchException; +import org.apache.metron.indexing.dao.search.SearchRequest; +import org.apache.metron.indexing.dao.search.SearchResponse; +import org.apache.metron.indexing.dao.update.Document; +import org.elasticsearch.action.get.GetResponse; +import org.elasticsearch.action.get.MultiGetItemResponse; +import org.elasticsearch.action.get.MultiGetResponse; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.SearchHitField; +import org.elasticsearch.search.SearchHits; +import org.json.simple.JSONArray; +import org.json.simple.JSONObject; +import org.json.simple.parser.JSONParser; +import org.json.simple.parser.ParseException; +import org.junit.Test; + +public class ElasticsearchMetaAlertDaoTest { + + @Test + @SuppressWarnings("unchecked") + public void testBuildUpdatedMetaAlertSingleAlert() throws IOException, ParseException { + // Construct the expected result + JSONObject expected = new JSONObject(); + expected.put("average", 5.0); + expected.put("min", 5.0); + expected.put("median", 5.0); + expected.put("max", 5.0); + expected.put("count", 1L); + expected.put(Constants.GUID, "m1"); + expected.put("sum", 5.0); + expected.put(MetaAlertDao.STATUS_FIELD, MetaAlertStatus.ACTIVE.getStatusString()); + JSONArray expectedAlerts = new JSONArray(); + JSONObject expectedAlert = new JSONObject(); + expectedAlert.put(MetaAlertDao.THREAT_FIELD_DEFAULT, 5L); + expectedAlert.put("fakekey", "fakevalue"); + expectedAlerts.add(expectedAlert); + expected.put(MetaAlertDao.ALERT_FIELD, expectedAlerts); + + // Construct the meta alert object + Map<String, Object> metaSource = new HashMap<>(); + metaSource.put(Constants.GUID, "m1"); + metaSource.put(MetaAlertDao.STATUS_FIELD, MetaAlertStatus.ACTIVE.getStatusString()); + List<Double> alertScores = new ArrayList<>(); + alertScores.add(10d); + metaSource.putAll(new MetaScores(alertScores).getMetaScores()); + SearchHit metaHit = mock(SearchHit.class); + when(metaHit.getSource()).thenReturn(metaSource); + + // Construct the inner alert + SearchHit innerAlertHit = mock(SearchHit.class); + HashMap<String, Object> innerAlertSource = new HashMap<>(); + innerAlertSource.put(Constants.GUID, "a1"); + when(innerAlertHit.sourceAsMap()).thenReturn(innerAlertSource); + SearchHitField field = mock(SearchHitField.class); + when(field.getValue()).thenReturn(10d); + when(innerAlertHit.field(MetaAlertDao.THREAT_FIELD_DEFAULT)).thenReturn(field); + SearchHit[] innerHitArray = new SearchHit[1]; + innerHitArray[0] = innerAlertHit; + + // Construct the inner hits that contains the alert + SearchHits searchHits = mock(SearchHits.class); + when(searchHits.getHits()).thenReturn(innerHitArray); + Map<String, SearchHits> innerHits = new HashMap<>(); + innerHits.put(MetaAlertDao.ALERT_FIELD, searchHits); + when(metaHit.getInnerHits()).thenReturn(innerHits); + + // Construct the updated Document + Map<String, Object> updateMap = new HashMap<>(); + updateMap.put(MetaAlertDao.THREAT_FIELD_DEFAULT, 5); + updateMap.put("fakekey", "fakevalue"); + Document update = new Document(updateMap, "a1", "bro_doc", 0L); + + ElasticsearchDao esDao = new ElasticsearchDao(); + ElasticsearchMetaAlertDao emaDao = new ElasticsearchMetaAlertDao(); + emaDao.init(esDao); + XContentBuilder builder = emaDao.buildUpdatedMetaAlert(update, metaHit); + JSONParser parser = new JSONParser(); + Object obj = parser.parse(builder.string()); + JSONObject actual = (JSONObject) obj; + + assertEquals(expected, actual); + } + + @Test + @SuppressWarnings("unchecked") + public void testBuildUpdatedMetaAlertMultipleAlerts() throws IOException, ParseException { + // Construct the expected result + JSONObject expected = new JSONObject(); + expected.put("average", 7.5); + expected.put("min", 5.0); + expected.put("median", 7.5); + expected.put("max", 10.0); + expected.put("count", 2L); + expected.put(Constants.GUID, "m1"); + expected.put("sum", 15.0); + expected.put(MetaAlertDao.STATUS_FIELD, MetaAlertStatus.ACTIVE.getStatusString()); + JSONArray expectedAlerts = new JSONArray(); + JSONObject expectedAlertOne = new JSONObject(); + expectedAlertOne.put(MetaAlertDao.THREAT_FIELD_DEFAULT, 5d); + expectedAlertOne.put("fakekey", "fakevalue"); + expectedAlerts.add(expectedAlertOne); + JSONObject expectedAlertTwo = new JSONObject(); + expectedAlertTwo.put(MetaAlertDao.THREAT_FIELD_DEFAULT, 10d); + String guidTwo = "a2"; + expectedAlertTwo.put(Constants.GUID, guidTwo); + expectedAlerts.add(expectedAlertTwo); + expected.put(MetaAlertDao.ALERT_FIELD, expectedAlerts); + + // Construct the meta alert object + Map<String, Object> metaSource = new HashMap<>(); + metaSource.put(Constants.GUID, "m1"); + metaSource.put(MetaAlertDao.STATUS_FIELD, MetaAlertStatus.ACTIVE.getStatusString()); + double threatValueOne = 5d; + double threatValueTwo = 10d; + List<Double> alertScores = new ArrayList<>(); + alertScores.add(threatValueOne); + alertScores.add(threatValueTwo); + metaSource.putAll(new MetaScores(alertScores).getMetaScores()); + SearchHit metaHit = mock(SearchHit.class); + when(metaHit.getSource()).thenReturn(metaSource); + + // Construct the inner alerts + SearchHit innerAlertHitOne = mock(SearchHit.class); + HashMap<String, Object> innerAlertSourceOne = new HashMap<>(); + String guidOne = "a1"; + innerAlertSourceOne.put(Constants.GUID, guidOne); + when(innerAlertHitOne.sourceAsMap()).thenReturn(innerAlertSourceOne); + when(innerAlertHitOne.getId()).thenReturn(guidOne); + SearchHitField triageOne = mock(SearchHitField.class); + when(triageOne.getValue()).thenReturn(threatValueOne); + Map<String, Object> innerAlertHitOneSource = new HashMap<>(); + innerAlertHitOneSource.put(MetaAlertDao.THREAT_FIELD_DEFAULT, threatValueTwo); + innerAlertHitOneSource.put(Constants.GUID, guidOne); + when(innerAlertHitOne.getSource()).thenReturn(innerAlertHitOneSource); + when(innerAlertHitOne.field(MetaAlertDao.THREAT_FIELD_DEFAULT)).thenReturn(triageOne); + + SearchHit innerAlertHitTwo = mock(SearchHit.class); + HashMap<String, Object> innerAlertSourceTwo = new HashMap<>(); + innerAlertSourceTwo.put(Constants.GUID, guidTwo); + when(innerAlertHitTwo.sourceAsMap()).thenReturn(innerAlertSourceTwo); + when(innerAlertHitOne.getId()).thenReturn(guidTwo); + SearchHitField triageTwo = mock(SearchHitField.class); + when(triageTwo.getValue()).thenReturn(threatValueTwo); + Map<String, Object> innerAlertHitTwoSource = new HashMap<>(); + innerAlertHitTwoSource.put(MetaAlertDao.THREAT_FIELD_DEFAULT, threatValueTwo); + innerAlertHitTwoSource.put(Constants.GUID, guidTwo); + when(innerAlertHitTwo.getSource()).thenReturn(innerAlertHitTwoSource); + when(innerAlertHitTwo.field(MetaAlertDao.THREAT_FIELD_DEFAULT)).thenReturn(triageTwo); + + SearchHit[] innerHitArray = new SearchHit[2]; + innerHitArray[0] = innerAlertHitOne; + innerHitArray[1] = innerAlertHitTwo; + + // Construct the inner hits that contains the alert + SearchHits searchHits = mock(SearchHits.class); + when(searchHits.getHits()).thenReturn(innerHitArray); + Map<String, SearchHits> innerHits = new HashMap<>(); + innerHits.put(MetaAlertDao.ALERT_FIELD, searchHits); + when(metaHit.getInnerHits()).thenReturn(innerHits); + + // Construct the updated Document + Map<String, Object> updateMap = new HashMap<>(); + updateMap.put(MetaAlertDao.THREAT_FIELD_DEFAULT, threatValueOne); + updateMap.put("fakekey", "fakevalue"); + Document update = new Document(updateMap, guidOne, "bro_doc", 0L); + + ElasticsearchDao esDao = new ElasticsearchDao(); + ElasticsearchMetaAlertDao emaDao = new ElasticsearchMetaAlertDao(); + MultiIndexDao multiIndexDao = new MultiIndexDao(esDao); + emaDao.init(multiIndexDao); + XContentBuilder builder = emaDao.buildUpdatedMetaAlert(update, metaHit); + + JSONParser parser = new JSONParser(); + Object obj = parser.parse(builder.string()); + JSONObject actual = (JSONObject) obj; + + assertEquals(expected, actual); + } + + @Test(expected = IllegalArgumentException.class) + public void testInvalidInit() { + IndexDao dao = new IndexDao() { + @Override + public SearchResponse search(SearchRequest searchRequest) throws InvalidSearchException { + return null; + } + + @Override + public GroupResponse group(GroupRequest groupRequest) throws InvalidSearchException { + return null; + } + + @Override + public void init(AccessConfig config) { + } + + @Override + public Document getLatest(String guid, String sensorType) throws IOException { + return null; + } + + @Override + public void update(Document update, Optional<String> index) throws IOException { + } + + @Override + public Map<String, Map<String, FieldType>> getColumnMetadata(List<String> indices) + throws IOException { + return null; + } + + @Override + public Map<String, FieldType> getCommonColumnMetadata(List<String> indices) + throws IOException { + return null; + } + }; + ElasticsearchMetaAlertDao metaAlertDao = new ElasticsearchMetaAlertDao(); + metaAlertDao.init(dao); + } + + @Test + public void testBuildCreateDocumentSingleAlert() throws InvalidCreateException, IOException { + ElasticsearchDao esDao = new ElasticsearchDao(); + ElasticsearchMetaAlertDao emaDao = new ElasticsearchMetaAlertDao(); + emaDao.init(esDao); + + List<String> groups = new ArrayList<>(); + groups.add("group_one"); + groups.add("group_two"); + + // Build the first response from the multiget + Map<String, Object> alertOne = new HashMap<>(); + alertOne.put(Constants.GUID, "alert_one"); + alertOne.put(MetaAlertDao.THREAT_FIELD_DEFAULT, 10.0d); + GetResponse getResponseOne = mock(GetResponse.class); + when(getResponseOne.isExists()).thenReturn(true); + when(getResponseOne.getSource()).thenReturn(alertOne); + MultiGetItemResponse multiGetItemResponseOne = mock(MultiGetItemResponse.class); + when(multiGetItemResponseOne.getResponse()).thenReturn(getResponseOne); + + // Add it to the iterator + @SuppressWarnings("unchecked") + Iterator<MultiGetItemResponse> mockIterator = mock(Iterator.class); + when(mockIterator.hasNext()).thenReturn(true, false); + when(mockIterator.next()).thenReturn(multiGetItemResponseOne); + + // Add it to the response + MultiGetResponse mockResponse = mock(MultiGetResponse.class); + when(mockResponse.iterator()).thenReturn(mockIterator); + + // Actually build the doc + Document actual = emaDao.buildCreateDocument(mockResponse, groups); + + ArrayList<Map<String, Object>> alertList = new ArrayList<>(); + alertList.add(alertOne); + + Map<String, Object> actualDocument = actual.getDocument(); + assertEquals( + MetaAlertStatus.ACTIVE.getStatusString(), + actualDocument.get(MetaAlertDao.STATUS_FIELD) + ); + assertArrayEquals( + alertList.toArray(), + (Object[]) actualDocument.get(MetaAlertDao.ALERT_FIELD) + ); + assertArrayEquals( + groups.toArray(), + (Object[]) actualDocument.get(MetaAlertDao.GROUPS_FIELD) + ); + + // Don't care about the result, just that it's a UUID. Exception will be thrown if not. + UUID.fromString((String) actualDocument.get(Constants.GUID)); + } + + @Test + public void testBuildCreateDocumentMultipleAlerts() throws InvalidCreateException, IOException { + ElasticsearchDao esDao = new ElasticsearchDao(); + ElasticsearchMetaAlertDao emaDao = new ElasticsearchMetaAlertDao(); + emaDao.init(esDao); + + List<String> groups = new ArrayList<>(); + groups.add("group_one"); + groups.add("group_two"); + + // Build the first response from the multiget + Map<String, Object> alertOne = new HashMap<>(); + alertOne.put(Constants.GUID, "alert_one"); + alertOne.put(MetaAlertDao.THREAT_FIELD_DEFAULT, 10.0d); + GetResponse getResponseOne = mock(GetResponse.class); + when(getResponseOne.isExists()).thenReturn(true); + when(getResponseOne.getSource()).thenReturn(alertOne); + MultiGetItemResponse multiGetItemResponseOne = mock(MultiGetItemResponse.class); + when(multiGetItemResponseOne.getResponse()).thenReturn(getResponseOne); + + // Build the second response from the multiget + Map<String, Object> alertTwo = new HashMap<>(); + alertTwo.put(Constants.GUID, "alert_one"); + alertTwo.put(MetaAlertDao.THREAT_FIELD_DEFAULT, 5.0d); + GetResponse getResponseTwo = mock(GetResponse.class); + when(getResponseTwo.isExists()).thenReturn(true); + when(getResponseTwo.getSource()).thenReturn(alertTwo); + MultiGetItemResponse multiGetItemResponseTwo = mock(MultiGetItemResponse.class); + when(multiGetItemResponseTwo.getResponse()).thenReturn(getResponseTwo); + + // Add it to the iterator + @SuppressWarnings("unchecked") + Iterator<MultiGetItemResponse> mockIterator = mock(Iterator.class); + when(mockIterator.hasNext()).thenReturn(true, true, false); + when(mockIterator.next()).thenReturn(multiGetItemResponseOne, multiGetItemResponseTwo); + + // Add them to the response + MultiGetResponse mockResponse = mock(MultiGetResponse.class); + when(mockResponse.iterator()).thenReturn(mockIterator); + + // Actually build the doc + Document actual = emaDao.buildCreateDocument(mockResponse, groups); + + ArrayList<Map<String, Object>> alertList = new ArrayList<>(); + alertList.add(alertOne); + alertList.add(alertTwo); + + Map<String, Object> actualDocument = actual.getDocument(); + assertNotNull(actualDocument.get(Fields.TIMESTAMP.getName())); + assertArrayEquals( + alertList.toArray(), + (Object[]) actualDocument.get(MetaAlertDao.ALERT_FIELD) + ); + assertArrayEquals( + groups.toArray(), + (Object[]) actualDocument.get(MetaAlertDao.GROUPS_FIELD) + ); + + // Don't care about the result, just that it's a UUID. Exception will be thrown if not. + UUID.fromString((String) actualDocument.get(Constants.GUID)); + } + + @Test(expected = InvalidCreateException.class) + public void testCreateMetaAlertEmptyGuids() throws InvalidCreateException, IOException { + ElasticsearchDao esDao = new ElasticsearchDao(); + ElasticsearchMetaAlertDao emaDao = new ElasticsearchMetaAlertDao(); + emaDao.init(esDao); + + MetaAlertCreateRequest createRequest = new MetaAlertCreateRequest(); + emaDao.createMetaAlert(createRequest); + } + + @Test(expected = InvalidCreateException.class) + public void testCreateMetaAlertEmptyGroups() throws InvalidCreateException, IOException { + ElasticsearchDao esDao = new ElasticsearchDao(); + ElasticsearchMetaAlertDao emaDao = new ElasticsearchMetaAlertDao(); + emaDao.init(esDao); + + MetaAlertCreateRequest createRequest = new MetaAlertCreateRequest(); + HashMap<String, String> guidsToGroups = new HashMap<>(); + guidsToGroups.put("don't", "care"); + createRequest.setGuidToIndices(guidsToGroups); + emaDao.createMetaAlert(createRequest); + } + + @Test + public void testCalculateMetaScores() { + List<Map<String, Object>> alertList = new ArrayList<>(); + Map<String, Object> alertMap = new HashMap<>(); + alertMap.put(MetaAlertDao.THREAT_FIELD_DEFAULT, 10.0d); + alertList.add(alertMap); + Map<String, Object> docMap = new HashMap<>(); + docMap.put(MetaAlertDao.ALERT_FIELD, alertList); + + Document doc = new Document(docMap, "guid", MetaAlertDao.METAALERT_TYPE, 0L); + + List<Double> scores = new ArrayList<>(); + scores.add(10.0d); + MetaScores expected = new MetaScores(scores); + + ElasticsearchMetaAlertDao metaAlertDao = new ElasticsearchMetaAlertDao(); + MetaScores actual = metaAlertDao.calculateMetaScores(doc); + assertEquals(expected.getMetaScores(), actual.getMetaScores()); + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/40c93527/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java new file mode 100644 index 0000000..fda62ab --- /dev/null +++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java @@ -0,0 +1,317 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.elasticsearch.integration; + +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; + +import com.fasterxml.jackson.core.JsonProcessingException; +import java.io.File; +import java.io.IOException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; +import org.apache.metron.common.Constants; +import org.apache.metron.common.utils.JSONUtils; +import org.apache.metron.elasticsearch.dao.ElasticsearchDao; +import org.apache.metron.elasticsearch.dao.ElasticsearchMetaAlertDao; +import org.apache.metron.elasticsearch.dao.MetaAlertStatus; +import org.apache.metron.elasticsearch.integration.components.ElasticSearchComponent; +import org.apache.metron.indexing.dao.AccessConfig; +import org.apache.metron.indexing.dao.IndexDao; +import org.apache.metron.indexing.dao.MetaAlertDao; +import org.apache.metron.indexing.dao.update.Document; +import org.apache.metron.indexing.dao.update.ReplaceRequest; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +public class ElasticsearchMetaAlertIntegrationTest { + + private static final int MAX_RETRIES = 10; + private static final int SLEEP_MS = 500; + private static final String SENSOR_NAME = "test"; + private static final String INDEX_DIR = "target/elasticsearch_meta"; + private static final String DATE_FORMAT = "yyyy.MM.dd.HH"; + private static final String INDEX = + SENSOR_NAME + "_index_" + new SimpleDateFormat(DATE_FORMAT).format(new Date()); + private static final String NEW_FIELD = "new-field"; + + private static IndexDao esDao; + private static IndexDao metaDao; + private static ElasticSearchComponent es; + + @BeforeClass + public static void setup() throws Exception { + // setup the client + es = new ElasticSearchComponent.Builder() + .withHttpPort(9211) + .withIndexDir(new File(INDEX_DIR)) + .build(); + es.start(); + + es.createIndexWithMapping(MetaAlertDao.METAALERTS_INDEX, MetaAlertDao.METAALERT_DOC, + buildMetaMappingSource()); + + AccessConfig accessConfig = new AccessConfig(); + Map<String, Object> globalConfig = new HashMap<String, Object>() { + { + put("es.clustername", "metron"); + put("es.port", "9300"); + put("es.ip", "localhost"); + put("es.date.format", DATE_FORMAT); + } + }; + accessConfig.setGlobalConfigSupplier(() -> globalConfig); + + esDao = new ElasticsearchDao(); + esDao.init(accessConfig); + metaDao = new ElasticsearchMetaAlertDao(esDao); + } + + @AfterClass + public static void teardown() { + if (es != null) { + es.stop(); + } + } + + protected static String buildMetaMappingSource() throws IOException { + return jsonBuilder().prettyPrint() + .startObject() + .startObject(MetaAlertDao.METAALERT_DOC) + .startObject("properties") + .startObject("guid") + .field("type", "string") + .field("index", "not_analyzed") + .endObject() + .startObject("score") + .field("type", "integer") + .field("index", "not_analyzed") + .endObject() + .startObject("alert") + .field("type", "nested") + .endObject() + .endObject() + .endObject() + .endObject() + .string(); + } + + + @SuppressWarnings("unchecked") + @Test + public void test() throws Exception { + List<Map<String, Object>> inputData = new ArrayList<>(); + for (int i = 0; i < 2; ++i) { + final String name = "message" + i; + int finalI = i; + inputData.add( + new HashMap<String, Object>() { + { + put("source:type", SENSOR_NAME); + put("name", name); + put(MetaAlertDao.THREAT_FIELD_DEFAULT, finalI); + put("timestamp", System.currentTimeMillis()); + put(Constants.GUID, name); + } + } + ); + } + + elasticsearchAdd(inputData, INDEX, SENSOR_NAME); + + List<Map<String, Object>> metaInputData = new ArrayList<>(); + final String name = "meta_message"; + Map<String, Object>[] alertArray = new Map[1]; + alertArray[0] = inputData.get(0); + metaInputData.add( + new HashMap<String, Object>() { + { + put("source:type", SENSOR_NAME); + put("alert", alertArray); + put(Constants.GUID, name + "_active"); + put(MetaAlertDao.STATUS_FIELD, MetaAlertStatus.ACTIVE.getStatusString()); + } + } + ); + // Add an inactive message + metaInputData.add( + new HashMap<String, Object>() { + { + put("source:type", SENSOR_NAME); + put("alert", alertArray); + put(Constants.GUID, name + "_inactive"); + put(MetaAlertDao.STATUS_FIELD, MetaAlertStatus.INACTIVE.getStatusString()); + } + } + ); + + // We pass MetaAlertDao.METAALERT_TYPE, because the "_doc" gets appended automatically. + elasticsearchAdd(metaInputData, MetaAlertDao.METAALERTS_INDEX, MetaAlertDao.METAALERT_TYPE); + + List<Map<String, Object>> docs = null; + for (int t = 0; t < MAX_RETRIES; ++t, Thread.sleep(SLEEP_MS)) { + docs = es.getAllIndexedDocs(INDEX, SENSOR_NAME + "_doc"); + if (docs.size() >= 10) { + break; + } + } + Assert.assertEquals(2, docs.size()); + { + //modify the first message and add a new field + Map<String, Object> message0 = new HashMap<String, Object>(inputData.get(0)) { + { + put(NEW_FIELD, "metron"); + put(MetaAlertDao.THREAT_FIELD_DEFAULT, "10"); + } + }; + String guid = "" + message0.get(Constants.GUID); + metaDao.replace(new ReplaceRequest() { + { + setReplacement(message0); + setGuid(guid); + setSensorType(SENSOR_NAME); + } + }, Optional.empty()); + + { + //ensure alerts in ES are up-to-date + Document doc = metaDao.getLatest(guid, SENSOR_NAME); + Assert.assertEquals(message0, doc.getDocument()); + long cnt = 0; + for (int t = 0; t < MAX_RETRIES && cnt == 0; ++t, Thread.sleep(SLEEP_MS)) { + docs = es.getAllIndexedDocs(INDEX, SENSOR_NAME + "_doc"); + cnt = docs + .stream() + .filter(d -> { + Object newfield = d.get(NEW_FIELD); + return newfield != null && newfield.equals(message0.get(NEW_FIELD)); + }).count(); + } + if (cnt == 0) { + Assert.fail("Elasticsearch is not updated!"); + } + } + + { + //ensure meta alerts in ES are up-to-date + long cnt = 0; + for (int t = 0; t < MAX_RETRIES && cnt == 0; ++t, Thread.sleep(SLEEP_MS)) { + docs = es.getAllIndexedDocs(MetaAlertDao.METAALERTS_INDEX, MetaAlertDao.METAALERT_DOC); + cnt = docs + .stream() + .filter(d -> { + List<Map<String, Object>> alerts = (List<Map<String, Object>>) d + .get(MetaAlertDao.ALERT_FIELD); + + for (Map<String, Object> alert : alerts) { + Object newField = alert.get(NEW_FIELD); + if (newField != null && newField.equals(message0.get(NEW_FIELD))) { + return true; + } + } + + return false; + }).count(); + } + if (cnt == 0) { + Assert.fail("Elasticsearch metaalerts not updated!"); + } + } + } + //modify the same message and modify the new field + { + Map<String, Object> message0 = new HashMap<String, Object>(inputData.get(0)) { + { + put(NEW_FIELD, "metron2"); + } + }; + String guid = "" + message0.get(Constants.GUID); + metaDao.replace(new ReplaceRequest() { + { + setReplacement(message0); + setGuid(guid); + setSensorType(SENSOR_NAME); + } + }, Optional.empty()); + + Document doc = metaDao.getLatest(guid, SENSOR_NAME); + Assert.assertEquals(message0, doc.getDocument()); + { + //ensure ES is up-to-date + long cnt = 0; + for (int t = 0; t < MAX_RETRIES && cnt == 0; ++t, Thread.sleep(SLEEP_MS)) { + docs = es.getAllIndexedDocs(INDEX, SENSOR_NAME + "_doc"); + cnt = docs + .stream() + .filter(d -> message0.get(NEW_FIELD).equals(d.get(NEW_FIELD))) + .count(); + } + Assert.assertNotEquals("Elasticsearch is not updated!", cnt, 0); + if (cnt == 0) { + Assert.fail("Elasticsearch is not updated!"); + } + } + { + //ensure meta alerts in ES are up-to-date + long cnt = 0; + for (int t = 0; t < MAX_RETRIES && cnt == 0; ++t, Thread.sleep(SLEEP_MS)) { + docs = es.getAllIndexedDocs(MetaAlertDao.METAALERTS_INDEX, MetaAlertDao.METAALERT_DOC); + cnt = docs + .stream() + .filter(d -> { + List<Map<String, Object>> alerts = (List<Map<String, Object>>) d + .get(MetaAlertDao.ALERT_FIELD); + + for (Map<String, Object> alert : alerts) { + Object newField = alert.get(NEW_FIELD); + if (newField != null && newField.equals(message0.get(NEW_FIELD))) { + return true; + } + } + + return false; + }).count(); + } + if (cnt == 0) { + Assert.fail("Elasticsearch metaalerts not updated!"); + } + } + } + } + + protected void elasticsearchAdd(List<Map<String, Object>> inputData, String index, String docType) + throws IOException { + es.add(index, docType, inputData.stream().map(m -> { + try { + return JSONUtils.INSTANCE.toJSON(m, true); + } catch (JsonProcessingException e) { + throw new IllegalStateException(e.getMessage(), e); + } + } + ).collect(Collectors.toList()) + ); + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/40c93527/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java index 5de9fd2..adb69ee 100644 --- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java +++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java @@ -20,9 +20,11 @@ package org.apache.metron.elasticsearch.integration; import org.adrianwalker.multilinestring.Multiline; import org.apache.metron.elasticsearch.dao.ElasticsearchDao; +import org.apache.metron.elasticsearch.dao.ElasticsearchMetaAlertDao; import org.apache.metron.elasticsearch.integration.components.ElasticSearchComponent; import org.apache.metron.indexing.dao.AccessConfig; import org.apache.metron.indexing.dao.IndexDao; +import org.apache.metron.indexing.dao.MetaAlertDao; import org.apache.metron.indexing.dao.SearchIntegrationTest; import org.apache.metron.integration.InMemoryComponent; import org.elasticsearch.action.bulk.BulkRequestBuilder; @@ -87,8 +89,8 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest { @Override protected IndexDao createDao() throws Exception { - IndexDao ret = new ElasticsearchDao(); - ret.init( + IndexDao elasticsearchDao = new ElasticsearchDao(); + elasticsearchDao.init( new AccessConfig() {{ setMaxSearchResults(100); setMaxSearchGroups(100); @@ -102,7 +104,9 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest { ); }} ); - return ret; + MetaAlertDao ret = new ElasticsearchMetaAlertDao(); + ret.init(elasticsearchDao); + return elasticsearchDao; } @Override @@ -140,6 +144,14 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest { indexRequestBuilder = indexRequestBuilder.setTimestamp(jsonObject.get("timestamp").toString()); bulkRequest.add(indexRequestBuilder); } + JSONArray metaAlertArray = (JSONArray) new JSONParser().parse(metaAlertData); + for(Object o: metaAlertArray) { + JSONObject jsonObject = (JSONObject) o; + IndexRequestBuilder indexRequestBuilder = es.getClient().prepareIndex("metaalerts", "metaalert_doc"); + indexRequestBuilder = indexRequestBuilder.setSource(jsonObject.toJSONString()); +// indexRequestBuilder = indexRequestBuilder.setTimestamp(jsonObject.get("timestamp").toString()); + bulkRequest.add(indexRequestBuilder); + } BulkResponse bulkResponse = bulkRequest.execute().actionGet(); if (bulkResponse.hasFailures()) { throw new RuntimeException("Failed to index test data"); http://git-wip-us.apache.org/repos/asf/metron/blob/40c93527/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchUpdateIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchUpdateIntegrationTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchUpdateIntegrationTest.java index 9a1d7a7..fddf056 100644 --- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchUpdateIntegrationTest.java +++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchUpdateIntegrationTest.java @@ -33,6 +33,8 @@ import org.apache.metron.hbase.mock.MockHBaseTableProvider; import org.apache.metron.indexing.dao.*; import org.apache.metron.indexing.dao.update.Document; import org.apache.metron.indexing.dao.update.ReplaceRequest; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.index.query.QueryBuilders; import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; @@ -144,6 +146,7 @@ public class ElasticsearchUpdateIntegrationTest { setGuid(guid); setSensorType(SENSOR_NAME); }}, Optional.empty()); + Assert.assertEquals(1, table.size()); Document doc = dao.getLatest(guid, SENSOR_NAME); Assert.assertEquals(message0, doc.getDocument()); http://git-wip-us.apache.org/repos/asf/metron/blob/40c93527/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/components/ElasticSearchComponent.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/components/ElasticSearchComponent.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/components/ElasticSearchComponent.java index 7facff5..171b6ab 100644 --- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/components/ElasticSearchComponent.java +++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/components/ElasticSearchComponent.java @@ -19,6 +19,7 @@ package org.apache.metron.elasticsearch.integration.components; import com.fasterxml.jackson.core.type.TypeReference; import org.apache.commons.io.FileUtils; +import org.apache.metron.common.Constants; import org.apache.metron.common.utils.JSONUtils; import org.apache.metron.integration.InMemoryComponent; import org.apache.metron.integration.UnableToStartException; @@ -26,6 +27,8 @@ import org.elasticsearch.ElasticsearchTimeoutException; import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction; import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; +import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; +import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse; import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.index.IndexRequestBuilder; @@ -112,6 +115,7 @@ public class ElasticSearchComponent implements InMemoryComponent { indexRequestBuilder = indexRequestBuilder.setSource(doc); Map<String, Object> esDoc = JSONUtils.INSTANCE.load(doc, new TypeReference<Map<String, Object>>() { }); + indexRequestBuilder.setId((String) esDoc.get(Constants.GUID)); Object ts = esDoc.get("timestamp"); if(ts != null) { indexRequestBuilder = indexRequestBuilder.setTimestamp(ts.toString()); @@ -126,6 +130,17 @@ public class ElasticSearchComponent implements InMemoryComponent { return response; } + public void createIndexWithMapping(String indexName, String mappingType, String mappingSource) + throws IOException { + CreateIndexResponse cir = client.admin().indices().prepareCreate(indexName) + .addMapping(mappingType, mappingSource) + .get(); + + if (!cir.isAcknowledged()) { + throw new IOException("Create index was not acknowledged"); + } + } + @Override public void start() throws UnableToStartException { File logDir= new File(indexDir, "/logs"); http://git-wip-us.apache.org/repos/asf/metron/blob/40c93527/metron-platform/metron-indexing/README.md ---------------------------------------------------------------------- diff --git a/metron-platform/metron-indexing/README.md b/metron-platform/metron-indexing/README.md index aea670c..e65152c 100644 --- a/metron-platform/metron-indexing/README.md +++ b/metron-platform/metron-indexing/README.md @@ -146,6 +146,23 @@ in parallel. This enables a flexible strategy for specifying your backing store For instance, currently the REST API supports the update functionality and may be configured with a list of IndexDao implementations to use to support the updates. +### The `MetaAlertDao` + +The goal of meta alerts is to be able to group together a set of alerts while being able to transparently perform actions +like searches, as if meta alerts were normal alerts. `org.apache.metron.indexing.dao.MetaAlertDao` extends `IndexDao` and +enables a couple extra features: creation of a meta alert and the ability to get all meta alerts associated with an alert. + +The implementation of this is to denormalize the relationship between alerts and meta alerts, and store alerts as a nested field within a meta alert. +The use of nested fields is to avoid the limitations of parent-child relationships (one-to-many) and merely linking by IDs +(which causes issues with pagination as a result of being unable to join indices). + +The search functionality of `IndexDao` is wrapped by the `MetaAlertDao` in order to provide both regular and meta alerts side-by-side with sorting. +The updating capabilities are similarly wrapped, in order to ensure updates are carried through both the alerts and associated meta alerts. +Both of these functions are handled under the hood. + +In addition, an API endpoint is added for the meta alert specific features of creation and going from meta alert to alert. +The denormalization handles the case of going from meta alert to alert automatically. + # Notes on Performance Tuning Default installed Metron is untuned for production deployment. By far http://git-wip-us.apache.org/repos/asf/metron/blob/40c93527/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MetaAlertDao.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MetaAlertDao.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MetaAlertDao.java new file mode 100644 index 0000000..4e0851b --- /dev/null +++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MetaAlertDao.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.indexing.dao; + +import java.io.IOException; +import org.apache.metron.indexing.dao.metaalert.MetaAlertCreateRequest; +import org.apache.metron.indexing.dao.metaalert.MetaAlertCreateResponse; +import org.apache.metron.indexing.dao.search.InvalidCreateException; +import org.apache.metron.indexing.dao.search.InvalidSearchException; +import org.apache.metron.indexing.dao.search.SearchResponse; + +public interface MetaAlertDao extends IndexDao { + + String METAALERTS_INDEX = "metaalerts"; + String METAALERT_TYPE = "metaalert"; + String METAALERT_DOC = METAALERT_TYPE + "_doc"; + String THREAT_FIELD_DEFAULT = "threat:triage:score"; + String THREAT_SORT_DEFAULT = "sum"; + String ALERT_FIELD = "alert"; + String STATUS_FIELD = "status"; + String GROUPS_FIELD = "groups"; + + /** + * Given an alert GUID, retrieve all associated meta alerts. + * @param guid The alert GUID to be searched for + * @return All meta alerts with a child alert having the GUID + * @throws InvalidSearchException If a problem occurs with the search + */ + SearchResponse getAllMetaAlertsForAlert(String guid) throws InvalidSearchException; + + /** + * Create a meta alert. + * @param request The parameters for creating the new meta alert + * @return A response indicating success or failure + * @throws InvalidCreateException If a malformed create request is provided + * @throws IOException If a problem occurs during communication + */ + MetaAlertCreateResponse createMetaAlert(MetaAlertCreateRequest request) + throws InvalidCreateException, IOException; + + /** + * Initializes a Meta Alert DAO with default "sum" meta alert threat sorting. + * @param indexDao The DAO to wrap for our queries. + */ + default void init(IndexDao indexDao) { + init(indexDao, null); + } + + /** + * Initializes a Meta Alert DAO. + * @param indexDao The DAO to wrap for our queries + * @param threatSort The aggregation to use as the threat field. E.g. "sum", "median", etc. + * null is "sum" + */ + void init(IndexDao indexDao, String threatSort); +} http://git-wip-us.apache.org/repos/asf/metron/blob/40c93527/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MultiIndexDao.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MultiIndexDao.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MultiIndexDao.java index 61c6231..2df06fc 100644 --- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MultiIndexDao.java +++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MultiIndexDao.java @@ -169,4 +169,8 @@ public class MultiIndexDao implements IndexDao { } return ret; } + + public List<IndexDao> getIndices() { + return indices; + } } http://git-wip-us.apache.org/repos/asf/metron/blob/40c93527/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertCreateRequest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertCreateRequest.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertCreateRequest.java new file mode 100644 index 0000000..388527a --- /dev/null +++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertCreateRequest.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.indexing.dao.metaalert; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class MetaAlertCreateRequest { + // A map from the alert GUID to the Document index + private Map<String, String> guidToIndices; + private List<String> groups; + + public MetaAlertCreateRequest() { + this.guidToIndices = new HashMap<>(); + this.groups = new ArrayList<>(); + } + + public Map<String, String> getGuidToIndices() { + return guidToIndices; + } + + public void setGuidToIndices(Map<String, String> guidToIndices) { + this.guidToIndices = guidToIndices; + } + + public List<String> getGroups() { + return groups; + } + + public void setGroups(List<String> groups) { + this.groups = groups; + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/40c93527/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertCreateResponse.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertCreateResponse.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertCreateResponse.java new file mode 100644 index 0000000..e84286e --- /dev/null +++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertCreateResponse.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.indexing.dao.metaalert; + +public class MetaAlertCreateResponse { + private boolean created; + + public boolean isCreated() { + return created; + } + + public void setCreated(boolean created) { + this.created = created; + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/40c93527/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaScores.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaScores.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaScores.java new file mode 100644 index 0000000..632cfd2 --- /dev/null +++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaScores.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.indexing.dao.metaalert; + +import java.util.DoubleSummaryStatistics; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.commons.math3.stat.descriptive.rank.Median; + +public class MetaScores { + + protected Map<String, Object> metaScores = new HashMap<>(); + + public MetaScores(List<Double> scores) { + // A meta alert could be entirely alerts with no values. + DoubleSummaryStatistics stats = scores + .stream() + .mapToDouble(a -> a) + .summaryStatistics(); + metaScores.put("max", stats.getMax()); + metaScores.put("min", stats.getMin()); + metaScores.put("average", stats.getAverage()); + metaScores.put("count", stats.getCount()); + metaScores.put("sum", stats.getSum()); + + // median isn't in the stats summary + double[] arr = scores + .stream() + .mapToDouble(d -> d) + .toArray(); + metaScores.put("median", new Median().evaluate(arr)); + } + + public Map<String, Object> getMetaScores() { + return metaScores; + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/40c93527/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/FieldType.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/FieldType.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/FieldType.java index 5848cb3..1f00cf5 100644 --- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/FieldType.java +++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/FieldType.java @@ -36,6 +36,8 @@ public enum FieldType { DOUBLE("double"), @JsonProperty("boolean") BOOLEAN("boolean"), + @JsonProperty("nested") + NESTED("nested"), @JsonProperty("other") OTHER("other"); http://git-wip-us.apache.org/repos/asf/metron/blob/40c93527/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/InvalidCreateException.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/InvalidCreateException.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/InvalidCreateException.java new file mode 100644 index 0000000..be32cee --- /dev/null +++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/InvalidCreateException.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.indexing.dao.search; + +public class InvalidCreateException extends Exception { + public InvalidCreateException(String message) { + super(message); + } + public InvalidCreateException(String message, Throwable t) { + super(message, t); + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/40c93527/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SearchResult.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SearchResult.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SearchResult.java index 9c00bea..da4fac1 100644 --- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SearchResult.java +++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SearchResult.java @@ -73,4 +73,14 @@ public class SearchResult { public void setScore(float score) { this.score = score; } + + @Override + public String toString() { + return "SearchResult{" + + "id='" + id + '\'' + + ", source=" + source + + ", score=" + score + + ", index='" + index + '\'' + + '}'; + } } http://git-wip-us.apache.org/repos/asf/metron/blob/40c93527/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/Document.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/Document.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/Document.java index 85c079f..461ce3e 100644 --- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/Document.java +++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/Document.java @@ -18,14 +18,11 @@ package org.apache.metron.indexing.dao.update; -import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.JsonNode; import org.apache.metron.common.utils.JSONUtils; import java.io.IOException; import java.util.Map; -import java.util.Optional; public class Document { Long timestamp; @@ -85,4 +82,14 @@ public class Document { public void setGuid(String guid) { this.guid = guid; } + + @Override + public String toString() { + return "Document{" + + "timestamp=" + timestamp + + ", document=" + document + + ", guid='" + guid + '\'' + + ", sensorType='" + sensorType + '\'' + + '}'; + } } http://git-wip-us.apache.org/repos/asf/metron/blob/40c93527/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryDao.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryDao.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryDao.java index 6e48b58..c83f6aa 100644 --- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryDao.java +++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryDao.java @@ -30,6 +30,7 @@ import java.io.IOException; import java.util.*; public class InMemoryDao implements IndexDao { + // Map from index to list of documents as JSON strings public static Map<String, List<String>> BACKING_STORE = new HashMap<>(); public static Map<String, Map<String, FieldType>> COLUMN_METADATA; private AccessConfig config; @@ -123,6 +124,9 @@ public class InMemoryDao implements IndexDao { } private static boolean isMatch(String query, Map<String, Object> doc) { + if (query == null) { + return false; + } if(query.equals("*")) { return true; } @@ -130,12 +134,36 @@ public class InMemoryDao implements IndexDao { Iterable<String> splits = Splitter.on(":").split(query.trim()); String field = Iterables.getFirst(splits, ""); String val = Iterables.getLast(splits, ""); - Object o = doc.get(field); - if(o == null) { + + // Immediately quit if there's no value ot find + if (val == null) { return false; } - else { - return o.equals(val); + + // Check if we're looking into a nested field. The '|' is arbitrarily chosen. + String nestingField = null; + if (field.contains("|")) { + Iterable<String> fieldSplits = Splitter.on('|').split(field); + nestingField = Iterables.getFirst(fieldSplits, null); + field = Iterables.getLast(fieldSplits, null); + } + if (nestingField == null) { + // Just grab directly + Object o = doc.get(field); + return val.equals(o); + } else { + // We need to look into a nested field for the value + @SuppressWarnings("unchecked") + List<Map<String, Object>> nestedList = (List<Map<String, Object>>) doc.get(nestingField); + if (nestedList == null) { + return false; + } else { + for (Map<String, Object> nestedEntry : nestedList) { + if (val.equals(nestedEntry.get(field))) { + return true; + } + } + } } } return false; @@ -185,7 +213,7 @@ public class InMemoryDao implements IndexDao { } } } - + public Map<String, Map<String, FieldType>> getColumnMetadata(List<String> indices) throws IOException { Map<String, Map<String, FieldType>> columnMetadata = new HashMap<>(); for(String index: indices) { http://git-wip-us.apache.org/repos/asf/metron/blob/40c93527/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryMetaAlertDao.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryMetaAlertDao.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryMetaAlertDao.java new file mode 100644 index 0000000..8807bbc --- /dev/null +++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryMetaAlertDao.java @@ -0,0 +1,198 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.indexing.dao; + +import com.google.common.collect.ImmutableList; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import org.adrianwalker.multilinestring.Multiline; +import org.apache.metron.common.Constants; +import org.apache.metron.common.utils.JSONUtils; +import org.apache.metron.indexing.dao.metaalert.MetaAlertCreateRequest; +import org.apache.metron.indexing.dao.metaalert.MetaAlertCreateResponse; +import org.apache.metron.indexing.dao.metaalert.MetaScores; +import org.apache.metron.indexing.dao.search.FieldType; +import org.apache.metron.indexing.dao.search.GetRequest; +import org.apache.metron.indexing.dao.search.GroupRequest; +import org.apache.metron.indexing.dao.search.GroupResponse; +import org.apache.metron.indexing.dao.search.InvalidCreateException; +import org.apache.metron.indexing.dao.search.InvalidSearchException; +import org.apache.metron.indexing.dao.search.SearchRequest; +import org.apache.metron.indexing.dao.search.SearchResponse; +import org.apache.metron.indexing.dao.search.SearchResult; +import org.apache.metron.indexing.dao.update.Document; +import org.apache.metron.indexing.dao.update.OriginalNotFoundException; +import org.apache.metron.indexing.dao.update.PatchRequest; +import org.apache.metron.indexing.dao.update.ReplaceRequest; +import org.json.simple.JSONArray; +import org.json.simple.JSONObject; + +public class InMemoryMetaAlertDao implements MetaAlertDao { + + private IndexDao indexDao; + + /** + * { + * "indices": ["metaalerts"], + * "query": "alert|guid:${GUID}", + * "from": 0, + * "size": 10, + * "sort": [ + * { + * "field": "guid", + * "sortOrder": "desc" + * } + * ] + * } + */ + @Multiline + public static String metaAlertsForAlertQuery; + + @Override + public SearchResponse search(SearchRequest searchRequest) throws InvalidSearchException { + return indexDao.search(searchRequest); + } + + @Override + public GroupResponse group(GroupRequest groupRequest) throws InvalidSearchException { + return indexDao.group(groupRequest); + } + + @Override + public void init(AccessConfig config) { + // Do nothing + } + + @Override + public void init(IndexDao indexDao, String threatSort) { + this.indexDao = indexDao; + // Ignore threatSort for test. + } + + @Override + public Document getLatest(String guid, String sensorType) throws IOException { + return indexDao.getLatest(guid, sensorType); + } + + @Override + public void update(Document update, Optional<String> index) throws IOException { + indexDao.update(update, index); + } + + @Override + public Map<String, Map<String, FieldType>> getColumnMetadata(List<String> indices) + throws IOException { + return indexDao.getColumnMetadata(indices); + } + + @Override + public Map<String, FieldType> getCommonColumnMetadata(List<String> indices) throws IOException { + return indexDao.getCommonColumnMetadata(indices); + } + + @Override + public Optional<Map<String, Object>> getLatestResult(GetRequest request) throws IOException { + return indexDao.getLatestResult(request); + } + + @Override + public void patch(PatchRequest request, Optional<Long> timestamp) + throws OriginalNotFoundException, IOException { + indexDao.patch(request, timestamp); + } + + @Override + public void replace(ReplaceRequest request, Optional<Long> timestamp) throws IOException { + indexDao.replace(request, timestamp); + } + + @Override + public SearchResponse getAllMetaAlertsForAlert(String guid) throws InvalidSearchException { + SearchRequest request; + try { + String replacedQuery = metaAlertsForAlertQuery.replace("${GUID}", guid); + request = JSONUtils.INSTANCE.load(replacedQuery, SearchRequest.class); + } catch (IOException e) { + throw new InvalidSearchException("Unable to process query:", e); + } + return search(request); + } + + @SuppressWarnings("unchecked") + @Override + public MetaAlertCreateResponse createMetaAlert(MetaAlertCreateRequest request) + throws InvalidCreateException, IOException { + if (request.getGuidToIndices().isEmpty()) { + MetaAlertCreateResponse response = new MetaAlertCreateResponse(); + response.setCreated(false); + return response; + } + // Build meta alert json. Give it a reasonable GUID + JSONObject metaAlert = new JSONObject(); + metaAlert.put(Constants.GUID, + "meta_" + (InMemoryDao.BACKING_STORE.get(MetaAlertDao.METAALERTS_INDEX).size() + 1)); + + JSONArray groupsArray = new JSONArray(); + groupsArray.addAll(request.getGroups()); + metaAlert.put(MetaAlertDao.GROUPS_FIELD, groupsArray); + + // Retrieve the alert for each guid + // For the purpose of testing, we're just using guids for the alerts field and grabbing the scores. + JSONArray alertArray = new JSONArray(); + List<Double> threatScores = new ArrayList<>(); + for (Map.Entry<String, String> entry : request.getGuidToIndices().entrySet()) { + SearchRequest searchRequest = new SearchRequest(); + searchRequest.setIndices(ImmutableList.of(entry.getValue())); + searchRequest.setQuery("guid:" + entry.getKey()); + try { + SearchResponse searchResponse = search(searchRequest); + List<SearchResult> searchResults = searchResponse.getResults(); + if (searchResults.size() > 1) { + throw new InvalidCreateException( + "Found more than one result for: " + entry.getKey() + ". Values: " + searchResults + ); + } + + if (searchResults.size() == 1) { + SearchResult result = searchResults.get(0); + alertArray.add(result.getSource()); + Double threatScore = Double + .parseDouble(result.getSource().getOrDefault(THREAT_FIELD_DEFAULT, "0").toString()); + + threatScores.add(threatScore); + } + } catch (InvalidSearchException e) { + throw new InvalidCreateException("Unable to find guid: " + entry.getKey(), e); + } + } + + metaAlert.put(MetaAlertDao.ALERT_FIELD, alertArray); + metaAlert.putAll(new MetaScores(threatScores).getMetaScores()); + + // Add the alert to the store, but make sure not to overwrite existing results + InMemoryDao.BACKING_STORE.get(MetaAlertDao.METAALERTS_INDEX).add(metaAlert.toJSONString()); + + MetaAlertCreateResponse createResponse = new MetaAlertCreateResponse(); + createResponse.setCreated(true); + return createResponse; + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/40c93527/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java index 0db8e37..26d1a75 100644 --- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java +++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java @@ -43,11 +43,11 @@ import java.util.Map; public abstract class SearchIntegrationTest { /** * [ - * {"source:type": "bro", "ip_src_addr":"192.168.1.1", "ip_src_port": 8010, "long_field": 10000, "timestamp":1, "latitude": 48.5839, "score": 10.0, "is_alert":true, "location_point": "48.5839,7.7455", "bro_field": "bro data 1", "duplicate_name_field": "data 1"}, - * {"source:type": "bro", "ip_src_addr":"192.168.1.2", "ip_src_port": 8009, "long_field": 20000, "timestamp":2, "latitude": 48.0001, "score": 50.0, "is_alert":false, "location_point": "48.5839,7.7455", "bro_field": "bro data 2", "duplicate_name_field": "data 2"}, - * {"source:type": "bro", "ip_src_addr":"192.168.1.3", "ip_src_port": 8008, "long_field": 10000, "timestamp":3, "latitude": 48.5839, "score": 20.0, "is_alert":true, "location_point": "50.0,7.7455", "bro_field": "bro data 3", "duplicate_name_field": "data 3"}, - * {"source:type": "bro", "ip_src_addr":"192.168.1.4", "ip_src_port": 8007, "long_field": 10000, "timestamp":4, "latitude": 48.5839, "score": 10.0, "is_alert":true, "location_point": "48.5839,7.7455", "bro_field": "bro data 4", "duplicate_name_field": "data 4"}, - * {"source:type": "bro", "ip_src_addr":"192.168.1.5", "ip_src_port": 8006, "long_field": 10000, "timestamp":5, "latitude": 48.5839, "score": 98.0, "is_alert":true, "location_point": "48.5839,7.7455", "bro_field": "bro data 5", "duplicate_name_field": "data 5"} + * {"source:type": "bro", "ip_src_addr":"192.168.1.1", "ip_src_port": 8010, "long_field": 10000, "timestamp":1, "latitude": 48.5839, "score": 10.0, "is_alert":true, "location_point": "48.5839,7.7455", "bro_field": "bro data 1", "duplicate_name_field": "data 1", "guid":"bro_1"}, + * {"source:type": "bro", "ip_src_addr":"192.168.1.2", "ip_src_port": 8009, "long_field": 20000, "timestamp":2, "latitude": 48.0001, "score": 50.0, "is_alert":false, "location_point": "48.5839,7.7455", "bro_field": "bro data 2", "duplicate_name_field": "data 2", "guid":"bro_2"}, + * {"source:type": "bro", "ip_src_addr":"192.168.1.3", "ip_src_port": 8008, "long_field": 10000, "timestamp":3, "latitude": 48.5839, "score": 20.0, "is_alert":true, "location_point": "50.0,7.7455", "bro_field": "bro data 3", "duplicate_name_field": "data 3", "guid":"bro_3"}, + * {"source:type": "bro", "ip_src_addr":"192.168.1.4", "ip_src_port": 8007, "long_field": 10000, "timestamp":4, "latitude": 48.5839, "score": 10.0, "is_alert":true, "location_point": "48.5839,7.7455", "bro_field": "bro data 4", "duplicate_name_field": "data 4", "guid":"bro_4"}, + * {"source:type": "bro", "ip_src_addr":"192.168.1.5", "ip_src_port": 8006, "long_field": 10000, "timestamp":5, "latitude": 48.5839, "score": 98.0, "is_alert":true, "location_point": "48.5839,7.7455", "bro_field": "bro data 5", "duplicate_name_field": "data 5", "guid":"bro_5"} * ] */ @Multiline @@ -55,17 +55,26 @@ public abstract class SearchIntegrationTest { /** * [ - * {"source:type": "snort", "ip_src_addr":"192.168.1.6", "ip_src_port": 8005, "long_field": 10000, "timestamp":6, "latitude": 48.5839, "score": 50.0, "is_alert":false, "location_point": "50.0,7.7455", "snort_field": 10, "duplicate_name_field": 1}, - * {"source:type": "snort", "ip_src_addr":"192.168.1.1", "ip_src_port": 8004, "long_field": 10000, "timestamp":7, "latitude": 48.5839, "score": 10.0, "is_alert":true, "location_point": "48.5839,7.7455", "snort_field": 20, "duplicate_name_field": 2}, - * {"source:type": "snort", "ip_src_addr":"192.168.1.7", "ip_src_port": 8003, "long_field": 10000, "timestamp":8, "latitude": 48.5839, "score": 20.0, "is_alert":false, "location_point": "48.5839,7.7455", "snort_field": 30, "duplicate_name_field": 3}, - * {"source:type": "snort", "ip_src_addr":"192.168.1.1", "ip_src_port": 8002, "long_field": 20000, "timestamp":9, "latitude": 48.0001, "score": 50.0, "is_alert":true, "location_point": "48.5839,7.7455", "snort_field": 40, "duplicate_name_field": 4}, - * {"source:type": "snort", "ip_src_addr":"192.168.1.8", "ip_src_port": 8001, "long_field": 10000, "timestamp":10, "latitude": 48.5839, "score": 10.0, "is_alert":false, "location_point": "48.5839,7.7455", "snort_field": 50, "duplicate_name_field": 5} + * {"source:type": "snort", "ip_src_addr":"192.168.1.6", "ip_src_port": 8005, "long_field": 10000, "timestamp":6, "latitude": 48.5839, "score": 50.0, "is_alert":false, "location_point": "50.0,7.7455", "snort_field": 10, "duplicate_name_field": 1, "guid":"snort_1"}, + * {"source:type": "snort", "ip_src_addr":"192.168.1.1", "ip_src_port": 8004, "long_field": 10000, "timestamp":7, "latitude": 48.5839, "score": 10.0, "is_alert":true, "location_point": "48.5839,7.7455", "snort_field": 20, "duplicate_name_field": 2, "guid":"snort_2"}, + * {"source:type": "snort", "ip_src_addr":"192.168.1.7", "ip_src_port": 8003, "long_field": 10000, "timestamp":8, "latitude": 48.5839, "score": 20.0, "is_alert":false, "location_point": "48.5839,7.7455", "snort_field": 30, "duplicate_name_field": 3, "guid":"snort_3"}, + * {"source:type": "snort", "ip_src_addr":"192.168.1.1", "ip_src_port": 8002, "long_field": 20000, "timestamp":9, "latitude": 48.0001, "score": 50.0, "is_alert":true, "location_point": "48.5839,7.7455", "snort_field": 40, "duplicate_name_field": 4, "guid":"snort_4"}, + * {"source:type": "snort", "ip_src_addr":"192.168.1.8", "ip_src_port": 8001, "long_field": 10000, "timestamp":10, "latitude": 48.5839, "score": 10.0, "is_alert":false, "location_point": "48.5839,7.7455", "snort_field": 50, "duplicate_name_field": 5, "guid":"snort_5"} * ] */ @Multiline public static String snortData; /** + * [ + *{"guid":"meta_1","alert":[{"guid":"bro_1"}],"average":"5.0","min":"5.0","median":"5.0","max":"5.0","count":"1.0","sum":"5.0"}, + *{"guid":"meta_2","alert":[{"guid":"bro_1"},{"guid":"bro_2"},{"guid":"snort_1"}],"average":"5.0","min":"0.0","median":"5.0","max":"10.0","count":"3.0","sum":"15.0"} + * ] + */ + @Multiline + public static String metaAlertData; + + /** * { * "indices": ["bro", "snort"], * "query": "*", @@ -258,6 +267,25 @@ public abstract class SearchIntegrationTest { /** * { + * "fields": ["guid"], + * "indices": ["metaalerts"], + * "query": "*", + * "from": 0, + * "size": 10, + * "sort": [ + * { + * "field": "guid", + * "sortOrder": "asc" + * } + * ] + * } + * } + */ + @Multiline + public static String metaAlertsFieldQuery; + + /** + * { * "groups": [ * { * "field":"is_alert" @@ -497,7 +525,7 @@ public abstract class SearchIntegrationTest { Map<String, Map<String, FieldType>> fieldTypes = dao.getColumnMetadata(Arrays.asList("bro", "snort")); Assert.assertEquals(2, fieldTypes.size()); Map<String, FieldType> broTypes = fieldTypes.get("bro"); - Assert.assertEquals(11, broTypes.size()); + Assert.assertEquals(12, broTypes.size()); Assert.assertEquals(FieldType.STRING, broTypes.get("source:type")); Assert.assertEquals(FieldType.IP, broTypes.get("ip_src_addr")); Assert.assertEquals(FieldType.INTEGER, broTypes.get("ip_src_port")); @@ -509,8 +537,9 @@ public abstract class SearchIntegrationTest { Assert.assertEquals(FieldType.OTHER, broTypes.get("location_point")); Assert.assertEquals(FieldType.STRING, broTypes.get("bro_field")); Assert.assertEquals(FieldType.STRING, broTypes.get("duplicate_name_field")); + Assert.assertEquals(FieldType.STRING, broTypes.get("guid")); Map<String, FieldType> snortTypes = fieldTypes.get("snort"); - Assert.assertEquals(11, snortTypes.size()); + Assert.assertEquals(12, snortTypes.size()); Assert.assertEquals(FieldType.STRING, snortTypes.get("source:type")); Assert.assertEquals(FieldType.IP, snortTypes.get("ip_src_addr")); Assert.assertEquals(FieldType.INTEGER, snortTypes.get("ip_src_port")); @@ -522,13 +551,14 @@ public abstract class SearchIntegrationTest { Assert.assertEquals(FieldType.OTHER, snortTypes.get("location_point")); Assert.assertEquals(FieldType.INTEGER, snortTypes.get("snort_field")); Assert.assertEquals(FieldType.INTEGER, snortTypes.get("duplicate_name_field")); + Assert.assertEquals(FieldType.STRING, broTypes.get("guid")); } // getColumnMetadata with only bro { Map<String, Map<String, FieldType>> fieldTypes = dao.getColumnMetadata(Collections.singletonList("bro")); Assert.assertEquals(1, fieldTypes.size()); Map<String, FieldType> broTypes = fieldTypes.get("bro"); - Assert.assertEquals(11, broTypes.size()); + Assert.assertEquals(12, broTypes.size()); Assert.assertEquals(FieldType.STRING, broTypes.get("bro_field")); } // getColumnMetadata with only snort @@ -536,14 +566,14 @@ public abstract class SearchIntegrationTest { Map<String, Map<String, FieldType>> fieldTypes = dao.getColumnMetadata(Collections.singletonList("snort")); Assert.assertEquals(1, fieldTypes.size()); Map<String, FieldType> snortTypes = fieldTypes.get("snort"); - Assert.assertEquals(11, snortTypes.size()); + Assert.assertEquals(12, snortTypes.size()); Assert.assertEquals(FieldType.INTEGER, snortTypes.get("snort_field")); } // getCommonColumnMetadata with multiple Indices { Map<String, FieldType> fieldTypes = dao.getCommonColumnMetadata(Arrays.asList("bro", "snort")); // Should only return fields in both - Assert.assertEquals(9, fieldTypes.size()); + Assert.assertEquals(10, fieldTypes.size()); Assert.assertEquals(FieldType.STRING, fieldTypes.get("source:type")); Assert.assertEquals(FieldType.IP, fieldTypes.get("ip_src_addr")); Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("ip_src_port")); @@ -553,18 +583,19 @@ public abstract class SearchIntegrationTest { Assert.assertEquals(FieldType.DOUBLE, fieldTypes.get("score")); Assert.assertEquals(FieldType.BOOLEAN, fieldTypes.get("is_alert")); Assert.assertEquals(FieldType.OTHER, fieldTypes.get("location_point")); + Assert.assertEquals(FieldType.STRING, fieldTypes.get("guid")); } // getCommonColumnMetadata with only bro { Map<String, FieldType> fieldTypes = dao.getCommonColumnMetadata(Collections.singletonList("bro")); - Assert.assertEquals(11, fieldTypes.size()); + Assert.assertEquals(12, fieldTypes.size()); Assert.assertEquals(FieldType.STRING, fieldTypes.get("bro_field")); Assert.assertEquals(FieldType.STRING, fieldTypes.get("duplicate_name_field")); } // getCommonColumnMetadata with only snort { Map<String, FieldType> fieldTypes = dao.getCommonColumnMetadata(Collections.singletonList("snort")); - Assert.assertEquals(11, fieldTypes.size()); + Assert.assertEquals(12, fieldTypes.size()); Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("snort_field")); Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("duplicate_name_field")); } @@ -585,6 +616,18 @@ public abstract class SearchIntegrationTest { Assert.assertNotNull(source.get("ip_src_addr")); } } + //Meta Alerts Fields query + { + SearchRequest request = JSONUtils.INSTANCE.load(metaAlertsFieldQuery, SearchRequest.class); + SearchResponse response = dao.search(request); + Assert.assertEquals(2, response.getTotal()); + List<SearchResult> results = response.getResults(); + for (int i = 0;i < 2;++i) { + Map<String, Object> source = results.get(i).getSource(); + Assert.assertEquals(1, source.size()); + Assert.assertEquals(source.get("guid"), "meta_" + (i + 1)); + } + } //No results fields query { SearchRequest request = JSONUtils.INSTANCE.load(noResultsFieldsQuery, SearchRequest.class); http://git-wip-us.apache.org/repos/asf/metron/blob/40c93527/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/BasicStellarTest.java ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/BasicStellarTest.java b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/BasicStellarTest.java index af86902..2b20feb 100644 --- a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/BasicStellarTest.java +++ b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/BasicStellarTest.java @@ -548,6 +548,11 @@ public class BasicStellarTest { } @Test + public void testToStringNull() { + Assert.assertEquals("null", run("TO_STRING(\"null\")", ImmutableMap.of("foo", "null"))); + } + + @Test public void testToInteger() { Assert.assertEquals(5, run("TO_INTEGER(foo)", ImmutableMap.of("foo", "5"))); Assert.assertEquals(5, run("TO_INTEGER(foo)", ImmutableMap.of("foo", 5))); http://git-wip-us.apache.org/repos/asf/metron/blob/40c93527/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 6e92772..3f0af7e 100644 --- a/pom.xml +++ b/pom.xml @@ -307,6 +307,7 @@ <exclude>**/*.tokens</exclude> <exclude>**/*.log</exclude> <exclude>**/*.template</exclude> + <exclude>**/*.mapping</exclude> <exclude>**/.*</exclude> <exclude>**/.*/**</exclude> <exclude>**/*.seed</exclude>
