Repository: metron Updated Branches: refs/heads/master b47c9a572 -> 7f3eb2a4c
METRON-1247 REST search and findOne endpoints return unexpected or incorrect results for guids (justinleet) closes apache/metron#798 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/7f3eb2a4 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/7f3eb2a4 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/7f3eb2a4 Branch: refs/heads/master Commit: 7f3eb2a4c102aac1566ea969c4c1cde4e8942bdc Parents: b47c9a5 Author: justinleet <[email protected]> Authored: Fri Oct 13 09:39:57 2017 -0400 Committer: leet <[email protected]> Committed: Fri Oct 13 09:39:57 2017 -0400 ---------------------------------------------------------------------- metron-interface/metron-alerts/README.md | 6 +- metron-interface/metron-rest/README.md | 4 +- .../elasticsearch/dao/ElasticsearchDao.java | 60 ++++++------------ .../ElasticsearchSearchIntegrationTest.java | 65 ++++++++++++++++++-- .../indexing/dao/SearchIntegrationTest.java | 20 ++++++ 5 files changed, 106 insertions(+), 49 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/7f3eb2a4/metron-interface/metron-alerts/README.md ---------------------------------------------------------------------- diff --git a/metron-interface/metron-alerts/README.md b/metron-interface/metron-alerts/README.md index c312efa..b0433d0 100644 --- a/metron-interface/metron-alerts/README.md +++ b/metron-interface/metron-alerts/README.md @@ -6,7 +6,11 @@ - [Installing on an existing Cluster](#installing-on-an-existing-cluster) ## Caveats -* UI uses local storage to save all the data. A middleware needs to be designed and developed for persisting the data +### Local Storage +UI uses local storage to save all the data. A middleware needs to be designed and developed for persisting the data + +### Search for Alert GUIDs +Alert GUIDs must be double-quoted when being searched on to ensure correctness of results, e.g. guid:"id1". ## Prerequisites * The Metron REST application should be up and running and Elasticsearch should have some alerts populated by Metron topologies http://git-wip-us.apache.org/repos/asf/metron/blob/7f3eb2a4/metron-interface/metron-rest/README.md ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/README.md b/metron-interface/metron-rest/README.md index 394f4a8..3824917 100644 --- a/metron-interface/metron-rest/README.md +++ b/metron-interface/metron-rest/README.md @@ -430,14 +430,14 @@ Request and Response objects are JSON formatted. The JSON schemas are available * 200 - The meta alert was created ### `POST /api/v1/search/search` - * Description: Searches the indexing store + * Description: Searches the indexing store. GUIDs must be quoted to ensure correct results. * Input: * searchRequest - Search request * Returns: * 200 - Search response ### `POST /api/v1/search/group` - * Description: Searches the indexing store and returns field groups. Groups are hierarchical and nested in the order the fields appear in the 'groups' request parameter. The default sorting within groups is by count descending. A groupOrder type of count will sort based on then number of documents in a group while a groupType of term will sort by the groupBy term. + * Description: Searches the indexing store and returns field groups. GUIDs must be quoted to ensure correct results. Groups are hierarchical and nested in the order the fields appear in the 'groups' request parameter. The default sorting within groups is by count descending. A groupOrder type of count will sort based on then number of documents in a group while a groupType of term will sort by the groupBy term. * Input: * groupRequest - Group request * indices - list of indices to search http://git-wip-us.apache.org/repos/asf/metron/blob/7f3eb2a4/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java index e097a99..aa56ed0 100644 --- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java +++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java @@ -32,7 +32,6 @@ import java.util.Map; import java.util.Optional; import java.util.function.Function; import java.util.stream.Collectors; -import org.apache.metron.common.Constants; import org.apache.metron.elasticsearch.utils.ElasticsearchUtils; import org.apache.metron.indexing.dao.AccessConfig; import org.apache.metron.indexing.dao.IndexDao; @@ -46,20 +45,16 @@ import org.apache.metron.indexing.dao.search.GroupResult; import org.apache.metron.indexing.dao.search.InvalidSearchException; import org.apache.metron.indexing.dao.search.SearchRequest; import org.apache.metron.indexing.dao.search.SearchResponse; -import org.elasticsearch.action.ActionWriteResponse.ShardInfo; -import org.elasticsearch.action.index.IndexRequest; -import org.elasticsearch.action.search.*; -import org.elasticsearch.action.update.UpdateRequest; import org.apache.metron.indexing.dao.search.SearchResult; import org.apache.metron.indexing.dao.search.SortOrder; import org.apache.metron.indexing.dao.update.Document; +import org.elasticsearch.action.ActionWriteResponse.ShardInfo; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest; import org.elasticsearch.action.index.IndexRequest; -import org.elasticsearch.action.search.MultiSearchResponse; import org.elasticsearch.action.search.SearchPhaseExecutionException; -import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.update.UpdateRequest; +import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.client.transport.TransportClient; import org.elasticsearch.cluster.metadata.MappingMetaData; import org.elasticsearch.common.collect.ImmutableOpenMap; @@ -78,24 +73,6 @@ import org.elasticsearch.search.aggregations.bucket.terms.TermsBuilder; import org.elasticsearch.search.aggregations.metrics.sum.Sum; import org.elasticsearch.search.aggregations.metrics.sum.SumBuilder; import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.elasticsearch.search.sort.*; -import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.search.SearchHit; -import org.elasticsearch.search.SearchHits; -import java.io.IOException; -import java.util.Arrays; -import java.util.Date; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.function.Function; -import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -253,30 +230,31 @@ public class ElasticsearchDao implements IndexDao { * Return the search hit based on the UUID and sensor type. * A callback can be specified to transform the hit into a type T. * If more than one hit happens, the first one will be returned. - * @throws IOException */ - <T> Optional<T> searchByGuid(String guid, String sensorType, Function<SearchHit, Optional<T>> callback) throws IOException{ - QueryBuilder query = QueryBuilders.matchQuery(Constants.GUID, guid); + <T> Optional<T> searchByGuid(String guid, String sensorType, + Function<SearchHit, Optional<T>> callback) { + QueryBuilder query = QueryBuilders.idsQuery(sensorType + "_doc").ids(guid); SearchRequestBuilder request = client.prepareSearch() - .setTypes(sensorType + "_doc") .setQuery(query) .setSource("message") ; - MultiSearchResponse response = client.prepareMultiSearch() - .add(request) - .get(); - for(MultiSearchResponse.Item i : response) { - org.elasticsearch.action.search.SearchResponse resp = i.getResponse(); - SearchHits hits = resp.getHits(); - for(SearchHit hit : hits) { - Optional<T> ret = callback.apply(hit); - if(ret.isPresent()) { - return ret; - } + org.elasticsearch.action.search.SearchResponse response = request.get(); + SearchHits hits = response.getHits(); + long totalHits = hits.getTotalHits(); + if (totalHits > 1) { + LOG.warn("Encountered {} results for guid {} in sensor {}. Returning first hit.", + totalHits, + guid, + sensorType + ); + } + for (SearchHit hit : hits) { + Optional<T> ret = callback.apply(hit); + if (ret.isPresent()) { + return ret; } } return Optional.empty(); - } @Override http://git-wip-us.apache.org/repos/asf/metron/blob/7f3eb2a4/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java index adb69ee..e21bb13 100644 --- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java +++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java @@ -18,6 +18,10 @@ package org.apache.metron.elasticsearch.integration; +import java.io.File; +import java.io.IOException; +import java.util.HashMap; +import java.util.concurrent.ExecutionException; import org.adrianwalker.multilinestring.Multiline; import org.apache.metron.elasticsearch.dao.ElasticsearchDao; import org.apache.metron.elasticsearch.dao.ElasticsearchMetaAlertDao; @@ -29,18 +33,21 @@ import org.apache.metron.indexing.dao.SearchIntegrationTest; import org.apache.metron.integration.InMemoryComponent; import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.index.IndexRequestBuilder; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.SearchHit; import org.json.simple.JSONArray; import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; import org.json.simple.parser.ParseException; -import java.io.File; -import java.util.HashMap; - public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest { private static String indexDir = "target/elasticsearch_search"; private static String dateFormat = "yyyy.MM.dd.HH"; + private static final int MAX_RETRIES = 10; + private static final int SLEEP_MS = 500; /** * { @@ -120,7 +127,8 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest { } @Override - protected void loadTestData() throws ParseException { + protected void loadTestData() + throws ParseException, IOException, ExecutionException, InterruptedException { ElasticSearchComponent es = (ElasticSearchComponent)indexComponent; es.getClient().admin().indices().prepareCreate("bro_index_2017.01.01.01") .addMapping("bro_doc", broTypeMappings).get(); @@ -149,12 +157,59 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest { JSONObject jsonObject = (JSONObject) o; IndexRequestBuilder indexRequestBuilder = es.getClient().prepareIndex("metaalerts", "metaalert_doc"); indexRequestBuilder = indexRequestBuilder.setSource(jsonObject.toJSONString()); -// indexRequestBuilder = indexRequestBuilder.setTimestamp(jsonObject.get("timestamp").toString()); bulkRequest.add(indexRequestBuilder); } BulkResponse bulkResponse = bulkRequest.execute().actionGet(); if (bulkResponse.hasFailures()) { throw new RuntimeException("Failed to index test data"); } + + SearchResponse broDocs = es.getClient() + .prepareSearch("bro_index_2017.01.01.01") + .setTypes("bro_doc") + .setQuery(QueryBuilders.matchAllQuery()) + .get(); + // We're changing the _id field, we need to create a copy and delete the original. + for (SearchHit hit : broDocs.getHits()) { + // Bro GUIDs to collide while using the standard analyzer + // Use timestamp as part of guid because query may not return in order each time + IndexRequest indexRequest = new IndexRequest() + .index("bro_index_2017.01.01.01") + .type("bro_doc") + .id("bro-" + hit.getSource().get("timestamp")) + .source(hit.getSource()); + es.getClient().index(indexRequest).get(); + + // Delete the original + es.getClient() + .prepareDelete("bro_index_2017.01.01.01", "bro_doc", hit.getId()) + .get(); + } + + // Wait until everything is updated + // Assume true until proven otherwise. + boolean allUpdated = true; + for (int t = 0; t < MAX_RETRIES; ++t, Thread.sleep(SLEEP_MS)) { + allUpdated = true; + SearchResponse response = es.getClient() + .prepareSearch("bro_index_2017.01.01.01") + .setTypes("bro_doc") + .setQuery(QueryBuilders.matchAllQuery()) + .get(); + if (response.getHits().getTotalHits() == 0) { + throw new IllegalStateException("Bro index is empty. No docs to validate were updated"); + } + for (SearchHit hit : response.getHits()) { + if (!hit.getId().startsWith("bro-")) { + allUpdated = false; + } + } + if (allUpdated) { + break; + } + } + if (!allUpdated) { + throw new IllegalStateException("Unable to update Elasticsearch ids properly"); + } } } http://git-wip-us.apache.org/repos/asf/metron/blob/7f3eb2a4/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java index 26d1a75..e2a37f1 100644 --- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java +++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java @@ -17,9 +17,11 @@ */ package org.apache.metron.indexing.dao; +import java.util.Optional; import org.adrianwalker.multilinestring.Multiline; import org.apache.metron.common.utils.JSONUtils; import org.apache.metron.indexing.dao.search.FieldType; +import org.apache.metron.indexing.dao.search.GetRequest; import org.apache.metron.indexing.dao.search.GroupRequest; import org.apache.metron.indexing.dao.search.GroupResponse; import org.apache.metron.indexing.dao.search.InvalidSearchException; @@ -93,6 +95,15 @@ public abstract class SearchIntegrationTest { /** * { + * "guid": "bro-3", + * "sensorType": "bro" + * } + */ + @Multiline + public static String findOneGuidQuery; + + /** + * { * "indices": ["bro", "snort"], * "query": "ip_src_addr:192.168.1.1", * "from": 0, @@ -370,6 +381,15 @@ public abstract class SearchIntegrationTest { Assert.assertEquals(10-i, results.get(i).getSource().get("timestamp")); } } + //Find One Guid Testcase + { + GetRequest request = JSONUtils.INSTANCE.load(findOneGuidQuery, GetRequest.class); + Optional<Map<String, Object>> response = dao.getLatestResult(request); + Assert.assertTrue(response.isPresent()); + Map<String, Object> doc = response.get(); + Assert.assertEquals("bro", doc.get("source:type")); + Assert.assertEquals(3, doc.get("timestamp")); + } //Filter test case { SearchRequest request = JSONUtils.INSTANCE.load(filterQuery, SearchRequest.class);
