Repository: metron Updated Branches: refs/heads/master 8059c9fd4 -> f34affdb2
METRON-1073 Add option to limit fields returned from search query (merrimanr) closes apache/metron#677 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/f34affdb Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/f34affdb Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/f34affdb Branch: refs/heads/master Commit: f34affdb25422f19f8e70a91b7a796a15a7745a5 Parents: 8059c9f Author: merrimanr <[email protected]> Authored: Thu Aug 24 15:32:23 2017 -0500 Committer: merrimanr <[email protected]> Committed: Thu Aug 24 15:32:23 2017 -0500 ---------------------------------------------------------------------- .../elasticsearch/dao/ElasticsearchDao.java | 20 ++++++- .../indexing/dao/search/SearchRequest.java | 9 +++ .../indexing/dao/SearchIntegrationTest.java | 63 ++++++++++++++++++++ 3 files changed, 90 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/f34affdb/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java index 217da84..01c113c 100644 --- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java +++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java @@ -69,6 +69,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Optional; import java.util.function.Function; import java.util.stream.Collectors; @@ -114,8 +115,14 @@ public class ElasticsearchDao implements IndexDao { .size(searchRequest.getSize()) .from(searchRequest.getFrom()) .query(new QueryStringQueryBuilder(searchRequest.getQuery())) - .fetchSource(true) + .trackScores(true); + Optional<List<String>> fields = searchRequest.getFields(); + if (fields.isPresent()) { + searchSourceBuilder.fields(fields.get()); + } else { + searchSourceBuilder.fetchSource(true); + } for (SortField sortField : searchRequest.getSort()) { FieldSortBuilder fieldSortBuilder = new FieldSortBuilder(sortField.getField()); if (sortField.getSortOrder() == org.apache.metron.indexing.dao.search.SortOrder.DESC) { @@ -142,7 +149,16 @@ public class ElasticsearchDao implements IndexDao { searchResponse.setResults(Arrays.stream(elasticsearchResponse.getHits().getHits()).map(searchHit -> { SearchResult searchResult = new SearchResult(); searchResult.setId(searchHit.getId()); - searchResult.setSource(searchHit.getSource()); + Map<String, Object> source; + if (fields.isPresent()) { + source = new HashMap<>(); + searchHit.getFields().forEach((key, value) -> { + source.put(key, value.getValues().size() == 1 ? value.getValue() : value.getValues()); + }); + } else { + source = searchHit.getSource(); + } + searchResult.setSource(source); searchResult.setScore(searchHit.getScore()); searchResult.setIndex(searchHit.getIndex()); return searchResult; http://git-wip-us.apache.org/repos/asf/metron/blob/f34affdb/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SearchRequest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SearchRequest.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SearchRequest.java index 897f918..3a72002 100644 --- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SearchRequest.java +++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SearchRequest.java @@ -28,6 +28,7 @@ public class SearchRequest { private int size; private int from; private List<SortField> sort; + private List<String> fields; private List<String> facetFields; public SearchRequest() { @@ -99,6 +100,14 @@ public class SearchRequest { this.sort = sort; } + public Optional<List<String>> getFields() { + return fields == null || fields.size() == 0 ? Optional.empty() : Optional.of(fields); + } + + public void setFields(List<String> fields) { + this.fields = fields; + } + public Optional<List<String>> getFacetFields() { return facetFields == null || facetFields.size() == 0 ? Optional.empty() : Optional.of(facetFields); } http://git-wip-us.apache.org/repos/asf/metron/blob/f34affdb/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java index e262269..2645df2 100644 --- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java +++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java @@ -25,6 +25,10 @@ import org.apache.metron.indexing.dao.search.SearchRequest; import org.apache.metron.indexing.dao.search.SearchResponse; import org.apache.metron.indexing.dao.search.SearchResult; import org.apache.metron.integration.InMemoryComponent; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; import org.junit.*; import java.util.ArrayList; @@ -213,6 +217,42 @@ public abstract class SearchIntegrationTest { @Multiline public static String exceededMaxResultsQuery; + /** + * { + * "fields": ["ip_src_addr"], + * "indices": ["bro", "snort"], + * "query": "*", + * "from": 0, + * "size": 10, + * "sort": [ + * { + * "field": "timestamp", + * "sortOrder": "desc" + * } + * ] + * } + */ + @Multiline + public static String fieldsQuery; + + /** + * { + * "fields": ["ip_src_addr"], + * "indices": ["bro", "snort"], + * "query": "ip_src_addr:192.168.1.9", + * "from": 0, + * "size": 10, + * "sort": [ + * { + * "field": "timestamp", + * "sortOrder": "desc" + * } + * ] + * } + */ + @Multiline + public static String noResultsFieldsQuery; + protected static IndexDao dao; protected static InMemoryComponent indexComponent; @@ -464,6 +504,29 @@ public abstract class SearchIntegrationTest { Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("snort_field")); Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("duplicate_name_field")); } + //Fields query + { + SearchRequest request = JSONUtils.INSTANCE.load(fieldsQuery, SearchRequest.class); + SearchResponse response = dao.search(request); + Assert.assertEquals(10, response.getTotal()); + List<SearchResult> results = response.getResults(); + for(int i = 0;i < 5;++i) { + Map<String, Object> source = results.get(i).getSource(); + Assert.assertEquals(1, source.size()); + Assert.assertNotNull(source.get("ip_src_addr")); + } + for(int i = 5;i < 10;++i) { + Map<String, Object> source = results.get(i).getSource(); + Assert.assertEquals(1, source.size()); + Assert.assertNotNull(source.get("ip_src_addr")); + } + } + //No results fields query + { + SearchRequest request = JSONUtils.INSTANCE.load(noResultsFieldsQuery, SearchRequest.class); + SearchResponse response = dao.search(request); + Assert.assertEquals(0, response.getTotal()); + } } @AfterClass
