Repository: metron Updated Branches: refs/heads/master fd4a6d164 -> 768a6fada
METRON-1294 IP addresses are not formatted correctly in facet and group results (merrimanr) closes apache/metron#827 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/768a6fad Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/768a6fad Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/768a6fad Branch: refs/heads/master Commit: 768a6fada36bdb03be63052024d59ad3754d616d Parents: fd4a6d1 Author: merrimanr <[email protected]> Authored: Fri Nov 17 11:04:41 2017 -0600 Committer: merrimanr <[email protected]> Committed: Fri Nov 17 11:04:41 2017 -0600 ---------------------------------------------------------------------- .../src/app/service/search.service.ts | 11 +- metron-interface/metron-rest/README.md | 12 +- .../rest/controller/SearchController.java | 15 +-- .../metron/rest/service/SearchService.java | 3 +- .../rest/service/impl/SearchServiceImpl.java | 12 +- .../SearchControllerIntegrationTest.java | 77 ++++-------- .../elasticsearch/dao/ElasticsearchDao.java | 115 +++++++++-------- .../dao/ElasticsearchMetaAlertDao.java | 8 +- .../dao/ElasticsearchMetaAlertDaoTest.java | 8 +- .../ElasticsearchSearchIntegrationTest.java | 15 +++ .../apache/metron/indexing/dao/HBaseDao.java | 7 +- .../apache/metron/indexing/dao/IndexDao.java | 3 +- .../metron/indexing/dao/MultiIndexDao.java | 15 +-- .../apache/metron/indexing/dao/InMemoryDao.java | 35 +++--- .../indexing/dao/InMemoryMetaAlertDao.java | 7 +- .../indexing/dao/SearchIntegrationTest.java | 125 +++++++++---------- 16 files changed, 197 insertions(+), 271 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/768a6fad/metron-interface/metron-alerts/src/app/service/search.service.ts ---------------------------------------------------------------------- diff --git a/metron-interface/metron-alerts/src/app/service/search.service.ts b/metron-interface/metron-alerts/src/app/service/search.service.ts index 4bbcc2d..1e97e96 100644 --- a/metron-interface/metron-alerts/src/app/service/search.service.ts +++ b/metron-interface/metron-alerts/src/app/service/search.service.ts @@ -43,13 +43,10 @@ export class SearchService { let processedKeys: string[] = []; let columnMetadatas: ColumnMetadata[] = []; - for (let index of Object.keys(response)) { - let indexMetaData = response[index]; - for (let key of Object.keys(indexMetaData)) { - if (processedKeys.indexOf(key) === -1) { - processedKeys.push(key); - columnMetadatas.push(new ColumnMetadata(key, indexMetaData[key])); - } + for (let key of Object.keys(response)) { + if (processedKeys.indexOf(key) === -1) { + processedKeys.push(key); + columnMetadatas.push(new ColumnMetadata(key, response[key])); } } http://git-wip-us.apache.org/repos/asf/metron/blob/768a6fad/metron-interface/metron-rest/README.md ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/README.md b/metron-interface/metron-rest/README.md index 724239b..e46f865 100644 --- a/metron-interface/metron-rest/README.md +++ b/metron-interface/metron-rest/README.md @@ -226,7 +226,6 @@ Request and Response objects are JSON formatted. The JSON schemas are available | [ `POST /api/v1/search/group`](#get-apiv1searchgroup)| | [ `GET /api/v1/search/findOne`](#get-apiv1searchfindone)| | [ `GET /api/v1/search/column/metadata`](#get-apiv1searchcolumnmetadata)| -| [ `GET /api/v1/search/column/metadata/common`](#get-apiv1searchcolumnmetadatacommon)| | [ `GET /api/v1/sensor/enrichment/config`](#get-apiv1sensorenrichmentconfig)| | [ `GET /api/v1/sensor/enrichment/config/list/available/enrichments`](#get-apiv1sensorenrichmentconfiglistavailableenrichments)| | [ `GET /api/v1/sensor/enrichment/config/list/available/threat/triage/aggregators`](#get-apiv1sensorenrichmentconfiglistavailablethreattriageaggregators)| @@ -489,18 +488,11 @@ Request and Response objects are JSON formatted. The JSON schemas are available * 404 - Document with UUID and sensor type not found ### `GET /api/v1/search/column/metadata` - * Description: Get column metadata for each index in the list of indicies + * Description: Get index column metadata for a list of sensor types with duplicates removed. Column names and types for each sensor are retrieved from the most recent index. Columns that exist in multiple indices with different types will default to type 'other'. * Input: - * indices - Indices + * sensorTypes - Sensor Types * Returns: * 200 - Column Metadata - -### `GET /api/v1/search/column/metadata/common` - * Description: Get metadata for columns shared by the list of indices - * Input: - * indices - Indices - * Returns: - * 200 - Common Column Metadata ### `GET /api/v1/sensor/enrichment/config` * Description: Retrieves all SensorEnrichmentConfigs from Zookeeper http://git-wip-us.apache.org/repos/asf/metron/blob/768a6fad/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/SearchController.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/SearchController.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/SearchController.java index e215413..2748858 100644 --- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/SearchController.java +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/SearchController.java @@ -81,17 +81,12 @@ public class SearchController { } } - @ApiOperation(value = "Get column metadata for each index in the list of indices") + @ApiOperation(value = "Get index column metadata for a list of sensor types with duplicates removed. " + + "Column names and types for each sensor are retrieved from the most recent index. " + + "Columns that exist in multiple indices with different types will default to type 'other'.") @ApiResponse(message = "Column Metadata", code = 200) @RequestMapping(value = "/column/metadata", method = RequestMethod.POST) - ResponseEntity<Map<String, Map<String, FieldType>>> getColumnMetadata(final @ApiParam(name = "indices", value = "Indices", required = true) @RequestBody List<String> indices) throws RestException { - return new ResponseEntity<>(searchService.getColumnMetadata(indices), HttpStatus.OK); - } - - @ApiOperation(value = "Get metadata for columns shared by the list of indices") - @ApiResponse(message = "Common Column Metadata", code = 200) - @RequestMapping(value = "/column/metadata/common", method = RequestMethod.POST) - ResponseEntity<Map<String, FieldType>> getCommonColumnMetadata(final @ApiParam(name = "indices", value = "Indices", required = true) @RequestBody List<String> indices) throws RestException { - return new ResponseEntity<>(searchService.getCommonColumnMetadata(indices), HttpStatus.OK); + ResponseEntity<Map<String, FieldType>> getColumnMetadata(final @ApiParam(name = "sensorTypes", value = "Sensor Types", required = true) @RequestBody List<String> sensorTypes) throws RestException { + return new ResponseEntity<>(searchService.getColumnMetadata(sensorTypes), HttpStatus.OK); } } http://git-wip-us.apache.org/repos/asf/metron/blob/768a6fad/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/SearchService.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/SearchService.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/SearchService.java index 5899765..a2f9c11 100644 --- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/SearchService.java +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/SearchService.java @@ -34,7 +34,6 @@ public interface SearchService { SearchResponse search(SearchRequest searchRequest) throws RestException; GroupResponse group(GroupRequest groupRequest) throws RestException; Optional<Map<String, Object>> getLatest(GetRequest request) throws RestException; - Map<String, Map<String, FieldType>> getColumnMetadata(List<String> indices) throws RestException; - Map<String, FieldType> getCommonColumnMetadata(List<String> indices) throws RestException; + Map<String, FieldType> getColumnMetadata(List<String> indices) throws RestException; } http://git-wip-us.apache.org/repos/asf/metron/blob/768a6fad/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SearchServiceImpl.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SearchServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SearchServiceImpl.java index 433eae3..a696f68 100644 --- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SearchServiceImpl.java +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SearchServiceImpl.java @@ -94,7 +94,7 @@ public class SearchServiceImpl implements SearchService { } @Override - public Map<String, Map<String, FieldType>> getColumnMetadata(List<String> indices) throws RestException { + public Map<String, FieldType> getColumnMetadata(List<String> indices) throws RestException { try { return dao.getColumnMetadata(indices); } @@ -103,16 +103,6 @@ public class SearchServiceImpl implements SearchService { } } - @Override - public Map<String, FieldType> getCommonColumnMetadata(List<String> indices) throws RestException { - try { - return dao.getCommonColumnMetadata(indices); - } - catch(IOException ioe) { - throw new RestException(ioe.getMessage(), ioe); - } - } - private List<String> getDefaultIndices() throws RestException { // Pull the indices from the cache by default List<String> indices = Lists.newArrayList((sensorIndexingConfigService.getAllIndices(environment.getProperty(INDEX_WRITER_NAME)))); http://git-wip-us.apache.org/repos/asf/metron/blob/768a6fad/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SearchControllerIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SearchControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SearchControllerIntegrationTest.java index 2c15671..3673654 100644 --- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SearchControllerIntegrationTest.java +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SearchControllerIntegrationTest.java @@ -236,60 +236,37 @@ public class SearchControllerIntegrationTest extends DaoControllerTest { .andExpect(jsonPath("$.groupResults[0].groupResults[0].score").value(50)); this.mockMvc.perform(post(searchUrl + "/column/metadata").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content("[\"bro\",\"snort\"]")) - .andExpect(status().isOk()) - .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) - .andExpect(jsonPath("$.*", hasSize(2))) - .andExpect(jsonPath("$.bro.common_string_field").value("string")) - .andExpect(jsonPath("$.bro.common_integer_field").value("integer")) - .andExpect(jsonPath("$.bro.bro_field").value("boolean")) - .andExpect(jsonPath("$.bro.duplicate_field").value("date")) - .andExpect(jsonPath("$.snort.common_string_field").value("string")) - .andExpect(jsonPath("$.snort.common_integer_field").value("integer")) - .andExpect(jsonPath("$.snort.snort_field").value("double")) - .andExpect(jsonPath("$.snort.duplicate_field").value("long")); - - this.mockMvc.perform(post(searchUrl + "/column/metadata/common").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content("[\"bro\",\"snort\"]")) - .andExpect(status().isOk()) - .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) - .andExpect(jsonPath("$.*", hasSize(2))) - .andExpect(jsonPath("$.common_string_field").value("string")) - .andExpect(jsonPath("$.common_integer_field").value("integer")); + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) + .andExpect(jsonPath("$.*", hasSize(5))) + .andExpect(jsonPath("$.common_string_field").value("string")) + .andExpect(jsonPath("$.common_integer_field").value("integer")) + .andExpect(jsonPath("$.bro_field").value("boolean")) + .andExpect(jsonPath("$.snort_field").value("double")) + .andExpect(jsonPath("$.duplicate_field").value("other")); this.mockMvc.perform(post(searchUrl + "/column/metadata").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content("[\"bro\"]")) - .andExpect(status().isOk()) - .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) - .andExpect(jsonPath("$.*", hasSize(1))) - .andExpect(jsonPath("$.bro.common_string_field").value("string")) - .andExpect(jsonPath("$.bro.common_integer_field").value("integer")) - .andExpect(jsonPath("$.bro.bro_field").value("boolean")) - .andExpect(jsonPath("$.bro.duplicate_field").value("date")); - - this.mockMvc.perform(post(searchUrl + "/column/metadata/common").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content("[\"bro\"]")) - .andExpect(status().isOk()) - .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) - .andExpect(jsonPath("$.*", hasSize(4))) - .andExpect(jsonPath("$.common_string_field").value("string")) - .andExpect(jsonPath("$.common_integer_field").value("integer")) - .andExpect(jsonPath("$.bro_field").value("boolean")) - .andExpect(jsonPath("$.duplicate_field").value("date")); + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) + .andExpect(jsonPath("$.*", hasSize(4))) + .andExpect(jsonPath("$.common_string_field").value("string")) + .andExpect(jsonPath("$.common_integer_field").value("integer")) + .andExpect(jsonPath("$.bro_field").value("boolean")) + .andExpect(jsonPath("$.duplicate_field").value("date")); this.mockMvc.perform(post(searchUrl + "/column/metadata").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content("[\"snort\"]")) - .andExpect(status().isOk()) - .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) - .andExpect(jsonPath("$.*", hasSize(1))) - .andExpect(jsonPath("$.snort.common_string_field").value("string")) - .andExpect(jsonPath("$.snort.common_integer_field").value("integer")) - .andExpect(jsonPath("$.snort.snort_field").value("double")) - .andExpect(jsonPath("$.snort.duplicate_field").value("long")); - - this.mockMvc.perform(post(searchUrl + "/column/metadata/common").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content("[\"snort\"]")) - .andExpect(status().isOk()) - .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) - .andExpect(jsonPath("$.*", hasSize(4))) - .andExpect(jsonPath("$.common_string_field").value("string")) - .andExpect(jsonPath("$.common_integer_field").value("integer")) - .andExpect(jsonPath("$.snort_field").value("double")) - .andExpect(jsonPath("$.duplicate_field").value("long")); + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) + .andExpect(jsonPath("$.*", hasSize(4))) + .andExpect(jsonPath("$.common_string_field").value("string")) + .andExpect(jsonPath("$.common_integer_field").value("integer")) + .andExpect(jsonPath("$.snort_field").value("double")) + .andExpect(jsonPath("$.duplicate_field").value("long")); + + this.mockMvc.perform(post(searchUrl + "/column/metadata").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content("[\"someindex\"]")) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) + .andExpect(jsonPath("$.*", hasSize(0))); } http://git-wip-us.apache.org/repos/asf/metron/blob/768a6fad/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java index 61d5472..87ad7f7 100644 --- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java +++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java @@ -34,6 +34,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; import org.apache.metron.elasticsearch.utils.ElasticsearchUtils; @@ -88,12 +89,10 @@ public class ElasticsearchDao implements IndexDao { private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); private transient TransportClient client; private AccessConfig accessConfig; - private List<String> ignoredIndices = new ArrayList<>(); protected ElasticsearchDao(TransportClient client, AccessConfig config) { this.client = client; this.accessConfig = config; - this.ignoredIndices.add(".kibana"); } public ElasticsearchDao() { @@ -166,7 +165,7 @@ public class ElasticsearchDao implements IndexDao { if (facetFields.isPresent()) { Map<String, FieldType> commonColumnMetadata; try { - commonColumnMetadata = getCommonColumnMetadata(searchRequest.getIndices()); + commonColumnMetadata = getColumnMetadata(searchRequest.getIndices()); } catch (IOException e) { throw new InvalidSearchException(String.format("Could not get common column metadata for indices %s", Arrays.toString(searchRequest.getIndices().toArray()))); } @@ -211,7 +210,7 @@ public class ElasticsearchDao implements IndexDao { } Map<String, FieldType> commonColumnMetadata; try { - commonColumnMetadata = getCommonColumnMetadata(groupRequest.getIndices()); + commonColumnMetadata = getColumnMetadata(groupRequest.getIndices()); } catch (IOException e) { throw new InvalidSearchException(String .format("Could not get common column metadata for indices %s", @@ -406,72 +405,70 @@ public class ElasticsearchDao implements IndexDao { @SuppressWarnings("unchecked") @Override - public Map<String, Map<String, FieldType>> getColumnMetadata(List<String> indices) throws IOException { - Map<String, Map<String, FieldType>> allColumnMetadata = new HashMap<>(); - String[] latestIndices = getLatestIndices(indices); - ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> mappings = client - .admin() - .indices() - .getMappings(new GetMappingsRequest().indices(latestIndices)) - .actionGet() - .getMappings(); - for(Object key: mappings.keys().toArray()) { - String indexName = key.toString(); - - Map<String, FieldType> indexColumnMetadata = new HashMap<>(); - ImmutableOpenMap<String, MappingMetaData> mapping = mappings.get(indexName); - Iterator<String> mappingIterator = mapping.keysIt(); - while(mappingIterator.hasNext()) { - MappingMetaData mappingMetaData = mapping.get(mappingIterator.next()); - Map<String, Map<String, String>> map = (Map<String, Map<String, String>>) mappingMetaData.getSourceAsMap().get("properties"); - for(String field: map.keySet()) { - indexColumnMetadata.put(field, elasticsearchSearchTypeMap.getOrDefault(map.get(field).get("type"), FieldType.OTHER)); - } - } + public Map<String, FieldType> getColumnMetadata(List<String> indices) throws IOException { + Map<String, FieldType> indexColumnMetadata = new HashMap<>(); - String baseIndexName = ElasticsearchUtils.getBaseIndexName(indexName); - allColumnMetadata.put(baseIndexName, indexColumnMetadata); - } - return allColumnMetadata; - } + // Keep track of the last index used to inspect a field type so we can print a helpful error message on type mismatch + Map<String, String> previousIndices = new HashMap<>(); + // If we have detected a field type mismatch, ignore the field going forward since the type has been set to OTHER + Set<String> fieldBlackList = new HashSet<>(); - @SuppressWarnings("unchecked") - @Override - public Map<String, FieldType> getCommonColumnMetadata(List<String> indices) throws IOException { - Map<String, FieldType> commonColumnMetadata = null; - ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> mappings = - client.admin().indices().getMappings(new GetMappingsRequest().indices(getLatestIndices(indices))).actionGet().getMappings(); - for(Object index: mappings.keys().toArray()) { - ImmutableOpenMap<String, MappingMetaData> mapping = mappings.get(index.toString()); - Iterator<String> mappingIterator = mapping.keysIt(); - while(mappingIterator.hasNext()) { - MappingMetaData mappingMetaData = mapping.get(mappingIterator.next()); - Map<String, Map<String, String>> map = (Map<String, Map<String, String>>) mappingMetaData.getSourceAsMap().get("properties"); - Map<String, FieldType> mappingsWithTypes = map.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, - e-> elasticsearchSearchTypeMap.getOrDefault(e.getValue().get("type"), FieldType.OTHER))); - if (commonColumnMetadata == null) { - commonColumnMetadata = mappingsWithTypes; - } else { - commonColumnMetadata.entrySet().retainAll(mappingsWithTypes.entrySet()); + String[] latestIndices = getLatestIndices(indices); + if (latestIndices.length > 0) { + ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> mappings = client + .admin() + .indices() + .getMappings(new GetMappingsRequest().indices(latestIndices)) + .actionGet() + .getMappings(); + for (Object key : mappings.keys().toArray()) { + String indexName = key.toString(); + ImmutableOpenMap<String, MappingMetaData> mapping = mappings.get(indexName); + Iterator<String> mappingIterator = mapping.keysIt(); + while (mappingIterator.hasNext()) { + MappingMetaData mappingMetaData = mapping.get(mappingIterator.next()); + Map<String, Map<String, String>> map = (Map<String, Map<String, String>>) mappingMetaData + .getSourceAsMap().get("properties"); + for (String field : map.keySet()) { + if (!fieldBlackList.contains(field)) { + FieldType type = elasticsearchSearchTypeMap + .getOrDefault(map.get(field).get("type"), FieldType.OTHER); + if (indexColumnMetadata.containsKey(field)) { + FieldType previousType = indexColumnMetadata.get(field); + if (!type.equals(previousType)) { + String previousIndexName = previousIndices.get(field); + LOG.error(String.format( + "Field type mismatch: %s.%s has type %s while %s.%s has type %s. Defaulting type to %s.", + indexName, field, type.getFieldType(), + previousIndexName, field, previousType.getFieldType(), + FieldType.OTHER.getFieldType())); + indexColumnMetadata.put(field, FieldType.OTHER); + // Detected a type mismatch so ignore the field from now on + fieldBlackList.add(field); + } + } else { + indexColumnMetadata.put(field, type); + previousIndices.put(field, indexName); + } + } + } } } } - return commonColumnMetadata; + return indexColumnMetadata; } protected String[] getLatestIndices(List<String> includeIndices) { Map<String, String> latestIndices = new HashMap<>(); String[] indices = client.admin().indices().prepareGetIndex().setFeatures().get().getIndices(); for (String index : indices) { - if (!ignoredIndices.contains(index)) { - int prefixEnd = index.indexOf(INDEX_NAME_DELIMITER); - if (prefixEnd != -1) { - String prefix = index.substring(0, prefixEnd); - if (includeIndices.contains(prefix)) { - String latestIndex = latestIndices.get(prefix); - if (latestIndex == null || index.compareTo(latestIndex) > 0) { - latestIndices.put(prefix, index); - } + int prefixEnd = index.indexOf(INDEX_NAME_DELIMITER); + if (prefixEnd != -1) { + String prefix = index.substring(0, prefixEnd); + if (includeIndices.contains(prefix)) { + String latestIndex = latestIndices.get(prefix); + if (latestIndex == null || index.compareTo(latestIndex) > 0) { + latestIndices.put(prefix, index); } } } http://git-wip-us.apache.org/repos/asf/metron/blob/768a6fad/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java index c24ba0c..90d5410 100644 --- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java +++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java @@ -574,18 +574,12 @@ public class ElasticsearchMetaAlertDao implements MetaAlertDao { } @Override - public Map<String, Map<String, FieldType>> getColumnMetadata(List<String> indices) + public Map<String, FieldType> getColumnMetadata(List<String> indices) throws IOException { return indexDao.getColumnMetadata(indices); } @Override - public Map<String, FieldType> getCommonColumnMetadata(List<String> indices) throws - IOException { - return indexDao.getCommonColumnMetadata(indices); - } - - @Override public GroupResponse group(GroupRequest groupRequest) throws InvalidSearchException { // Wrap the query to hide any alerts already contained in meta alerts QueryBuilder qb = QueryBuilders.boolQuery() http://git-wip-us.apache.org/repos/asf/metron/blob/768a6fad/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDaoTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDaoTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDaoTest.java index a1027f7..ffafe52 100644 --- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDaoTest.java +++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDaoTest.java @@ -87,13 +87,7 @@ public class ElasticsearchMetaAlertDaoTest { } @Override - public Map<String, Map<String, FieldType>> getColumnMetadata(List<String> indices) - throws IOException { - return null; - } - - @Override - public Map<String, FieldType> getCommonColumnMetadata(List<String> indices) + public Map<String, FieldType> getColumnMetadata(List<String> indices) throws IOException { return null; } http://git-wip-us.apache.org/repos/asf/metron/blob/768a6fad/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java index e7b609e..07cc708 100644 --- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java +++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java @@ -93,6 +93,19 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest { @Multiline private static String snortTypeMappings; + /** + * { + * "metaalert_doc": { + * "properties": { + * "source:type": { "type": "string" }, + * "alert": { "type": "nested"} + * } + * } + * } + */ + @Multiline + private static String metaAlertTypeMappings; + @Override protected IndexDao createDao() throws Exception { @@ -134,6 +147,8 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest { .addMapping("bro_doc", broTypeMappings).get(); es.getClient().admin().indices().prepareCreate("snort_index_2017.01.01.02") .addMapping("snort_doc", snortTypeMappings).get(); + es.getClient().admin().indices().prepareCreate("metaalert_index") + .addMapping("metaalert_doc", metaAlertTypeMappings).get(); BulkRequestBuilder bulkRequest = es.getClient().prepareBulk().setRefresh(true); JSONArray broArray = (JSONArray) new JSONParser().parse(broData); http://git-wip-us.apache.org/repos/asf/metron/blob/768a6fad/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/HBaseDao.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/HBaseDao.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/HBaseDao.java index 3103ea7..72f2980 100644 --- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/HBaseDao.java +++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/HBaseDao.java @@ -260,12 +260,7 @@ public class HBaseDao implements IndexDao { @Override - public Map<String, Map<String, FieldType>> getColumnMetadata(List<String> indices) throws IOException { - return null; - } - - @Override - public Map<String, FieldType> getCommonColumnMetadata(List<String> indices) throws IOException { + public Map<String, FieldType> getColumnMetadata(List<String> indices) throws IOException { return null; } } http://git-wip-us.apache.org/repos/asf/metron/blob/768a6fad/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/IndexDao.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/IndexDao.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/IndexDao.java index 8855a14..03d348a 100644 --- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/IndexDao.java +++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/IndexDao.java @@ -167,6 +167,5 @@ public interface IndexDao { update(d, Optional.ofNullable(request.getIndex())); } - Map<String, Map<String, FieldType>> getColumnMetadata(List<String> indices) throws IOException; - Map<String, FieldType> getCommonColumnMetadata(List<String> indices) throws IOException; + Map<String, FieldType> getColumnMetadata(List<String> indices) throws IOException; } http://git-wip-us.apache.org/repos/asf/metron/blob/768a6fad/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MultiIndexDao.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MultiIndexDao.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MultiIndexDao.java index ed8bc95..dad08d6 100644 --- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MultiIndexDao.java +++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MultiIndexDao.java @@ -88,20 +88,9 @@ public class MultiIndexDao implements IndexDao { } @Override - public Map<String, Map<String, FieldType>> getColumnMetadata(List<String> in) throws IOException { + public Map<String, FieldType> getColumnMetadata(List<String> in) throws IOException { for(IndexDao dao : indices) { - Map<String, Map<String, FieldType>> r = dao.getColumnMetadata(in); - if(r != null) { - return r; - } - } - return null; - } - - @Override - public Map<String, FieldType> getCommonColumnMetadata(List<String> in) throws IOException { - for(IndexDao dao : indices) { - Map<String, FieldType> r = dao.getCommonColumnMetadata(in); + Map<String, FieldType> r = dao.getColumnMetadata(in); if(r != null) { return r; } http://git-wip-us.apache.org/repos/asf/metron/blob/768a6fad/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryDao.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryDao.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryDao.java index 3bce4d0..f2108de 100644 --- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryDao.java +++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryDao.java @@ -17,13 +17,11 @@ */ package org.apache.metron.indexing.dao; -import static org.apache.metron.common.Constants.SENSOR_TYPE; - import com.fasterxml.jackson.core.type.TypeReference; import com.google.common.base.Splitter; import com.google.common.collect.ComparisonChain; import com.google.common.collect.Iterables; -import java.util.stream.Collectors; +import java.util.Map.Entry; import org.apache.metron.common.Constants; import org.apache.metron.common.utils.JSONUtils; import org.apache.metron.indexing.dao.search.*; @@ -242,25 +240,26 @@ public class InMemoryDao implements IndexDao { } } - public Map<String, Map<String, FieldType>> getColumnMetadata(List<String> indices) throws IOException { - Map<String, Map<String, FieldType>> columnMetadata = new HashMap<>(); - for(String index: indices) { - columnMetadata.put(index, new HashMap<>(COLUMN_METADATA.get(index))); - } - return columnMetadata; - } - @Override - public Map<String, FieldType> getCommonColumnMetadata(List<String> indices) throws IOException { - Map<String, FieldType> commonColumnMetadata = new HashMap<>(); + public Map<String, FieldType> getColumnMetadata(List<String> indices) throws IOException { + Map<String, FieldType> indexColumnMetadata = new HashMap<>(); for(String index: indices) { - if (commonColumnMetadata.isEmpty()) { - commonColumnMetadata = new HashMap<>(COLUMN_METADATA.get(index)); - } else { - commonColumnMetadata.entrySet().retainAll(COLUMN_METADATA.get(index).entrySet()); + if (COLUMN_METADATA.containsKey(index)) { + Map<String, FieldType> columnMetadata = COLUMN_METADATA.get(index); + for (Entry entry: columnMetadata.entrySet()) { + String field = (String) entry.getKey(); + FieldType type = (FieldType) entry.getValue(); + if (indexColumnMetadata.containsKey(field)) { + if (!type.equals(indexColumnMetadata.get(field))) { + indexColumnMetadata.put(field, FieldType.OTHER); + } + } else { + indexColumnMetadata.put(field, type); + } + } } } - return commonColumnMetadata; + return indexColumnMetadata; } public static void setColumnMetadata(Map<String, Map<String, FieldType>> columnMetadata) { http://git-wip-us.apache.org/repos/asf/metron/blob/768a6fad/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryMetaAlertDao.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryMetaAlertDao.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryMetaAlertDao.java index fad0eda..baa5416 100644 --- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryMetaAlertDao.java +++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryMetaAlertDao.java @@ -117,17 +117,12 @@ public class InMemoryMetaAlertDao implements MetaAlertDao { } @Override - public Map<String, Map<String, FieldType>> getColumnMetadata(List<String> indices) + public Map<String, FieldType> getColumnMetadata(List<String> indices) throws IOException { return indexDao.getColumnMetadata(indices); } @Override - public Map<String, FieldType> getCommonColumnMetadata(List<String> indices) throws IOException { - return indexDao.getCommonColumnMetadata(indices); - } - - @Override public Optional<Map<String, Object>> getLatestResult(GetRequest request) throws IOException { return indexDao.getLatestResult(request); } http://git-wip-us.apache.org/repos/asf/metron/blob/768a6fad/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java index d991d50..8f32946 100644 --- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java +++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java @@ -190,7 +190,7 @@ public abstract class SearchIntegrationTest { /** * { * "facetFields": ["source:type", "ip_src_addr", "ip_src_port", "long_field", "timestamp", "latitude", "score", "is_alert"], - * "indices": ["bro", "snort"], + * "indices": ["bro", "snort", "metaalert"], * "query": "*", * "from": 0, * "size": 10, @@ -323,7 +323,7 @@ public abstract class SearchIntegrationTest { * } * ], * "scoreField":"score", - * "indices": ["bro", "snort"], + * "indices": ["bro", "snort", "metaalert"], * "query": "*" * } */ @@ -348,7 +348,7 @@ public abstract class SearchIntegrationTest { * } * } * ], - * "indices": ["bro", "snort"], + * "indices": ["bro", "snort", "metaalert"], * "query": "*" * } */ @@ -369,6 +369,24 @@ public abstract class SearchIntegrationTest { @Multiline public static String badGroupQuery; + /** + * { + * "groups": [ + * { + * "field":"ip_src_addr", + * "order": { + * "groupOrderType": "term", + * "sortOrder": "DESC" + * } + * } + * ], + * "indices": ["bro", "snort"], + * "query": "*" + * } + */ + @Multiline + public static String groupByIpQuery; + protected static IndexDao dao; protected static InMemoryComponent indexComponent; @@ -572,58 +590,9 @@ public abstract class SearchIntegrationTest { } // getColumnMetadata with multiple indices { - Map<String, Map<String, FieldType>> fieldTypes = dao.getColumnMetadata(Arrays.asList("bro", "snort")); - Assert.assertEquals(2, fieldTypes.size()); - Map<String, FieldType> broTypes = fieldTypes.get("bro"); - Assert.assertEquals(12, broTypes.size()); - Assert.assertEquals(FieldType.STRING, broTypes.get("source:type")); - Assert.assertEquals(FieldType.IP, broTypes.get("ip_src_addr")); - Assert.assertEquals(FieldType.INTEGER, broTypes.get("ip_src_port")); - Assert.assertEquals(FieldType.LONG, broTypes.get("long_field")); - Assert.assertEquals(FieldType.DATE, broTypes.get("timestamp")); - Assert.assertEquals(FieldType.FLOAT, broTypes.get("latitude")); - Assert.assertEquals(FieldType.DOUBLE, broTypes.get("score")); - Assert.assertEquals(FieldType.BOOLEAN, broTypes.get("is_alert")); - Assert.assertEquals(FieldType.OTHER, broTypes.get("location_point")); - Assert.assertEquals(FieldType.STRING, broTypes.get("bro_field")); - Assert.assertEquals(FieldType.STRING, broTypes.get("duplicate_name_field")); - Assert.assertEquals(FieldType.STRING, broTypes.get("guid")); - Map<String, FieldType> snortTypes = fieldTypes.get("snort"); - Assert.assertEquals(12, snortTypes.size()); - Assert.assertEquals(FieldType.STRING, snortTypes.get("source:type")); - Assert.assertEquals(FieldType.IP, snortTypes.get("ip_src_addr")); - Assert.assertEquals(FieldType.INTEGER, snortTypes.get("ip_src_port")); - Assert.assertEquals(FieldType.LONG, snortTypes.get("long_field")); - Assert.assertEquals(FieldType.DATE, snortTypes.get("timestamp")); - Assert.assertEquals(FieldType.FLOAT, snortTypes.get("latitude")); - Assert.assertEquals(FieldType.DOUBLE, snortTypes.get("score")); - Assert.assertEquals(FieldType.BOOLEAN, snortTypes.get("is_alert")); - Assert.assertEquals(FieldType.OTHER, snortTypes.get("location_point")); - Assert.assertEquals(FieldType.INTEGER, snortTypes.get("snort_field")); - Assert.assertEquals(FieldType.INTEGER, snortTypes.get("duplicate_name_field")); - Assert.assertEquals(FieldType.STRING, broTypes.get("guid")); - } - // getColumnMetadata with only bro - { - Map<String, Map<String, FieldType>> fieldTypes = dao.getColumnMetadata(Collections.singletonList("bro")); - Assert.assertEquals(1, fieldTypes.size()); - Map<String, FieldType> broTypes = fieldTypes.get("bro"); - Assert.assertEquals(12, broTypes.size()); - Assert.assertEquals(FieldType.STRING, broTypes.get("bro_field")); - } - // getColumnMetadata with only snort - { - Map<String, Map<String, FieldType>> fieldTypes = dao.getColumnMetadata(Collections.singletonList("snort")); - Assert.assertEquals(1, fieldTypes.size()); - Map<String, FieldType> snortTypes = fieldTypes.get("snort"); - Assert.assertEquals(12, snortTypes.size()); - Assert.assertEquals(FieldType.INTEGER, snortTypes.get("snort_field")); - } - // getCommonColumnMetadata with multiple Indices - { - Map<String, FieldType> fieldTypes = dao.getCommonColumnMetadata(Arrays.asList("bro", "snort")); - // Should only return fields in both - Assert.assertEquals(10, fieldTypes.size()); + Map<String, FieldType> fieldTypes = dao.getColumnMetadata(Arrays.asList("bro", "snort")); + Assert.assertEquals(13, fieldTypes.size()); + Assert.assertEquals(FieldType.STRING, fieldTypes.get("guid")); Assert.assertEquals(FieldType.STRING, fieldTypes.get("source:type")); Assert.assertEquals(FieldType.IP, fieldTypes.get("ip_src_addr")); Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("ip_src_port")); @@ -633,21 +602,26 @@ public abstract class SearchIntegrationTest { Assert.assertEquals(FieldType.DOUBLE, fieldTypes.get("score")); Assert.assertEquals(FieldType.BOOLEAN, fieldTypes.get("is_alert")); Assert.assertEquals(FieldType.OTHER, fieldTypes.get("location_point")); - Assert.assertEquals(FieldType.STRING, fieldTypes.get("guid")); + Assert.assertEquals(FieldType.STRING, fieldTypes.get("bro_field")); + Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("snort_field")); + Assert.assertEquals(FieldType.OTHER, fieldTypes.get("duplicate_name_field")); } - // getCommonColumnMetadata with only bro + // getColumnMetadata with only bro { - Map<String, FieldType> fieldTypes = dao.getCommonColumnMetadata(Collections.singletonList("bro")); + Map<String, FieldType> fieldTypes = dao.getColumnMetadata(Collections.singletonList("bro")); Assert.assertEquals(12, fieldTypes.size()); Assert.assertEquals(FieldType.STRING, fieldTypes.get("bro_field")); - Assert.assertEquals(FieldType.STRING, fieldTypes.get("duplicate_name_field")); } - // getCommonColumnMetadata with only snort + // getColumnMetadata with only snort { - Map<String, FieldType> fieldTypes = dao.getCommonColumnMetadata(Collections.singletonList("snort")); + Map<String, FieldType> fieldTypes = dao.getColumnMetadata(Collections.singletonList("snort")); Assert.assertEquals(12, fieldTypes.size()); Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("snort_field")); - Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("duplicate_name_field")); + } + // getColumnMetadata with an index that doesn't exist + { + Map<String, FieldType> fieldTypes = dao.getColumnMetadata(Collections.singletonList("someindex")); + Assert.assertEquals(0, fieldTypes.size()); } //Fields query { @@ -701,7 +675,6 @@ public abstract class SearchIntegrationTest { List<GroupResult> trueLatitudeGroups = trueGroup.getGroupResults(); Assert.assertEquals(2, trueLatitudeGroups.size()); - // isAlert == true && latitude == 48.5839 group GroupResult trueLatitudeGroup2 = trueLatitudeGroups.get(0); Assert.assertEquals(48.5839, Double.parseDouble(trueLatitudeGroup2.getKey()), 0.00001); @@ -823,6 +796,32 @@ public abstract class SearchIntegrationTest { Assert.assertEquals("Could not execute search", ise.getMessage()); } } + //Group by IP query + { + { + GroupRequest request = JSONUtils.INSTANCE.load(groupByIpQuery, GroupRequest.class); + GroupResponse response = dao.group(request); + + // expect only 1 group for 'ip_src_addr' + Assert.assertEquals("ip_src_addr", response.getGroupedBy()); + + // there are 8 different 'ip_src_addr' values + List<GroupResult> groups = response.getGroupResults(); + Assert.assertEquals(8, groups.size()); + + // expect dotted-decimal notation in descending order + Assert.assertEquals("192.168.1.8", groups.get(0).getKey()); + Assert.assertEquals("192.168.1.7", groups.get(1).getKey()); + Assert.assertEquals("192.168.1.6", groups.get(2).getKey()); + Assert.assertEquals("192.168.1.5", groups.get(3).getKey()); + Assert.assertEquals("192.168.1.4", groups.get(4).getKey()); + Assert.assertEquals("192.168.1.3", groups.get(5).getKey()); + Assert.assertEquals("192.168.1.2", groups.get(6).getKey()); + Assert.assertEquals("192.168.1.1", groups.get(7).getKey()); + } + + + } } @AfterClass
