METRON-1385 Missing "properties" in index template causes ElasticsearchColumnMetadataDao.getColumnMetadata to fail (merrimanr) closes apache/metron#886
Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/91080727 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/91080727 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/91080727 Branch: refs/heads/feature/METRON-1211-extensions-parsers-gradual Commit: 9108072756b6ffeedade985d3cd52ef7338cd61a Parents: 3381b85 Author: merrimanr <[email protected]> Authored: Mon Jan 8 08:18:30 2018 -0600 Committer: merrimanr <[email protected]> Committed: Mon Jan 8 08:18:30 2018 -0600 ---------------------------------------------------------------------- .../dao/ElasticsearchColumnMetadataDao.java | 56 ++++++++++---------- .../ElasticsearchSearchIntegrationTest.java | 19 ++++++- 2 files changed, 47 insertions(+), 28 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/91080727/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchColumnMetadataDao.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchColumnMetadataDao.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchColumnMetadataDao.java index 8e210b4..9f21994 100644 --- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchColumnMetadataDao.java +++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchColumnMetadataDao.java @@ -95,33 +95,35 @@ public class ElasticsearchColumnMetadataDao implements ColumnMetadataDao { Iterator<String> mappingIterator = mapping.keysIt(); while (mappingIterator.hasNext()) { MappingMetaData mappingMetaData = mapping.get(mappingIterator.next()); - Map<String, Map<String, String>> map = (Map<String, Map<String, String>>) mappingMetaData - .getSourceAsMap().get("properties"); - - // for each field in the mapping - for (String field : map.keySet()) { - if (!fieldBlackList.contains(field)) { - FieldType type = toFieldType(map.get(field).get("type")); - - if(!indexColumnMetadata.containsKey(field)) { - indexColumnMetadata.put(field, type); - - // record the last index in which a field exists, to be able to print helpful error message on type mismatch - previousIndices.put(field, indexName); - - } else { - FieldType previousType = indexColumnMetadata.get(field); - if (!type.equals(previousType)) { - String previousIndexName = previousIndices.get(field); - LOG.error(String.format( - "Field type mismatch: %s.%s has type %s while %s.%s has type %s. Defaulting type to %s.", - indexName, field, type.getFieldType(), - previousIndexName, field, previousType.getFieldType(), - FieldType.OTHER.getFieldType())); - indexColumnMetadata.put(field, FieldType.OTHER); - - // the field is defined in multiple indices with different types; ignore the field as type has been set to OTHER - fieldBlackList.add(field); + Map<String, Object> sourceAsMap = mappingMetaData.getSourceAsMap(); + if (sourceAsMap.containsKey("properties")) { + Map<String, Map<String, String>> map = (Map<String, Map<String, String>>) sourceAsMap.get("properties"); + + // for each field in the mapping + for (String field : map.keySet()) { + if (!fieldBlackList.contains(field)) { + FieldType type = toFieldType(map.get(field).get("type")); + + if(!indexColumnMetadata.containsKey(field)) { + indexColumnMetadata.put(field, type); + + // record the last index in which a field exists, to be able to print helpful error message on type mismatch + previousIndices.put(field, indexName); + + } else { + FieldType previousType = indexColumnMetadata.get(field); + if (!type.equals(previousType)) { + String previousIndexName = previousIndices.get(field); + LOG.error(String.format( + "Field type mismatch: %s.%s has type %s while %s.%s has type %s. Defaulting type to %s.", + indexName, field, type.getFieldType(), + previousIndexName, field, previousType.getFieldType(), + FieldType.OTHER.getFieldType())); + indexColumnMetadata.put(field, FieldType.OTHER); + + // the field is defined in multiple indices with different types; ignore the field as type has been set to OTHER + fieldBlackList.add(field); + } } } } http://git-wip-us.apache.org/repos/asf/metron/blob/91080727/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java index 3d50e99..eac4851 100644 --- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java +++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java @@ -151,6 +151,23 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest { /** * { + * "bro_doc_default": { + * "dynamic_templates": [{ + * "strings": { + * "match_mapping_type": "string", + * "mapping": { + * "type": "text" + * } + * } + * }] + * } + * } + */ + @Multiline + private static String broDefaultStringMappings; + + /** + * { * "metaalert_doc": { * "properties": { * "source:type": { "type": "string" }, @@ -196,7 +213,7 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest { throws ParseException, IOException, ExecutionException, InterruptedException { ElasticSearchComponent es = (ElasticSearchComponent)indexComponent; es.getClient().admin().indices().prepareCreate("bro_index_2017.01.01.01") - .addMapping("bro_doc", broTypeMappings).get(); + .addMapping("bro_doc", broTypeMappings).addMapping("bro_doc_default", broDefaultStringMappings).get(); es.getClient().admin().indices().prepareCreate("snort_index_2017.01.01.02") .addMapping("snort_doc", snortTypeMappings).get(); es.getClient().admin().indices().prepareCreate("metaalert_index")
