METRON-1540 Solr Integration tests should use actual schemas (justinleet) closes apache/metron#1005
Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/832296f0 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/832296f0 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/832296f0 Branch: refs/heads/feature/METRON-1554-pcap-query-panel Commit: 832296f0d4615bf95aed1c0742362c6127bd733a Parents: d0a4e4c Author: justinleet <[email protected]> Authored: Wed May 2 10:10:27 2018 -0400 Committer: leet <[email protected]> Committed: Wed May 2 10:10:27 2018 -0400 ---------------------------------------------------------------------- .../ElasticsearchSearchIntegrationTest.java | 22 +- .../indexing/dao/SearchIntegrationTest.java | 55 +- .../metron/solr/dao/SolrColumnMetadataDao.java | 33 +- .../apache/metron/solr/dao/SolrSearchDao.java | 3 +- .../integration/SolrSearchIntegrationTest.java | 138 +- .../resources/config/bro/conf/managed-schema | 50 - .../resources/config/bro/conf/solrconfig.xml | 1601 ------------------ .../resources/config/snort/conf/managed-schema | 51 - .../resources/config/snort/conf/solrconfig.xml | 1601 ------------------ 9 files changed, 178 insertions(+), 3376 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/832296f0/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java index 5569c54..bb28abb 100644 --- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java +++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java @@ -29,6 +29,7 @@ import java.util.HashMap; import java.util.List; import java.util.concurrent.ExecutionException; import org.adrianwalker.multilinestring.Multiline; +import org.apache.metron.common.Constants; import org.apache.metron.common.utils.JSONUtils; import org.apache.metron.elasticsearch.dao.ElasticsearchDao; import org.apache.metron.elasticsearch.integration.components.ElasticSearchComponent; @@ -109,7 +110,7 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest { * "type": "text", * "fielddata" : "true" * }, - * "duplicate_name_field": { + * "ttl": { * "type": "text", * "fielddata" : "true" * }, @@ -162,7 +163,7 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest { * "snort_field": { * "type": "integer" * }, - * "duplicate_name_field": { + * "ttl": { * "type": "integer" * }, * "alert": { @@ -275,7 +276,7 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest { Map<String, FieldType> fieldTypes = dao.getColumnMetadata(Collections.singletonList("bro")); Assert.assertEquals(13, fieldTypes.size()); Assert.assertEquals(FieldType.TEXT, fieldTypes.get("bro_field")); - Assert.assertEquals(FieldType.TEXT, fieldTypes.get("duplicate_name_field")); + Assert.assertEquals(FieldType.TEXT, fieldTypes.get("ttl")); Assert.assertEquals(FieldType.KEYWORD, fieldTypes.get("guid")); Assert.assertEquals(FieldType.TEXT, fieldTypes.get("source:type")); Assert.assertEquals(FieldType.IP, fieldTypes.get("ip_src_addr")); @@ -287,7 +288,7 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest { Assert.assertEquals(FieldType.BOOLEAN, fieldTypes.get("is_alert")); Assert.assertEquals(FieldType.OTHER, fieldTypes.get("location_point")); Assert.assertEquals(FieldType.TEXT, fieldTypes.get("bro_field")); - Assert.assertEquals(FieldType.TEXT, fieldTypes.get("duplicate_name_field")); + Assert.assertEquals(FieldType.TEXT, fieldTypes.get("ttl")); Assert.assertEquals(FieldType.OTHER, fieldTypes.get("alert")); } // getColumnMetadata with only snort @@ -295,7 +296,7 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest { Map<String, FieldType> fieldTypes = dao.getColumnMetadata(Collections.singletonList("snort")); Assert.assertEquals(14, fieldTypes.size()); Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("snort_field")); - Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("duplicate_name_field")); + Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("ttl")); Assert.assertEquals(FieldType.KEYWORD, fieldTypes.get("guid")); Assert.assertEquals(FieldType.TEXT, fieldTypes.get("source:type")); Assert.assertEquals(FieldType.IP, fieldTypes.get("ip_src_addr")); @@ -306,7 +307,7 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest { Assert.assertEquals(FieldType.DOUBLE, fieldTypes.get("score")); Assert.assertEquals(FieldType.BOOLEAN, fieldTypes.get("is_alert")); Assert.assertEquals(FieldType.OTHER, fieldTypes.get("location_point")); - Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("duplicate_name_field")); + Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("ttl")); Assert.assertEquals(FieldType.OTHER, fieldTypes.get("alert")); } } @@ -328,7 +329,7 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest { Assert.assertEquals(FieldType.TEXT, fieldTypes.get("bro_field")); Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("snort_field")); //NOTE: This is because the field is in both bro and snort and they have different types. - Assert.assertEquals(FieldType.OTHER, fieldTypes.get("duplicate_name_field")); + Assert.assertEquals(FieldType.OTHER, fieldTypes.get("ttl")); Assert.assertEquals(FieldType.FLOAT, fieldTypes.get("threat:triage:score")); Assert.assertEquals(FieldType.OTHER, fieldTypes.get("alert")); } @@ -349,6 +350,11 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest { Assert.assertEquals(1, response.getTotal()); List<SearchResult> results = response.getResults(); Assert.assertEquals("bro", results.get(0).getSource().get("source:type")); - Assert.assertEquals("data 1", results.get(0).getSource().get("duplicate_name_field")); + Assert.assertEquals("data 1", results.get(0).getSource().get("ttl")); + } + + @Override + protected String getSourceTypeField() { + return Constants.SENSOR_TYPE.replace('.', ':'); } } http://git-wip-us.apache.org/repos/asf/metron/blob/832296f0/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java index 83046b8..56406f4 100644 --- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java +++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java @@ -40,6 +40,7 @@ import org.apache.metron.integration.InMemoryComponent; import org.junit.AfterClass; import org.junit.Assert; import org.junit.Before; +import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; @@ -47,11 +48,11 @@ import org.junit.rules.ExpectedException; public abstract class SearchIntegrationTest { /** * [ - * {"source:type": "bro", "ip_src_addr":"192.168.1.1", "ip_src_port": 8010, "long_field": 10000, "timestamp":1, "latitude": 48.5839, "score": 10.0, "is_alert":true, "location_point": "48.5839,7.7455", "bro_field": "bro data 1", "duplicate_name_field": "data 1", "guid":"bro_1"}, - * {"source:type": "bro", "ip_src_addr":"192.168.1.2", "ip_src_port": 8009, "long_field": 20000, "timestamp":2, "latitude": 48.0001, "score": 50.0, "is_alert":false, "location_point": "48.5839,7.7455", "bro_field": "bro data 2", "duplicate_name_field": "data 2", "guid":"bro_2"}, - * {"source:type": "bro", "ip_src_addr":"192.168.1.3", "ip_src_port": 8008, "long_field": 10000, "timestamp":3, "latitude": 48.5839, "score": 20.0, "is_alert":true, "location_point": "50.0,7.7455", "bro_field": "bro data 3", "duplicate_name_field": "data 3", "guid":"bro_3"}, - * {"source:type": "bro", "ip_src_addr":"192.168.1.4", "ip_src_port": 8007, "long_field": 10000, "timestamp":4, "latitude": 48.5839, "score": 10.0, "is_alert":true, "location_point": "48.5839,7.7455", "bro_field": "bro data 4", "duplicate_name_field": "data 4", "guid":"bro_4"}, - * {"source:type": "bro", "ip_src_addr":"192.168.1.5", "ip_src_port": 8006, "long_field": 10000, "timestamp":5, "latitude": 48.5839, "score": 98.0, "is_alert":true, "location_point": "48.5839,7.7455", "bro_field": "bro data 5", "duplicate_name_field": "data 5", "guid":"bro_5"} + * {"source:type": "bro", "ip_src_addr":"192.168.1.1", "ip_src_port": 8010, "long_field": 10000, "timestamp":1, "latitude": 48.5839, "score": 10.0, "is_alert":true, "location_point": "48.5839,7.7455", "bro_field": "bro data 1", "ttl": "data 1", "guid":"bro_1"}, + * {"source:type": "bro", "ip_src_addr":"192.168.1.2", "ip_src_port": 8009, "long_field": 20000, "timestamp":2, "latitude": 48.0001, "score": 50.0, "is_alert":false, "location_point": "48.5839,7.7455", "bro_field": "bro data 2", "ttl": "data 2", "guid":"bro_2"}, + * {"source:type": "bro", "ip_src_addr":"192.168.1.3", "ip_src_port": 8008, "long_field": 10000, "timestamp":3, "latitude": 48.5839, "score": 20.0, "is_alert":true, "location_point": "50.0,7.7455", "bro_field": "bro data 3", "ttl": "data 3", "guid":"bro_3"}, + * {"source:type": "bro", "ip_src_addr":"192.168.1.4", "ip_src_port": 8007, "long_field": 10000, "timestamp":4, "latitude": 48.5839, "score": 10.0, "is_alert":true, "location_point": "48.5839,7.7455", "bro_field": "bro data 4", "ttl": "data 4", "guid":"bro_4"}, + * {"source:type": "bro", "ip_src_addr":"192.168.1.5", "ip_src_port": 8006, "long_field": 10000, "timestamp":5, "latitude": 48.5839, "score": 98.0, "is_alert":true, "location_point": "48.5839,7.7455", "bro_field": "bro data 5", "ttl": "data 5", "guid":"bro_5"} * ] */ @Multiline @@ -59,11 +60,11 @@ public abstract class SearchIntegrationTest { /** * [ - * {"source:type": "snort", "ip_src_addr":"192.168.1.6", "ip_src_port": 8005, "long_field": 10000, "timestamp":6, "latitude": 48.5839, "score": 50.0, "is_alert":false, "location_point": "50.0,7.7455", "snort_field": 10, "duplicate_name_field": 1, "guid":"snort_1", "threat:triage:score":10.0}, - * {"source:type": "snort", "ip_src_addr":"192.168.1.1", "ip_src_port": 8004, "long_field": 10000, "timestamp":7, "latitude": 48.5839, "score": 10.0, "is_alert":true, "location_point": "48.5839,7.7455", "snort_field": 20, "duplicate_name_field": 2, "guid":"snort_2", "threat:triage:score":20.0}, - * {"source:type": "snort", "ip_src_addr":"192.168.1.7", "ip_src_port": 8003, "long_field": 10000, "timestamp":8, "latitude": 48.5839, "score": 20.0, "is_alert":false, "location_point": "48.5839,7.7455", "snort_field": 30, "duplicate_name_field": 3, "guid":"snort_3"}, - * {"source:type": "snort", "ip_src_addr":"192.168.1.1", "ip_src_port": 8002, "long_field": 20000, "timestamp":9, "latitude": 48.0001, "score": 50.0, "is_alert":true, "location_point": "48.5839,7.7455", "snort_field": 40, "duplicate_name_field": 4, "guid":"snort_4"}, - * {"source:type": "snort", "ip_src_addr":"192.168.1.8", "ip_src_port": 8001, "long_field": 10000, "timestamp":10, "latitude": 48.5839, "score": 10.0, "is_alert":false, "location_point": "48.5839,7.7455", "snort_field": 50, "duplicate_name_field": 5, "guid":"snort_5"} + * {"source:type": "snort", "ip_src_addr":"192.168.1.6", "ip_src_port": 8005, "long_field": 10000, "timestamp":6, "latitude": 48.5839, "score": 50.0, "is_alert":false, "location_point": "50.0,7.7455", "snort_field": 10, "ttl": 1, "guid":"snort_1", "threat:triage:score":10.0}, + * {"source:type": "snort", "ip_src_addr":"192.168.1.1", "ip_src_port": 8004, "long_field": 10000, "timestamp":7, "latitude": 48.5839, "score": 10.0, "is_alert":true, "location_point": "48.5839,7.7455", "snort_field": 20, "ttl": 2, "guid":"snort_2", "threat:triage:score":20.0}, + * {"source:type": "snort", "ip_src_addr":"192.168.1.7", "ip_src_port": 8003, "long_field": 10000, "timestamp":8, "latitude": 48.5839, "score": 20.0, "is_alert":false, "location_point": "48.5839,7.7455", "snort_field": 30, "ttl": 3, "guid":"snort_3"}, + * {"source:type": "snort", "ip_src_addr":"192.168.1.1", "ip_src_port": 8002, "long_field": 20000, "timestamp":9, "latitude": 48.0001, "score": 50.0, "is_alert":true, "location_point": "48.5839,7.7455", "snort_field": 40, "ttl": 4, "guid":"snort_4"}, + * {"source:type": "snort", "ip_src_addr":"192.168.1.8", "ip_src_port": 8001, "long_field": 10000, "timestamp":10, "latitude": 48.5839, "score": 10.0, "is_alert":false, "location_point": "48.5839,7.7455", "snort_field": 50, "ttl": 5, "guid":"snort_5"} * ] */ @Multiline @@ -234,7 +235,7 @@ public abstract class SearchIntegrationTest { * } */ @Multiline - public static String facetQuery; + public static String facetQueryRaw; /** * { @@ -291,7 +292,7 @@ public abstract class SearchIntegrationTest { /** * { - * "facetFields": ["duplicate_name_field"], + * "facetFields": ["ttl"], * "indices": ["bro", "snort"], * "query": "*:*", * "from": 0, @@ -457,7 +458,7 @@ public abstract class SearchIntegrationTest { /** * { * "indices": ["bro", "snort"], - * "query": "duplicate_name_field:\"data 1\"", + * "query": "ttl:\"data 1\"", * "from": 0, * "size": 10, * "sort": [ @@ -494,11 +495,11 @@ public abstract class SearchIntegrationTest { List<SearchResult> results = response.getResults(); Assert.assertEquals(10, results.size()); for(int i = 0;i < 5;++i) { - Assert.assertEquals("snort", results.get(i).getSource().get("source:type")); + Assert.assertEquals("snort", results.get(i).getSource().get(getSourceTypeField())); Assert.assertEquals(10 - i + "", results.get(i).getSource().get("timestamp").toString()); } for (int i = 5; i < 10; ++i) { - Assert.assertEquals("bro", results.get(i).getSource().get("source:type")); + Assert.assertEquals("bro", results.get(i).getSource().get(getSourceTypeField())); Assert.assertEquals(10 - i + "", results.get(i).getSource().get("timestamp").toString()); } } @@ -509,7 +510,7 @@ public abstract class SearchIntegrationTest { Optional<Map<String, Object>> response = dao.getLatestResult(request); Assert.assertTrue(response.isPresent()); Map<String, Object> doc = response.get(); - Assert.assertEquals("bro", doc.get("source:type")); + Assert.assertEquals("bro", doc.get(getSourceTypeField())); Assert.assertEquals("3", doc.get("timestamp").toString()); } @@ -524,8 +525,8 @@ public abstract class SearchIntegrationTest { Assert.assertEquals(2, docs.size()); Assert.assertTrue(docs.keySet().contains("bro_1")); Assert.assertTrue(docs.keySet().contains("snort_2")); - Assert.assertEquals("bro", docs.get("bro_1").getDocument().get("source:type")); - Assert.assertEquals("snort", docs.get("snort_2").getDocument().get("source:type")); + Assert.assertEquals("bro", docs.get("bro_1").getDocument().get(getSourceTypeField())); + Assert.assertEquals("snort", docs.get("snort_2").getDocument().get(getSourceTypeField())); } @Test @@ -534,11 +535,11 @@ public abstract class SearchIntegrationTest { SearchResponse response = dao.search(request); Assert.assertEquals(3, response.getTotal()); List<SearchResult> results = response.getResults(); - Assert.assertEquals("snort", results.get(0).getSource().get("source:type")); + Assert.assertEquals("snort", results.get(0).getSource().get(getSourceTypeField())); Assert.assertEquals("9", results.get(0).getSource().get("timestamp").toString()); - Assert.assertEquals("snort", results.get(1).getSource().get("source:type")); + Assert.assertEquals("snort", results.get(1).getSource().get(getSourceTypeField())); Assert.assertEquals("7", results.get(1).getSource().get("timestamp").toString()); - Assert.assertEquals("bro", results.get(2).getSource().get("source:type")); + Assert.assertEquals("bro", results.get(2).getSource().get(getSourceTypeField())); Assert.assertEquals("1", results.get(2).getSource().get("timestamp").toString()); } @@ -596,11 +597,11 @@ public abstract class SearchIntegrationTest { Assert.assertEquals(10, response.getTotal()); List<SearchResult> results = response.getResults(); Assert.assertEquals(3, results.size()); - Assert.assertEquals("snort", results.get(0).getSource().get("source:type")); + Assert.assertEquals("snort", results.get(0).getSource().get(getSourceTypeField())); Assert.assertEquals("6", results.get(0).getSource().get("timestamp").toString()); - Assert.assertEquals("bro", results.get(1).getSource().get("source:type")); + Assert.assertEquals("bro", results.get(1).getSource().get(getSourceTypeField())); Assert.assertEquals("5", results.get(1).getSource().get("timestamp").toString()); - Assert.assertEquals("bro", results.get(2).getSource().get("source:type")); + Assert.assertEquals("bro", results.get(2).getSource().get(getSourceTypeField())); Assert.assertEquals("4", results.get(2).getSource().get("timestamp").toString()); } @@ -611,19 +612,20 @@ public abstract class SearchIntegrationTest { Assert.assertEquals(5, response.getTotal()); List<SearchResult> results = response.getResults(); for (int i = 5, j = 0; i > 0; i--, j++) { - Assert.assertEquals("bro", results.get(j).getSource().get("source:type")); + Assert.assertEquals("bro", results.get(j).getSource().get(getSourceTypeField())); Assert.assertEquals(i + "", results.get(j).getSource().get("timestamp").toString()); } } @Test public void facet_query_yields_field_types() throws Exception { + String facetQuery = facetQueryRaw.replace("source:type", getSourceTypeField()); SearchRequest request = JSONUtils.INSTANCE.load(facetQuery, SearchRequest.class); SearchResponse response = dao.search(request); Assert.assertEquals(10, response.getTotal()); Map<String, Map<String, Long>> facetCounts = response.getFacetCounts(); Assert.assertEquals(8, facetCounts.size()); - Map<String, Long> sourceTypeCounts = facetCounts.get("source:type"); + Map<String, Long> sourceTypeCounts = facetCounts.get(getSourceTypeField()); Assert.assertEquals(2, sourceTypeCounts.size()); Assert.assertEquals(new Long(5), sourceTypeCounts.get("bro")); Assert.assertEquals(new Long(5), sourceTypeCounts.get("snort")); @@ -951,4 +953,5 @@ public abstract class SearchIntegrationTest { protected abstract IndexDao createDao() throws Exception; protected abstract InMemoryComponent startIndex() throws Exception; protected abstract void loadTestData() throws Exception; + protected abstract String getSourceTypeField(); } http://git-wip-us.apache.org/repos/asf/metron/blob/832296f0/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrColumnMetadataDao.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrColumnMetadataDao.java b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrColumnMetadataDao.java index 61f0209..1e074a9 100644 --- a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrColumnMetadataDao.java +++ b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrColumnMetadataDao.java @@ -20,17 +20,25 @@ package org.apache.metron.solr.dao; import com.google.common.collect.Sets; import java.io.IOException; import java.lang.invoke.MethodHandles; +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Set; import org.apache.metron.indexing.dao.ColumnMetadataDao; import org.apache.metron.indexing.dao.search.FieldType; +import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.impl.CloudSolrClient; +import org.apache.solr.client.solrj.request.LukeRequest; import org.apache.solr.client.solrj.request.schema.SchemaRequest; +import org.apache.solr.client.solrj.request.schema.SchemaRequest.DynamicFields; +import org.apache.solr.client.solrj.response.LukeResponse; +import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.client.solrj.response.schema.SchemaRepresentation; +import org.apache.solr.client.solrj.response.schema.SchemaResponse.DynamicFieldsResponse; import org.apache.solr.common.SolrException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -49,6 +57,7 @@ public class SolrColumnMetadataDao implements ColumnMetadataDao { fieldTypeMap.put("pfloat", FieldType.FLOAT); fieldTypeMap.put("pdouble", FieldType.DOUBLE); fieldTypeMap.put("boolean", FieldType.BOOLEAN); + fieldTypeMap.put("ip", FieldType.IP); solrTypeMap = Collections.unmodifiableMap(fieldTypeMap); } @@ -104,12 +113,30 @@ public class SolrColumnMetadataDao implements ColumnMetadataDao { return indexColumnMetadata; } - protected List<Map<String, Object>> getIndexFields(String index) throws IOException, SolrServerException { + protected List<Map<String, Object>> getIndexFields(String index) + throws IOException, SolrServerException { CloudSolrClient client = new CloudSolrClient.Builder().withZkHost(zkHost).build(); client.setDefaultCollection(index); + + List<Map<String, Object>> indexFields = new ArrayList<>(); + + // Get all the fields in use, including dynamic fields + LukeRequest lukeRequest = new LukeRequest(); + LukeResponse lukeResponse = lukeRequest.process(client); + for (Entry<String, LukeResponse.FieldInfo> field : lukeResponse.getFieldInfo().entrySet()) { + Map<String, Object> fieldData = new HashMap<>(); + fieldData.put("name", field.getValue().getName()); + fieldData.put("type", field.getValue().getType()); + indexFields.add(fieldData); + + } + + // Get all the schema fields SchemaRepresentation schemaRepresentation = new SchemaRequest().process(client) - .getSchemaRepresentation(); - return schemaRepresentation.getFields(); + .getSchemaRepresentation(); + indexFields.addAll(schemaRepresentation.getFields()); + + return indexFields; } /** http://git-wip-us.apache.org/repos/asf/metron/blob/832296f0/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrSearchDao.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrSearchDao.java b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrSearchDao.java index 272b96a..f1b7102 100644 --- a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrSearchDao.java +++ b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrSearchDao.java @@ -47,6 +47,7 @@ import org.apache.solr.client.solrj.response.PivotField; import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrDocumentList; +import org.apache.solr.common.SolrException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -91,7 +92,7 @@ public class SolrSearchDao implements SearchDao { SolrQuery query = buildSearchRequest(searchRequest); QueryResponse response = client.query(query); return buildSearchResponse(searchRequest, response); - } catch (IOException | SolrServerException e) { + } catch (SolrException | IOException | SolrServerException e) { String msg = e.getMessage(); LOG.error(msg, e); throw new InvalidSearchException(msg, e); http://git-wip-us.apache.org/repos/asf/metron/blob/832296f0/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrSearchIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrSearchIntegrationTest.java b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrSearchIntegrationTest.java index a9ce650..92941cd 100644 --- a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrSearchIntegrationTest.java +++ b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrSearchIntegrationTest.java @@ -21,6 +21,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.Map; +import org.apache.metron.common.Constants; import org.apache.metron.common.utils.JSONUtils; import org.apache.metron.indexing.dao.AccessConfig; import org.apache.metron.indexing.dao.IndexDao; @@ -36,12 +37,16 @@ import org.apache.solr.client.solrj.impl.CloudSolrClient; import org.json.simple.JSONArray; import org.json.simple.parser.JSONParser; import org.junit.Assert; +import org.junit.Ignore; import org.junit.Test; public class SolrSearchIntegrationTest extends SearchIntegrationTest { private SolrComponent solrComponent; + private String broData = SearchIntegrationTest.broData.replace("source:type", "source.type"); + private String snortData = SearchIntegrationTest.snortData.replace("source:type", "source.type"); + @Override protected IndexDao createDao() throws Exception { AccessConfig config = new AccessConfig(); @@ -60,9 +65,10 @@ public class SolrSearchIntegrationTest extends SearchIntegrationTest { @Override protected InMemoryComponent startIndex() throws Exception { + solrComponent = new SolrComponent.Builder() - .addCollection("bro", "../metron-solr/src/test/resources/config/bro/conf") - .addCollection("snort", "../metron-solr/src/test/resources/config/snort/conf") + .addCollection("bro", "../metron-solr/src/main/config/schema/bro") + .addCollection("snort", "../metron-solr/src/main/config/schema/snort") .build(); solrComponent.start(); return solrComponent; @@ -83,65 +89,122 @@ public class SolrSearchIntegrationTest extends SearchIntegrationTest { } @Override + @Test public void returns_column_metadata_for_specified_indices() throws Exception { // getColumnMetadata with only bro { Map<String, FieldType> fieldTypes = dao.getColumnMetadata(Collections.singletonList("bro")); - Assert.assertEquals(12, fieldTypes.size()); - Assert.assertEquals(FieldType.TEXT, fieldTypes.get("bro_field")); - Assert.assertEquals(FieldType.TEXT, fieldTypes.get("duplicate_name_field")); + // Don't test all 256, just test a sample of different fields + Assert.assertEquals(261, fieldTypes.size()); + + // Fields present in both with same type Assert.assertEquals(FieldType.TEXT, fieldTypes.get("guid")); - Assert.assertEquals(FieldType.TEXT, fieldTypes.get("source:type")); - Assert.assertEquals(FieldType.TEXT, fieldTypes.get("ip_src_addr")); + Assert.assertEquals(FieldType.TEXT, fieldTypes.get("source.type")); + Assert.assertEquals(FieldType.IP, fieldTypes.get("ip_src_addr")); Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("ip_src_port")); - Assert.assertEquals(FieldType.LONG, fieldTypes.get("long_field")); - Assert.assertEquals(FieldType.LONG, fieldTypes.get("timestamp")); - Assert.assertEquals(FieldType.FLOAT, fieldTypes.get("latitude")); - Assert.assertEquals(FieldType.DOUBLE, fieldTypes.get("score")); Assert.assertEquals(FieldType.BOOLEAN, fieldTypes.get("is_alert")); + + // Bro only field + Assert.assertEquals(FieldType.TEXT, fieldTypes.get("username")); + + // A dynamic field present in both with same type + Assert.assertEquals(FieldType.FLOAT, fieldTypes.get("score")); + + // Dyanamic field present in both with nonstandard types. Assert.assertEquals(FieldType.OTHER, fieldTypes.get("location_point")); - Assert.assertEquals(FieldType.TEXT, fieldTypes.get("bro_field")); - Assert.assertEquals(FieldType.TEXT, fieldTypes.get("duplicate_name_field")); + + // Field with nonstandard type + Assert.assertEquals(FieldType.OTHER, fieldTypes.get("timestamp")); + + // Bro only field in the dynamic catch all + Assert.assertEquals(FieldType.OTHER, fieldTypes.get("bro_field")); + + // A field is in both bro and snort and they have different types. + Assert.assertEquals(FieldType.TEXT, fieldTypes.get("ttl")); + + // Field only present in Snort + Assert.assertEquals(null, fieldTypes.get("dgmlen")); + + // Field that doesn't exist + Assert.assertEquals(null, fieldTypes.get("fake.field")); } // getColumnMetadata with only snort { Map<String, FieldType> fieldTypes = dao.getColumnMetadata(Collections.singletonList("snort")); - Assert.assertEquals(13, fieldTypes.size()); - Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("snort_field")); - Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("duplicate_name_field")); + Assert.assertEquals(31, fieldTypes.size()); + + // Fields present in both with same type Assert.assertEquals(FieldType.TEXT, fieldTypes.get("guid")); - Assert.assertEquals(FieldType.TEXT, fieldTypes.get("source:type")); - Assert.assertEquals(FieldType.TEXT, fieldTypes.get("ip_src_addr")); + Assert.assertEquals(FieldType.TEXT, fieldTypes.get("source.type")); + Assert.assertEquals(FieldType.IP, fieldTypes.get("ip_src_addr")); Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("ip_src_port")); - Assert.assertEquals(FieldType.LONG, fieldTypes.get("long_field")); - Assert.assertEquals(FieldType.LONG, fieldTypes.get("timestamp")); - Assert.assertEquals(FieldType.FLOAT, fieldTypes.get("latitude")); - Assert.assertEquals(FieldType.DOUBLE, fieldTypes.get("score")); Assert.assertEquals(FieldType.BOOLEAN, fieldTypes.get("is_alert")); + + // Snort only field + Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("dgmlen")); + + // A dynamic field present in both with same type + Assert.assertEquals(FieldType.FLOAT, fieldTypes.get("score")); + + // Dyanamic field present in both with nonstandard types. Assert.assertEquals(FieldType.OTHER, fieldTypes.get("location_point")); - Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("duplicate_name_field")); + + // Field with nonstandard type + Assert.assertEquals(FieldType.OTHER, fieldTypes.get("timestamp")); + + // Snort only field in the dynamic catch all + Assert.assertEquals(FieldType.OTHER, fieldTypes.get("snort_field")); + + // A field is in both bro and snort and they have different types. + Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("ttl")); + + // Field only present in Bro + Assert.assertEquals(null, fieldTypes.get("username")); + + // Field that doesn't exist + Assert.assertEquals(null, fieldTypes.get("fake.field")); } } @Override + @Test public void returns_column_data_for_multiple_indices() throws Exception { Map<String, FieldType> fieldTypes = dao.getColumnMetadata(Arrays.asList("bro", "snort")); - Assert.assertEquals(14, fieldTypes.size()); + // Don't test everything, just test a variety of fields, including fields across collections. + + // Fields present in both with same type Assert.assertEquals(FieldType.TEXT, fieldTypes.get("guid")); - Assert.assertEquals(FieldType.TEXT, fieldTypes.get("source:type")); - Assert.assertEquals(FieldType.TEXT, fieldTypes.get("ip_src_addr")); + Assert.assertEquals(FieldType.TEXT, fieldTypes.get("source.type")); + Assert.assertEquals(FieldType.IP, fieldTypes.get("ip_src_addr")); Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("ip_src_port")); - Assert.assertEquals(FieldType.LONG, fieldTypes.get("long_field")); - Assert.assertEquals(FieldType.LONG, fieldTypes.get("timestamp")); - Assert.assertEquals(FieldType.FLOAT, fieldTypes.get("latitude")); - Assert.assertEquals(FieldType.DOUBLE, fieldTypes.get("score")); Assert.assertEquals(FieldType.BOOLEAN, fieldTypes.get("is_alert")); + + // Bro only field + Assert.assertEquals(FieldType.TEXT, fieldTypes.get("username")); + + // Snort only field + Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("dgmlen")); + + // A dynamic field present in both with same type + Assert.assertEquals(FieldType.FLOAT, fieldTypes.get("score")); + + // Dyanamic field present in both with nonstandard types. Assert.assertEquals(FieldType.OTHER, fieldTypes.get("location_point")); - Assert.assertEquals(FieldType.TEXT, fieldTypes.get("bro_field")); - Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("snort_field")); - //NOTE: This is because the field is in both bro and snort and they have different types. - Assert.assertEquals(FieldType.OTHER, fieldTypes.get("duplicate_name_field")); - Assert.assertEquals(FieldType.FLOAT, fieldTypes.get("threat:triage:score")); + + // Field present in both with nonstandard type + Assert.assertEquals(FieldType.OTHER, fieldTypes.get("timestamp")); + + // Bro only field in the dynamic catch all + Assert.assertEquals(FieldType.OTHER, fieldTypes.get("bro_field")); + + // Snort only field in the dynamic catch all + Assert.assertEquals(FieldType.OTHER, fieldTypes.get("snort_field")); + + // A field is in both bro and snort and they have different types. + Assert.assertEquals(FieldType.OTHER, fieldTypes.get("ttl")); + + // Field that doesn't exist + Assert.assertEquals(null, fieldTypes.get("fake.field")); } @Test @@ -150,4 +213,9 @@ public class SolrSearchIntegrationTest extends SearchIntegrationTest { SearchRequest request = JSONUtils.INSTANCE.load(differentTypeFilterQuery, SearchRequest.class); SearchResponse response = dao.search(request); } + + @Override + protected String getSourceTypeField() { + return Constants.SENSOR_TYPE; + } } http://git-wip-us.apache.org/repos/asf/metron/blob/832296f0/metron-platform/metron-solr/src/test/resources/config/bro/conf/managed-schema ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/test/resources/config/bro/conf/managed-schema b/metron-platform/metron-solr/src/test/resources/config/bro/conf/managed-schema deleted file mode 100644 index 3bcb2bc..0000000 --- a/metron-platform/metron-solr/src/test/resources/config/bro/conf/managed-schema +++ /dev/null @@ -1,50 +0,0 @@ -<?xml version="1.0" encoding="UTF-8" ?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> -<schema name="example" version="1.6"> - <field name="_version_" type="plong" indexed="false" stored="false"/> - <field name="_root_" type="string" indexed="true" stored="false" docValues="false" /> - - <field name="guid" type="string" indexed="true" stored="true" required="true" multiValued="false" /> - - <field name="source:type" type="string" indexed="true" stored="true"/> - <field name="ip_src_addr" type="string" indexed="true" stored="true" /> - <field name="ip_src_port" type="pint" indexed="true" stored="true"/> - <field name="long_field" type="plong" indexed="true" stored="true" /> - <field name="timestamp" type="plong" indexed="true" stored="true" /> - <field name="latitude" type="pfloat" indexed="true" stored="true" /> - <field name="score" type="pdouble" indexed="true" stored="true" /> - <field name="is_alert" type="boolean" indexed="true" stored="true"/> - <field name="location_point" type="location" indexed="true" stored="true"/> - <field name="bro_field" type="string" indexed="true" stored="true" /> - <field name="duplicate_name_field" type="string" indexed="true" stored="true" /> - - <dynamicField name="*" type="ignored" multiValued="false" docValues="true"/> - - <uniqueKey>guid</uniqueKey> - - - <fieldType name="string" class="solr.StrField" sortMissingLast="true" /> - <fieldType name="boolean" class="solr.BoolField" sortMissingLast="true"/> - <fieldType name="pint" class="solr.IntPointField" docValues="true"/> - <fieldType name="pfloat" class="solr.FloatPointField" docValues="true"/> - <fieldType name="plong" class="solr.LongPointField" docValues="true"/> - <fieldType name="pdouble" class="solr.DoublePointField" docValues="true"/> - <fieldType name="location" class="solr.LatLonPointSpatialField" docValues="true"/> - <fieldType name="ignored" stored="false" indexed="false" multiValued="true" class="solr.StrField" /> - -</schema>
