Merge remote-tracking branch 'upstream/feature/METRON-1344-test-infrastructure' into feature/METRON-1344-test-infrastructure
Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/fc4ce010 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/fc4ce010 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/fc4ce010 Branch: refs/heads/feature/METRON-1344-test-infrastructure Commit: fc4ce01047345b065214089b466babbdc1d5d30d Parents: e265b36 ebc3258 Author: merrimanr <merrim...@gmail.com> Authored: Thu Feb 22 14:12:16 2018 -0600 Committer: merrimanr <merrim...@gmail.com> Committed: Thu Feb 22 14:12:16 2018 -0600 ---------------------------------------------------------------------- .travis.yml | 19 +- metron-contrib/metron-docker-e2e/.gitignore | 3 + .../compose/docker-compose.yml | 55 +++++ .../compose/metron-centos/Dockerfile | 19 ++ .../compose/metron-rest/Dockerfile | 34 +++ .../compose/metron-rest/bin/start.sh | 23 ++ .../metron-rest/config/application-docker.yml | 62 ++++++ .../metron-rest/config/zookeeper/global.json | 8 + .../compose/metron-ui/Dockerfile | 34 +++ .../compose/metron-ui/bin/start.sh | 20 ++ .../compose/metron-ui/config/alerts_ui.yml | 21 ++ metron-contrib/metron-docker-e2e/conf/.env | 3 + metron-contrib/metron-docker-e2e/pom.xml | 217 +++++++++++++++++++ .../metron-docker-e2e/scripts/wait_for_rest.sh | 29 +++ metron-contrib/pom.xml | 1 + .../configure-table/configure-table.e2e-spec.ts | 25 +-- .../metron-alerts/e2e/login/login.e2e-spec.ts | 3 +- .../metron-alerts/e2e/login/login.po.ts | 2 +- .../metron-alerts/e2e/utils/e2e_util.ts | 29 ++- metron-interface/metron-alerts/pom.xml | 46 ++++ .../metron-alerts/protractor.conf.js | 21 +- .../ElasticsearchSearchIntegrationTest.java | 86 +++++--- .../indexing/dao/SearchIntegrationTest.java | 90 +++++--- 23 files changed, 742 insertions(+), 108 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/fc4ce010/.travis.yml ---------------------------------------------------------------------- diff --cc .travis.yml index 71d7165,5bf4587..073dc1e --- a/.travis.yml +++ b/.travis.yml @@@ -34,9 -47,19 +47,12 @@@ before_install install: - time mvn -q -T 2C -DskipTests clean install + - cd $E2E_COMPOSE_HOME && docker-compose up -d + - if [ ! -f ${DOCKER_METRON_CENTOS} ]; then docker save metron-centos | gzip > ${DOCKER_METRON_CENTOS}; fi + - cd $TRAVIS_BUILD_DIR script: - - ls -la - - metron-contrib/metron-docker-e2e/scripts/wait_for_rest.sh localhost 8082 - - cd $E2E_COMPOSE_HOME && docker-compose logs metron-rest && cd ../../../ -# - mvn -q -T 2C surefire:test@unit-tests -# - mvn -q surefire:test@integration-tests -# - mvn -q test --projects metron-interface/metron-config - - mvn test -Pe2e --projects=metron-interface/metron-alerts -# - build_utils/verify_licenses.sh + - time mvn -q -T 2C surefire:test@unit-tests && time mvn -q surefire:test@integration-tests && time mvn -q test --projects metron-interface/metron-config && time dev-utilities/build-utils/verify_licenses.sh before_cache: - rm -rf $HOME/.m2/repository/org/apache/metron http://git-wip-us.apache.org/repos/asf/metron/blob/fc4ce010/metron-contrib/pom.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/fc4ce010/metron-interface/metron-alerts/e2e/utils/e2e_util.ts ---------------------------------------------------------------------- diff --cc metron-interface/metron-alerts/e2e/utils/e2e_util.ts index 8ae1de1,7127daf..9304a5c --- a/metron-interface/metron-alerts/e2e/utils/e2e_util.ts +++ b/metron-interface/metron-alerts/e2e/utils/e2e_util.ts @@@ -46,24 -46,36 +46,45 @@@ export function waitForStalenessOf (_el } export function loadTestData() { - deleteTestData(); + request.delete('http://user:password@' + browser.params.rest.url + '/api/v1/sensor/indexing/config/alerts_ui_e2e', function (e, response, body) { + request.post({url:'http://user:password@' + browser.params.rest.url + '/api/v1/sensor/indexing/config/alerts_ui_e2e', json: + { + "hdfs": { + "index": "alerts_ui_e2e", + "batchSize": 5, + "enabled": true + }, + "elasticsearch": { + "index": "alerts_ui_e2e", + "batchSize": 5, + "enabled": true + }, + "solr": { + "index": "alerts_ui_e2e", + "batchSize": 5, + "enabled": true + } + } + }, function (e, response, body) { + }); + }); - request.delete('http://' + browser.params.elasticsearch.url + '/alerts_ui_e2e_index*', function (e, response, body) { - fs.createReadStream('e2e/mock-data/alerts_ui_e2e_index.template') - .pipe(request.post('http://' + browser.params.elasticsearch.url + '/_template/alerts_ui_e2e_index', function (e, response, body) { - fs.createReadStream('e2e/mock-data/alerts_ui_e2e_index.data') - .pipe(request.post('http://' + browser.params.elasticsearch.url + '/alerts_ui_e2e_index/alerts_ui_e2e_doc/_bulk', function (e, response, body) { - })); - })); + let template = fs.readFileSync('e2e/mock-data/alerts_ui_e2e_index.template', 'utf8'); + request({ + url: 'http://node1:9200/_template/alerts_ui_e2e_index', + method: 'POST', + body: template + }, function(error, response, body) { + // add logging if desired + }); + + let data = fs.readFileSync('e2e/mock-data/alerts_ui_e2e_index.data', 'utf8'); + request({ + url: 'http://node1:9200/alerts_ui_e2e_index/alerts_ui_e2e_doc/_bulk', + method: 'POST', + body: data + }, function(error, response, body) { + // add logging if desired }); } http://git-wip-us.apache.org/repos/asf/metron/blob/fc4ce010/metron-interface/metron-alerts/pom.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/fc4ce010/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java ---------------------------------------------------------------------- diff --cc metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java index 1bc5b6e,ae8dbe0..7089033 --- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java +++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java @@@ -17,48 -17,47 +17,59 @@@ */ package org.apache.metron.elasticsearch.integration; - +import java.io.File; +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.concurrent.ExecutionException; + import java.util.Map; import org.adrianwalker.multilinestring.Multiline; +import org.apache.metron.common.utils.JSONUtils; import org.apache.metron.elasticsearch.dao.ElasticsearchDao; - import org.apache.metron.elasticsearch.integration.components.ElasticSearchComponent; + import org.apache.metron.elasticsearch.utils.ElasticsearchUtils; import org.apache.metron.indexing.dao.AccessConfig; import org.apache.metron.indexing.dao.IndexDao; import org.apache.metron.indexing.dao.SearchIntegrationTest; +import org.apache.metron.indexing.dao.search.GetRequest; +import org.apache.metron.integration.InMemoryComponent; import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; -import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.index.IndexRequestBuilder; +import org.elasticsearch.action.support.WriteRequest; + import org.elasticsearch.action.search.SearchResponse; + import org.elasticsearch.client.Client; + import org.elasticsearch.index.IndexNotFoundException; + import org.elasticsearch.index.query.QueryBuilders; + import org.elasticsearch.search.SearchHit; import org.json.simple.JSONArray; import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; import org.json.simple.parser.ParseException; +import org.junit.Test; - public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest { + import java.io.IOException; + import java.util.HashMap; + import java.util.concurrent.ExecutionException; + import org.junit.AfterClass; + import org.junit.BeforeClass; - private static String indexDir = "target/elasticsearch_search"; + public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest { + private static String host = "localhost"; + private static String port = "9310"; private static String dateFormat = "yyyy.MM.dd.HH"; private static final int MAX_RETRIES = 10; private static final int SLEEP_MS = 500; /** * { - * "bro_doc": { + * "searchintegrationtest_bro_doc": { * "properties": { * "source:type": { - * "type": "string", - * "index": "not_analyzed" + * "type": "text", + * "fielddata" : "true" + * }, + * "guid" : { + * "type" : "keyword" * }, * "ip_src_addr": { * "type": "ip" @@@ -105,14 -102,11 +116,14 @@@ /** * { - * "snort_doc": { + * "searchintegrationtest_snort_doc": { * "properties": { * "source:type": { - * "type": "string", - * "index": "not_analyzed" + * "type": "text", + * "fielddata" : "true" + * }, + * "guid" : { + * "type" : "keyword" * }, * "ip_src_addr": { * "type": "ip" @@@ -173,8 -163,34 +184,34 @@@ * } */ @Multiline - private static String metaAlertTypeMappings; + private static String broDefaultStringMappings; + private static Map<String, Object> globalConfig; + private static Client client; + + @BeforeClass + public static void start() { + globalConfig = new HashMap<String, Object>() {{ + put("es.clustername", "elasticsearch"); + put("es.port", port); + put("es.ip", host); + put("es.date.format", dateFormat); + }}; + client = ElasticsearchUtils.getClient(globalConfig, null); + clearIndices(); + } + + @AfterClass + public static void stop() throws Exception { + clearIndices(); + } + + private static void clearIndices() { + try { + client.admin().indices().prepareDelete(broIndex, snortIndex, metaAlertIndex).get(); + } catch (IndexNotFoundException infe) {} + } + @Override protected IndexDao createDao() throws Exception { AccessConfig config = new AccessConfig(); @@@ -195,25 -204,16 +225,14 @@@ } @Override - protected InMemoryComponent startIndex() throws Exception { - InMemoryComponent es = new ElasticSearchComponent.Builder() - .withHttpPort(9211) - .withIndexDir(new File(indexDir)) - .build(); - es.start(); - return es; - } - - @Override protected void loadTestData() throws ParseException, IOException, ExecutionException, InterruptedException { - ElasticSearchComponent es = (ElasticSearchComponent)indexComponent; - es.getClient().admin().indices().prepareCreate("bro_index_2017.01.01.01") - .addMapping("bro_doc", broTypeMappings).addMapping("bro_doc_default", broDefaultStringMappings).get(); - es.getClient().admin().indices().prepareCreate("snort_index_2017.01.01.02") - .addMapping("snort_doc", snortTypeMappings).get(); + client.admin().indices().prepareCreate(broIndex) + .addMapping(broType, broTypeMappings).get(); + client.admin().indices().prepareCreate(snortIndex) + .addMapping(snortType, snortTypeMappings).get(); - client.admin().indices().prepareCreate(metaAlertIndex) - .addMapping(metaAlertType, metaAlertTypeMappings).get(); - BulkRequestBuilder bulkRequest = es.getClient().prepareBulk().setRefreshPolicy(WriteRequest.RefreshPolicy.WAIT_UNTIL); + BulkRequestBuilder bulkRequest = client.prepareBulk().setRefresh(true); JSONArray broArray = (JSONArray) new JSONParser().parse(broData); for(Object o: broArray) { JSONObject jsonObject = (JSONObject) o; @@@ -236,7 -241,52 +253,6 @@@ if (bulkResponse.hasFailures()) { throw new RuntimeException("Failed to index test data"); } - - SearchResponse broDocs = client - .prepareSearch(broIndex) - .setTypes(broType) - .setQuery(QueryBuilders.matchAllQuery()) - .get(); - // We're changing the _id field, we need to create a copy and delete the original. - for (SearchHit hit : broDocs.getHits()) { - // Bro GUIDs to collide while using the standard analyzer - // Use timestamp as part of guid because query may not return in order each time - IndexRequest indexRequest = new IndexRequest() - .index(broIndex) - .type(broType) - .id("bro-" + hit.getSource().get("timestamp")) - .source(hit.getSource()); - client.index(indexRequest).get(); - - // Delete the original - client.prepareDelete(broIndex, broType, hit.getId()) - .get(); - } - - // Wait until everything is updated - // Assume true until proven otherwise. - boolean allUpdated = true; - for (int t = 0; t < MAX_RETRIES; ++t, Thread.sleep(SLEEP_MS)) { - allUpdated = true; - SearchResponse response = client - .prepareSearch(broIndex) - .setTypes(broType) - .setQuery(QueryBuilders.matchAllQuery()) - .get(); - if (response.getHits().getTotalHits() == 0) { - throw new IllegalStateException("Bro index is empty. No docs to validate were updated"); - } - for (SearchHit hit : response.getHits()) { - if (!hit.getId().startsWith("bro-")) { - allUpdated = false; - } - } - if (allUpdated) { - break; - } - } - if (!allUpdated) { - throw new IllegalStateException("Unable to update Elasticsearch ids properly"); - } } + - } http://git-wip-us.apache.org/repos/asf/metron/blob/fc4ce010/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java ---------------------------------------------------------------------- diff --cc metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java index b40db46,84f6f40..d7aa7c7 --- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java +++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java @@@ -41,11 -34,26 +41,20 @@@ import org.apache.metron.integration.In import org.junit.AfterClass; import org.junit.Assert; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Optional; +import org.junit.rules.ExpectedException; public abstract class SearchIntegrationTest { + + private static final String namespace = SearchIntegrationTest.class.getSimpleName().toLowerCase(); + protected static final String broIndex = namespace + "_bro_index"; + protected static final String snortIndex = namespace + "_snort_index"; + protected static final String metaAlertIndex = namespace + "_metaalert_index"; + protected static final String broType = namespace + "_bro_doc"; + protected static final String snortType = namespace + "_snort_doc"; + protected static final String metaAlertType = namespace + "_metaalert_doc"; + /** * [ * {"source:type": "bro", "ip_src_addr":"192.168.1.1", "ip_src_port": 8010, "long_field": 10000, "timestamp":1, "latitude": 48.5839, "score": 10.0, "is_alert":true, "location_point": "48.5839,7.7455", "bro_field": "bro data 1", "duplicate_name_field": "data 1", "guid":"bro_1"}, @@@ -71,8 -79,17 +80,8 @@@ public static String snortData; /** - * [ - *{"guid":"meta_1","alert":[{"guid":"bro_1"}],"average":"5.0","min":"5.0","median":"5.0","max":"5.0","count":"1.0","sum":"5.0"}, - *{"guid":"meta_2","alert":[{"guid":"bro_1"},{"guid":"bro_2"},{"guid":"snort_1"}],"average":"5.0","min":"0.0","median":"5.0","max":"10.0","count":"3.0","sum":"15.0"} - * ] - */ - @Multiline - public static String metaAlertData; - - /** * { - * "indices": ["bro", "snort"], + * "indices": ["searchintegrationtest_bro", "searchintegrationtest_snort"], * "query": "*", * "from": 0, * "size": 10, @@@ -89,8 -106,8 +98,8 @@@ /** * { - * "guid": "bro-3", + * "guid": "bro_3", - * "sensorType": "bro" + * "sensorType": "searchintegrationtest_bro" * } */ @Multiline @@@ -99,12 -116,12 +108,12 @@@ /** * [ * { - * "guid": "bro-1", + * "guid": "bro_1", - * "sensorType": "bro" + * "sensorType": "searchintegrationtest_bro" * }, * { - * "guid": "bro-2", + * "guid": "snort_2", - * "sensorType": "snort" + * "sensorType": "searchintegrationtest_bro" * } * ] */ @@@ -222,7 -239,7 +231,7 @@@ /** * { * "facetFields": ["source:type", "ip_src_addr", "ip_src_port", "long_field", "timestamp", "latitude", "score", "is_alert"], - * "indices": ["bro", "snort"], - * "indices": ["searchintegrationtest_bro", "searchintegrationtest_snort", "searchintegrationtest_metaalert"], ++ * "indices": ["searchintegrationtest_bro", "searchintegrationtest_snort"], * "query": "*", * "from": 0, * "size": 10, @@@ -328,7 -345,7 +337,7 @@@ /** * { * "fields": ["guid"], - * "indices": ["bro"], - * "indices": ["searchintegrationtest_metaalert"], ++ * "indices": ["searchintegrationtest_bro"], * "query": "*", * "from": 0, * "size": 10, @@@ -355,7 -372,7 +364,7 @@@ * } * ], * "scoreField":"score", - * "indices": ["bro", "snort"], - * "indices": ["searchintegrationtest_bro", "searchintegrationtest_snort", "searchintegrationtest_metaalert"], ++ * "indices": ["searchintegrationtest_bro", "searchintegrationtest_snort"], * "query": "*" * } */ @@@ -380,7 -397,7 +389,7 @@@ * } * } * ], - * "indices": ["bro", "snort"], - * "indices": ["searchintegrationtest_bro", "searchintegrationtest_snort", "searchintegrationtest_metaalert"], ++ * "indices": ["searchintegrationtest_bro", "searchintegrationtest_snort"], * "query": "*" * } */ @@@ -431,269 -447,283 +439,291 @@@ } } + @Rule + public ExpectedException thrown = ExpectedException.none(); + @Test - public void test() throws Exception { - //All Query Testcase - { - SearchRequest request = JSONUtils.INSTANCE.load(allQuery, SearchRequest.class); - SearchResponse response = dao.search(request); - Assert.assertEquals(10, response.getTotal()); - List<SearchResult> results = response.getResults(); - Assert.assertEquals(10, results.size()); - for(int i = 0;i < 5;++i) { - Assert.assertEquals("snort", results.get(i).getSource().get("source:type")); - Assert.assertEquals(10-i, results.get(i).getSource().get("timestamp")); - } - for(int i = 5;i < 10;++i) { - Assert.assertEquals("bro", results.get(i).getSource().get("source:type")); - Assert.assertEquals(10-i, results.get(i).getSource().get("timestamp")); - } - } - //Find One Guid Testcase - { - GetRequest request = JSONUtils.INSTANCE.load(findOneGuidQuery, GetRequest.class); - Optional<Map<String, Object>> response = dao.getLatestResult(request); - Assert.assertTrue(response.isPresent()); - Map<String, Object> doc = response.get(); - Assert.assertEquals("bro", doc.get("source:type")); - Assert.assertEquals(3, doc.get("timestamp")); - } - //Get All Latest Guid Testcase - { - List<GetRequest> request = JSONUtils.INSTANCE.load(getAllLatestQuery, new TypeReference<List<GetRequest>>() { - }); - Iterator<Document> response = dao.getAllLatest(request).iterator(); - Document bro2 = response.next(); - Assert.assertEquals("bro_2", bro2.getDocument().get("guid")); - Assert.assertEquals("bro", bro2.getDocument().get("source:type")); - Document snort2 = response.next(); - Assert.assertEquals("bro_1", snort2.getDocument().get("guid")); - Assert.assertEquals("bro", snort2.getDocument().get("source:type")); - Assert.assertFalse(response.hasNext()); - } - //Filter test case - { - SearchRequest request = JSONUtils.INSTANCE.load(filterQuery, SearchRequest.class); - SearchResponse response = dao.search(request); - Assert.assertEquals(3, response.getTotal()); - List<SearchResult> results = response.getResults(); - Assert.assertEquals("snort", results.get(0).getSource().get("source:type")); - Assert.assertEquals(9, results.get(0).getSource().get("timestamp")); - Assert.assertEquals("snort", results.get(1).getSource().get("source:type")); - Assert.assertEquals(7, results.get(1).getSource().get("timestamp")); - Assert.assertEquals("bro", results.get(2).getSource().get("source:type")); - Assert.assertEquals(1, results.get(2).getSource().get("timestamp")); - } - //Sort test case - { - SearchRequest request = JSONUtils.INSTANCE.load(sortQuery, SearchRequest.class); - SearchResponse response = dao.search(request); - Assert.assertEquals(10, response.getTotal()); - List<SearchResult> results = response.getResults(); - for(int i = 8001;i < 8011;++i) { - Assert.assertEquals(i, results.get(i-8001).getSource().get("ip_src_port")); - } + public void all_query_returns_all_results() throws Exception { + SearchRequest request = JSONUtils.INSTANCE.load(allQuery, SearchRequest.class); + SearchResponse response = dao.search(request); + Assert.assertEquals(10, response.getTotal()); + List<SearchResult> results = response.getResults(); + Assert.assertEquals(10, results.size()); + for(int i = 0;i < 5;++i) { + Assert.assertEquals("snort", results.get(i).getSource().get("source:type")); + Assert.assertEquals(10 - i, results.get(i).getSource().get("timestamp")); } - //Sort descending with missing fields - { - SearchRequest request = JSONUtils.INSTANCE.load(sortDescendingWithMissingFields, SearchRequest.class); - SearchResponse response = dao.search(request); - Assert.assertEquals(10, response.getTotal()); - List<SearchResult> results = response.getResults(); - Assert.assertEquals(10, results.size()); - - // validate sorted order - there are only 2 with a 'threat:triage:score' - Assert.assertEquals("20", results.get(0).getSource().get("threat:triage:score")); - Assert.assertEquals("10", results.get(1).getSource().get("threat:triage:score")); - - // the remaining are missing the 'threat:triage:score' and should be sorted last - Assert.assertFalse(results.get(2).getSource().containsKey("threat:triage:score")); - Assert.assertFalse(results.get(3).getSource().containsKey("threat:triage:score")); - Assert.assertFalse(results.get(4).getSource().containsKey("threat:triage:score")); - Assert.assertFalse(results.get(5).getSource().containsKey("threat:triage:score")); - Assert.assertFalse(results.get(6).getSource().containsKey("threat:triage:score")); - Assert.assertFalse(results.get(7).getSource().containsKey("threat:triage:score")); - Assert.assertFalse(results.get(8).getSource().containsKey("threat:triage:score")); - Assert.assertFalse(results.get(9).getSource().containsKey("threat:triage:score")); + for (int i = 5; i < 10; ++i) { + Assert.assertEquals("bro", results.get(i).getSource().get("source:type")); + Assert.assertEquals(10 - i, results.get(i).getSource().get("timestamp")); } - //Sort ascending with missing fields - { - SearchRequest request = JSONUtils.INSTANCE.load(sortAscendingWithMissingFields, SearchRequest.class); - SearchResponse response = dao.search(request); - Assert.assertEquals(10, response.getTotal()); - List<SearchResult> results = response.getResults(); - Assert.assertEquals(10, results.size()); - - // the remaining are missing the 'threat:triage:score' and should be sorted last - Assert.assertFalse(results.get(0).getSource().containsKey("threat:triage:score")); - Assert.assertFalse(results.get(1).getSource().containsKey("threat:triage:score")); - Assert.assertFalse(results.get(2).getSource().containsKey("threat:triage:score")); - Assert.assertFalse(results.get(3).getSource().containsKey("threat:triage:score")); - Assert.assertFalse(results.get(4).getSource().containsKey("threat:triage:score")); - Assert.assertFalse(results.get(5).getSource().containsKey("threat:triage:score")); - Assert.assertFalse(results.get(6).getSource().containsKey("threat:triage:score")); - Assert.assertFalse(results.get(7).getSource().containsKey("threat:triage:score")); - - // validate sorted order - there are only 2 with a 'threat:triage:score' - Assert.assertEquals("10", results.get(8).getSource().get("threat:triage:score")); - Assert.assertEquals("20", results.get(9).getSource().get("threat:triage:score")); - } - //pagination test case - { - SearchRequest request = JSONUtils.INSTANCE.load(paginationQuery, SearchRequest.class); - SearchResponse response = dao.search(request); - Assert.assertEquals(10, response.getTotal()); - List<SearchResult> results = response.getResults(); - Assert.assertEquals(3, results.size()); - Assert.assertEquals("snort", results.get(0).getSource().get("source:type")); - Assert.assertEquals(6, results.get(0).getSource().get("timestamp")); - Assert.assertEquals("bro", results.get(1).getSource().get("source:type")); - Assert.assertEquals(5, results.get(1).getSource().get("timestamp")); - Assert.assertEquals("bro", results.get(2).getSource().get("source:type")); - Assert.assertEquals(4, results.get(2).getSource().get("timestamp")); - } - //Index query - { - SearchRequest request = JSONUtils.INSTANCE.load(indexQuery, SearchRequest.class); - SearchResponse response = dao.search(request); - Assert.assertEquals(5, response.getTotal()); - List<SearchResult> results = response.getResults(); - for(int i = 5,j=0;i > 0;i--,j++) { - Assert.assertEquals("bro", results.get(j).getSource().get("source:type")); - Assert.assertEquals(i, results.get(j).getSource().get("timestamp")); - } - } - //Facet query including all field types - { - SearchRequest request = JSONUtils.INSTANCE.load(facetQuery, SearchRequest.class); - SearchResponse response = dao.search(request); - Assert.assertEquals(12, response.getTotal()); - - Map<String, Map<String, Long>> facetCounts = response.getFacetCounts(); - Assert.assertEquals(8, facetCounts.size()); - - // source:type - Map<String, Long> sourceTypeCounts = facetCounts.get("source:type"); - Assert.assertEquals(2, sourceTypeCounts.size()); - Assert.assertEquals(new Long(5), sourceTypeCounts.get("bro")); - Assert.assertEquals(new Long(5), sourceTypeCounts.get("snort")); - - // ip_src_addr - Map<String, Long> ipSrcAddrCounts = facetCounts.get("ip_src_addr"); - Assert.assertEquals(8, ipSrcAddrCounts.size()); - Assert.assertEquals(new Long(3), ipSrcAddrCounts.get("192.168.1.1")); - Assert.assertEquals(new Long(1), ipSrcAddrCounts.get("192.168.1.2")); - Assert.assertEquals(new Long(1), ipSrcAddrCounts.get("192.168.1.3")); - Assert.assertEquals(new Long(1), ipSrcAddrCounts.get("192.168.1.4")); - Assert.assertEquals(new Long(1), ipSrcAddrCounts.get("192.168.1.5")); - Assert.assertEquals(new Long(1), ipSrcAddrCounts.get("192.168.1.6")); - Assert.assertEquals(new Long(1), ipSrcAddrCounts.get("192.168.1.7")); - Assert.assertEquals(new Long(1), ipSrcAddrCounts.get("192.168.1.8")); - - // ip_src_port - Map<String, Long> ipSrcPortCounts = facetCounts.get("ip_src_port"); - Assert.assertEquals(10, ipSrcPortCounts.size()); - Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8001")); - Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8002")); - Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8003")); - Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8004")); - Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8005")); - Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8006")); - Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8007")); - Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8008")); - Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8009")); - Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8010")); - - // long_field - Map<String, Long> longFieldCounts = facetCounts.get("long_field"); - Assert.assertEquals(2, longFieldCounts.size()); - Assert.assertEquals(new Long(8), longFieldCounts.get("10000")); - Assert.assertEquals(new Long(2), longFieldCounts.get("20000")); - - // timestamp - Map<String, Long> timestampCounts = facetCounts.get("timestamp"); - Assert.assertEquals(10, timestampCounts.size()); - Assert.assertEquals(new Long(1), timestampCounts.get("1")); - Assert.assertEquals(new Long(1), timestampCounts.get("2")); - Assert.assertEquals(new Long(1), timestampCounts.get("3")); - Assert.assertEquals(new Long(1), timestampCounts.get("4")); - Assert.assertEquals(new Long(1), timestampCounts.get("5")); - Assert.assertEquals(new Long(1), timestampCounts.get("6")); - Assert.assertEquals(new Long(1), timestampCounts.get("7")); - Assert.assertEquals(new Long(1), timestampCounts.get("8")); - Assert.assertEquals(new Long(1), timestampCounts.get("9")); - Assert.assertEquals(new Long(1), timestampCounts.get("10")); - - // latitude - Map<String, Long> latitudeCounts = facetCounts.get("latitude"); - Assert.assertEquals(2, latitudeCounts.size()); - List<String> latitudeKeys = new ArrayList<>(latitudeCounts.keySet()); - Collections.sort(latitudeKeys); - Assert.assertEquals(48.0001, Double.parseDouble(latitudeKeys.get(0)), 0.00001); - Assert.assertEquals(48.5839, Double.parseDouble(latitudeKeys.get(1)), 0.00001); - Assert.assertEquals(new Long(2), latitudeCounts.get(latitudeKeys.get(0))); - Assert.assertEquals(new Long(8), latitudeCounts.get(latitudeKeys.get(1))); - - // score - Map<String, Long> scoreFieldCounts = facetCounts.get("score"); - Assert.assertEquals(4, scoreFieldCounts.size()); - List<String> scoreFieldKeys = new ArrayList<>(scoreFieldCounts.keySet()); - Collections.sort(scoreFieldKeys); - Assert.assertEquals(10.0, Double.parseDouble(scoreFieldKeys.get(0)), 0.00001); - Assert.assertEquals(20.0, Double.parseDouble(scoreFieldKeys.get(1)), 0.00001); - Assert.assertEquals(50.0, Double.parseDouble(scoreFieldKeys.get(2)), 0.00001); - Assert.assertEquals(98.0, Double.parseDouble(scoreFieldKeys.get(3)), 0.00001); - Assert.assertEquals(new Long(4), scoreFieldCounts.get(scoreFieldKeys.get(0))); - Assert.assertEquals(new Long(2), scoreFieldCounts.get(scoreFieldKeys.get(1))); - Assert.assertEquals(new Long(3), scoreFieldCounts.get(scoreFieldKeys.get(2))); - Assert.assertEquals(new Long(1), scoreFieldCounts.get(scoreFieldKeys.get(3))); - - // is_alert - Map<String, Long> isAlertCounts = facetCounts.get("is_alert"); - Assert.assertEquals(2, isAlertCounts.size()); - Assert.assertEquals(new Long(6), isAlertCounts.get("true")); - Assert.assertEquals(new Long(4), isAlertCounts.get("false")); + } + + @Test + public void find_one_guid() throws Exception { + GetRequest request = JSONUtils.INSTANCE.load(findOneGuidQuery, GetRequest.class); + Optional<Map<String, Object>> response = dao.getLatestResult(request); + Assert.assertTrue(response.isPresent()); + Map<String, Object> doc = response.get(); + Assert.assertEquals("bro", doc.get("source:type")); + Assert.assertEquals(3, doc.get("timestamp")); + } + + @Test + public void get_all_latest_guid() throws Exception { + List<GetRequest> request = JSONUtils.INSTANCE.load(getAllLatestQuery, new JSONUtils.ReferenceSupplier<List<GetRequest>>(){}); + Map<String, Document> docs = new HashMap<>(); + + for(Document doc : dao.getAllLatest(request)) { + docs.put(doc.getGuid(), doc); } - //Bad facet query - { - SearchRequest request = JSONUtils.INSTANCE.load(badFacetQuery, SearchRequest.class); - try { - dao.search(request); - Assert.fail("Exception expected, but did not come."); - } - catch(InvalidSearchException ise) { - // success - } + Assert.assertEquals(2, docs.size()); + Assert.assertTrue(docs.keySet().contains("bro_1")); + Assert.assertTrue(docs.keySet().contains("snort_2")); + Assert.assertEquals("bro", docs.get("bro_1").getDocument().get("source:type")); + Assert.assertEquals("snort", docs.get("snort_2").getDocument().get("source:type")); + } + + @Test + public void filter_query_filters_results() throws Exception { + SearchRequest request = JSONUtils.INSTANCE.load(filterQuery, SearchRequest.class); + SearchResponse response = dao.search(request); + Assert.assertEquals(3, response.getTotal()); + List<SearchResult> results = response.getResults(); + Assert.assertEquals("snort", results.get(0).getSource().get("source:type")); + Assert.assertEquals(9, results.get(0).getSource().get("timestamp")); + Assert.assertEquals("snort", results.get(1).getSource().get("source:type")); + Assert.assertEquals(7, results.get(1).getSource().get("timestamp")); + Assert.assertEquals("bro", results.get(2).getSource().get("source:type")); + Assert.assertEquals(1, results.get(2).getSource().get("timestamp")); + } + + @Test + public void sort_query_sorts_results_ascending() throws Exception { + SearchRequest request = JSONUtils.INSTANCE.load(sortQuery, SearchRequest.class); + SearchResponse response = dao.search(request); + Assert.assertEquals(10, response.getTotal()); + List<SearchResult> results = response.getResults(); + for (int i = 8001; i < 8011; ++i) { + Assert.assertEquals(i, results.get(i - 8001).getSource().get("ip_src_port")); } - //Disabled facet query - { - SearchRequest request = JSONUtils.INSTANCE.load(disabledFacetQuery, SearchRequest.class); - SearchResponse response = dao.search(request); - Assert.assertNull(response.getFacetCounts()); + } + + @Test + public void sort_ascending_with_missing_fields() throws Exception { + SearchRequest request = JSONUtils.INSTANCE.load(sortAscendingWithMissingFields, SearchRequest.class); + SearchResponse response = dao.search(request); + Assert.assertEquals(10, response.getTotal()); + List<SearchResult> results = response.getResults(); + Assert.assertEquals(10, results.size()); + + // the remaining are missing the 'threat:triage:score' and should be sorted last + for (int i = 0; i < 8; i++) { + Assert.assertFalse(results.get(i).getSource().containsKey("threat:triage:score")); } - //Exceeded maximum results query - { - SearchRequest request = JSONUtils.INSTANCE.load(exceededMaxResultsQuery, SearchRequest.class); - try { - dao.search(request); - Assert.fail("Exception expected, but did not come."); - } - catch(InvalidSearchException ise) { - Assert.assertEquals("Search result size must be less than 100", ise.getMessage()); - } + + // validate sorted order - there are only 2 with a 'threat:triage:score' + Assert.assertEquals("10", results.get(8).getSource().get("threat:triage:score")); + Assert.assertEquals("20", results.get(9).getSource().get("threat:triage:score")); + } + + @Test + public void sort_descending_with_missing_fields() throws Exception { + SearchRequest request = JSONUtils.INSTANCE.load(sortDescendingWithMissingFields, SearchRequest.class); + SearchResponse response = dao.search(request); + Assert.assertEquals(10, response.getTotal()); + List<SearchResult> results = response.getResults(); + Assert.assertEquals(10, results.size()); + + // validate sorted order - there are only 2 with a 'threat:triage:score' + Assert.assertEquals("20", results.get(0).getSource().get("threat:triage:score")); + Assert.assertEquals("10", results.get(1).getSource().get("threat:triage:score")); + + // the remaining are missing the 'threat:triage:score' and should be sorted last + for (int i = 2; i < 10; i++) { + Assert.assertFalse(results.get(i).getSource().containsKey("threat:triage:score")); } ++<<<<<<< HEAD + } + + @Test + public void results_are_paginated() throws Exception { + SearchRequest request = JSONUtils.INSTANCE.load(paginationQuery, SearchRequest.class); + SearchResponse response = dao.search(request); + Assert.assertEquals(10, response.getTotal()); + List<SearchResult> results = response.getResults(); + Assert.assertEquals(3, results.size()); + Assert.assertEquals("snort", results.get(0).getSource().get("source:type")); + Assert.assertEquals(6, results.get(0).getSource().get("timestamp")); + Assert.assertEquals("bro", results.get(1).getSource().get("source:type")); + Assert.assertEquals(5, results.get(1).getSource().get("timestamp")); + Assert.assertEquals("bro", results.get(2).getSource().get("source:type")); + Assert.assertEquals(4, results.get(2).getSource().get("timestamp")); + } + + @Test + public void returns_results_only_for_specified_indices() throws Exception { + SearchRequest request = JSONUtils.INSTANCE.load(indexQuery, SearchRequest.class); + SearchResponse response = dao.search(request); + Assert.assertEquals(5, response.getTotal()); + List<SearchResult> results = response.getResults(); + for (int i = 5, j = 0; i > 0; i--, j++) { + Assert.assertEquals("bro", results.get(j).getSource().get("source:type")); + Assert.assertEquals(i, results.get(j).getSource().get("timestamp")); ++======= + // getColumnMetadata with multiple indices + { + Map<String, FieldType> fieldTypes = dao.getColumnMetadata(Arrays.asList("searchintegrationtest_bro", "searchintegrationtest_snort")); + Assert.assertEquals(15, fieldTypes.size()); + Assert.assertEquals(FieldType.STRING, fieldTypes.get("guid")); + Assert.assertEquals(FieldType.STRING, fieldTypes.get("source:type")); + Assert.assertEquals(FieldType.IP, fieldTypes.get("ip_src_addr")); + Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("ip_src_port")); + Assert.assertEquals(FieldType.LONG, fieldTypes.get("long_field")); + Assert.assertEquals(FieldType.DATE, fieldTypes.get("timestamp")); + Assert.assertEquals(FieldType.FLOAT, fieldTypes.get("latitude")); + Assert.assertEquals(FieldType.DOUBLE, fieldTypes.get("score")); + Assert.assertEquals(FieldType.BOOLEAN, fieldTypes.get("is_alert")); + Assert.assertEquals(FieldType.OTHER, fieldTypes.get("location_point")); + Assert.assertEquals(FieldType.STRING, fieldTypes.get("bro_field")); + Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("snort_field")); + Assert.assertEquals(FieldType.OTHER, fieldTypes.get("duplicate_name_field")); + Assert.assertEquals(FieldType.FLOAT, fieldTypes.get("threat:triage:score")); + Assert.assertEquals(FieldType.OTHER, fieldTypes.get("alert")); ++>>>>>>> upstream/feature/METRON-1344-test-infrastructure } + } + + @Test + public void facet_query_yields_field_types() throws Exception { + SearchRequest request = JSONUtils.INSTANCE.load(facetQuery, SearchRequest.class); + SearchResponse response = dao.search(request); + Assert.assertEquals(10, response.getTotal()); + Map<String, Map<String, Long>> facetCounts = response.getFacetCounts(); + Assert.assertEquals(8, facetCounts.size()); + Map<String, Long> sourceTypeCounts = facetCounts.get("source:type"); + Assert.assertEquals(2, sourceTypeCounts.size()); + Assert.assertEquals(new Long(5), sourceTypeCounts.get("bro")); + Assert.assertEquals(new Long(5), sourceTypeCounts.get("snort")); + Map<String, Long> ipSrcAddrCounts = facetCounts.get("ip_src_addr"); + Assert.assertEquals(8, ipSrcAddrCounts.size()); + Assert.assertEquals(new Long(3), ipSrcAddrCounts.get("192.168.1.1")); + Assert.assertEquals(new Long(1), ipSrcAddrCounts.get("192.168.1.2")); + Assert.assertEquals(new Long(1), ipSrcAddrCounts.get("192.168.1.3")); + Assert.assertEquals(new Long(1), ipSrcAddrCounts.get("192.168.1.4")); + Assert.assertEquals(new Long(1), ipSrcAddrCounts.get("192.168.1.5")); + Assert.assertEquals(new Long(1), ipSrcAddrCounts.get("192.168.1.6")); + Assert.assertEquals(new Long(1), ipSrcAddrCounts.get("192.168.1.7")); + Assert.assertEquals(new Long(1), ipSrcAddrCounts.get("192.168.1.8")); + Map<String, Long> ipSrcPortCounts = facetCounts.get("ip_src_port"); + Assert.assertEquals(10, ipSrcPortCounts.size()); + Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8001")); + Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8002")); + Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8003")); + Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8004")); + Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8005")); + Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8006")); + Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8007")); + Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8008")); + Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8009")); + Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8010")); + Map<String, Long> longFieldCounts = facetCounts.get("long_field"); + Assert.assertEquals(2, longFieldCounts.size()); + Assert.assertEquals(new Long(8), longFieldCounts.get("10000")); + Assert.assertEquals(new Long(2), longFieldCounts.get("20000")); + Map<String, Long> timestampCounts = facetCounts.get("timestamp"); + Assert.assertEquals(10, timestampCounts.size()); + Assert.assertEquals(new Long(1), timestampCounts.get("1")); + Assert.assertEquals(new Long(1), timestampCounts.get("2")); + Assert.assertEquals(new Long(1), timestampCounts.get("3")); + Assert.assertEquals(new Long(1), timestampCounts.get("4")); + Assert.assertEquals(new Long(1), timestampCounts.get("5")); + Assert.assertEquals(new Long(1), timestampCounts.get("6")); + Assert.assertEquals(new Long(1), timestampCounts.get("7")); + Assert.assertEquals(new Long(1), timestampCounts.get("8")); + Assert.assertEquals(new Long(1), timestampCounts.get("9")); + Assert.assertEquals(new Long(1), timestampCounts.get("10")); + Map<String, Long> latitudeCounts = facetCounts.get("latitude"); + Assert.assertEquals(2, latitudeCounts.size()); + List<String> latitudeKeys = new ArrayList<>(latitudeCounts.keySet()); + Collections.sort(latitudeKeys); + Assert.assertEquals(48.0001, Double.parseDouble(latitudeKeys.get(0)), 0.00001); + Assert.assertEquals(48.5839, Double.parseDouble(latitudeKeys.get(1)), 0.00001); + Assert.assertEquals(new Long(2), latitudeCounts.get(latitudeKeys.get(0))); + Assert.assertEquals(new Long(8), latitudeCounts.get(latitudeKeys.get(1))); + Map<String, Long> scoreFieldCounts = facetCounts.get("score"); + Assert.assertEquals(4, scoreFieldCounts.size()); + List<String> scoreFieldKeys = new ArrayList<>(scoreFieldCounts.keySet()); + Collections.sort(scoreFieldKeys); + Assert.assertEquals(10.0, Double.parseDouble(scoreFieldKeys.get(0)), 0.00001); + Assert.assertEquals(20.0, Double.parseDouble(scoreFieldKeys.get(1)), 0.00001); + Assert.assertEquals(50.0, Double.parseDouble(scoreFieldKeys.get(2)), 0.00001); + Assert.assertEquals(98.0, Double.parseDouble(scoreFieldKeys.get(3)), 0.00001); + Assert.assertEquals(new Long(4), scoreFieldCounts.get(scoreFieldKeys.get(0))); + Assert.assertEquals(new Long(2), scoreFieldCounts.get(scoreFieldKeys.get(1))); + Assert.assertEquals(new Long(3), scoreFieldCounts.get(scoreFieldKeys.get(2))); + Assert.assertEquals(new Long(1), scoreFieldCounts.get(scoreFieldKeys.get(3))); + Map<String, Long> isAlertCounts = facetCounts.get("is_alert"); + Assert.assertEquals(2, isAlertCounts.size()); + Assert.assertEquals(new Long(6), isAlertCounts.get("true")); + Assert.assertEquals(new Long(4), isAlertCounts.get("false")); + } + + @Test + public void bad_facet_query_throws_exception() throws Exception { + thrown.expect(InvalidSearchException.class); + thrown.expectMessage("Failed to execute search"); + SearchRequest request = JSONUtils.INSTANCE.load(badFacetQuery, SearchRequest.class); + dao.search(request); + } + + @Test + public void disabled_facet_query_returns_null_count() throws Exception { + SearchRequest request = JSONUtils.INSTANCE.load(disabledFacetQuery, SearchRequest.class); + SearchResponse response = dao.search(request); + Assert.assertNull(response.getFacetCounts()); + } + + @Test + public void exceeding_max_resulsts_throws_exception() throws Exception { + thrown.expect(InvalidSearchException.class); + thrown.expectMessage("Search result size must be less than 100"); + SearchRequest request = JSONUtils.INSTANCE.load(exceededMaxResultsQuery, SearchRequest.class); + dao.search(request); + } + + @Test + public void returns_column_data_for_multiple_indices() throws Exception { + Map<String, FieldType> fieldTypes = dao.getColumnMetadata(Arrays.asList("bro", "snort")); + Assert.assertEquals(15, fieldTypes.size()); + Assert.assertEquals(FieldType.KEYWORD, fieldTypes.get("guid")); + Assert.assertEquals(FieldType.TEXT, fieldTypes.get("source:type")); + Assert.assertEquals(FieldType.IP, fieldTypes.get("ip_src_addr")); + Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("ip_src_port")); + Assert.assertEquals(FieldType.LONG, fieldTypes.get("long_field")); + Assert.assertEquals(FieldType.DATE, fieldTypes.get("timestamp")); + Assert.assertEquals(FieldType.FLOAT, fieldTypes.get("latitude")); + Assert.assertEquals(FieldType.DOUBLE, fieldTypes.get("score")); + Assert.assertEquals(FieldType.BOOLEAN, fieldTypes.get("is_alert")); + Assert.assertEquals(FieldType.OTHER, fieldTypes.get("location_point")); + Assert.assertEquals(FieldType.TEXT, fieldTypes.get("bro_field")); + Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("snort_field")); + //NOTE: This is because the field is in both bro and snort and they have different types. + Assert.assertEquals(FieldType.OTHER, fieldTypes.get("duplicate_name_field")); + Assert.assertEquals(FieldType.FLOAT, fieldTypes.get("threat:triage:score")); + Assert.assertEquals(FieldType.OTHER, fieldTypes.get("alert")); + } + + @Test + public void returns_column_metadata_for_specified_indices() throws Exception { // getColumnMetadata with only bro { - Map<String, FieldType> fieldTypes = dao.getColumnMetadata(Collections.singletonList("bro")); + Map<String, FieldType> fieldTypes = dao.getColumnMetadata(Collections.singletonList("searchintegrationtest_bro")); Assert.assertEquals(13, fieldTypes.size()); - Assert.assertEquals(FieldType.STRING, fieldTypes.get("guid")); - Assert.assertEquals(FieldType.STRING, fieldTypes.get("source:type")); + Assert.assertEquals(FieldType.TEXT, fieldTypes.get("bro_field")); + Assert.assertEquals(FieldType.TEXT, fieldTypes.get("duplicate_name_field")); + Assert.assertEquals(FieldType.KEYWORD, fieldTypes.get("guid")); + Assert.assertEquals(FieldType.TEXT, fieldTypes.get("source:type")); Assert.assertEquals(FieldType.IP, fieldTypes.get("ip_src_addr")); Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("ip_src_port")); Assert.assertEquals(FieldType.LONG, fieldTypes.get("long_field")); @@@ -708,12 -738,11 +738,12 @@@ } // getColumnMetadata with only snort { - Map<String, FieldType> fieldTypes = dao.getColumnMetadata(Collections.singletonList("snort")); + Map<String, FieldType> fieldTypes = dao.getColumnMetadata(Collections.singletonList("searchintegrationtest_snort")); Assert.assertEquals(14, fieldTypes.size()); Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("snort_field")); - Assert.assertEquals(FieldType.STRING, fieldTypes.get("guid")); - Assert.assertEquals(FieldType.STRING, fieldTypes.get("source:type")); + Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("duplicate_name_field")); + Assert.assertEquals(FieldType.KEYWORD, fieldTypes.get("guid")); + Assert.assertEquals(FieldType.TEXT, fieldTypes.get("source:type")); Assert.assertEquals(FieldType.IP, fieldTypes.get("ip_src_addr")); Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("ip_src_port")); Assert.assertEquals(FieldType.LONG, fieldTypes.get("long_field")); @@@ -727,211 -756,207 +757,211 @@@ } // getColumnMetadata with an index that doesn't exist { - Map<String, FieldType> fieldTypes = dao.getColumnMetadata(Collections.singletonList("someindex")); + Map<String, FieldType> fieldTypes = dao.getColumnMetadata(Collections.singletonList("searchintegrationtest_someindex")); Assert.assertEquals(0, fieldTypes.size()); } - //Fields query - { - SearchRequest request = JSONUtils.INSTANCE.load(fieldsQuery, SearchRequest.class); - SearchResponse response = dao.search(request); - Assert.assertEquals(10, response.getTotal()); - List<SearchResult> results = response.getResults(); - for(int i = 0;i < 5;++i) { - Map<String, Object> source = results.get(i).getSource(); - Assert.assertEquals(1, source.size()); - Assert.assertNotNull(source.get("ip_src_addr")); - } - for(int i = 5;i < 10;++i) { - Map<String, Object> source = results.get(i).getSource(); - Assert.assertEquals(1, source.size()); - Assert.assertNotNull(source.get("ip_src_addr")); - } - } - //Meta Alerts Fields query - { - SearchRequest request = JSONUtils.INSTANCE.load(metaAlertsFieldQuery, SearchRequest.class); - SearchResponse response = dao.search(request); - Assert.assertEquals(2, response.getTotal()); - List<SearchResult> results = response.getResults(); - for (int i = 0;i < 2;++i) { - Map<String, Object> source = results.get(i).getSource(); - Assert.assertEquals(1, source.size()); - Assert.assertEquals(source.get("guid"), "meta_" + (i + 1)); - } - } - //No results fields query - { - SearchRequest request = JSONUtils.INSTANCE.load(noResultsFieldsQuery, SearchRequest.class); - SearchResponse response = dao.search(request); - Assert.assertEquals(0, response.getTotal()); - } - // Group by test case, default order is count descending - { - GroupRequest request = JSONUtils.INSTANCE.load(groupByQuery, GroupRequest.class); - GroupResponse response = dao.group(request); - Assert.assertEquals("is_alert", response.getGroupedBy()); - List<GroupResult> isAlertGroups = response.getGroupResults(); - Assert.assertEquals(2, isAlertGroups.size()); - - // isAlert == true group - GroupResult trueGroup = isAlertGroups.get(0); - Assert.assertEquals("true", trueGroup.getKey()); - Assert.assertEquals(6, trueGroup.getTotal()); - Assert.assertEquals("latitude", trueGroup.getGroupedBy()); - Assert.assertEquals(198.0, trueGroup.getScore(), 0.00001); - List<GroupResult> trueLatitudeGroups = trueGroup.getGroupResults(); - Assert.assertEquals(2, trueLatitudeGroups.size()); - - // isAlert == true && latitude == 48.5839 group - GroupResult trueLatitudeGroup2 = trueLatitudeGroups.get(0); - Assert.assertEquals(48.5839, Double.parseDouble(trueLatitudeGroup2.getKey()), 0.00001); - Assert.assertEquals(5, trueLatitudeGroup2.getTotal()); - Assert.assertEquals(148.0, trueLatitudeGroup2.getScore(), 0.00001); - - // isAlert == true && latitude == 48.0001 group - GroupResult trueLatitudeGroup1 = trueLatitudeGroups.get(1); - Assert.assertEquals(48.0001, Double.parseDouble(trueLatitudeGroup1.getKey()), 0.00001); - Assert.assertEquals(1, trueLatitudeGroup1.getTotal()); - Assert.assertEquals(50.0, trueLatitudeGroup1.getScore(), 0.00001); - - // isAlert == false group - GroupResult falseGroup = isAlertGroups.get(1); - Assert.assertEquals("false", falseGroup.getKey()); - Assert.assertEquals("latitude", falseGroup.getGroupedBy()); - Assert.assertEquals(130.0, falseGroup.getScore(), 0.00001); - List<GroupResult> falseLatitudeGroups = falseGroup.getGroupResults(); - Assert.assertEquals(2, falseLatitudeGroups.size()); - - // isAlert == false && latitude == 48.5839 group - GroupResult falseLatitudeGroup2 = falseLatitudeGroups.get(0); - Assert.assertEquals(48.5839, Double.parseDouble(falseLatitudeGroup2.getKey()), 0.00001); - Assert.assertEquals(3, falseLatitudeGroup2.getTotal()); - Assert.assertEquals(80.0, falseLatitudeGroup2.getScore(), 0.00001); - - // isAlert == false && latitude == 48.0001 group - GroupResult falseLatitudeGroup1 = falseLatitudeGroups.get(1); - Assert.assertEquals(48.0001, Double.parseDouble(falseLatitudeGroup1.getKey()), 0.00001); - Assert.assertEquals(1, falseLatitudeGroup1.getTotal()); - Assert.assertEquals(50.0, falseLatitudeGroup1.getScore(), 0.00001); - } - // Group by with sorting test case where is_alert is sorted by count ascending and ip_src_addr is sorted by term descending - { - GroupRequest request = JSONUtils.INSTANCE.load(sortedGroupByQuery, GroupRequest.class); - GroupResponse response = dao.group(request); - Assert.assertEquals("is_alert", response.getGroupedBy()); - List<GroupResult> isAlertGroups = response.getGroupResults(); - Assert.assertEquals(2, isAlertGroups.size()); - - // isAlert == false group - GroupResult falseGroup = isAlertGroups.get(0); - Assert.assertEquals(4, falseGroup.getTotal()); - Assert.assertEquals("ip_src_addr", falseGroup.getGroupedBy()); - List<GroupResult> falseIpSrcAddrGroups = falseGroup.getGroupResults(); - Assert.assertEquals(4, falseIpSrcAddrGroups.size()); - - // isAlert == false && ip_src_addr == 192.168.1.8 group - GroupResult falseIpSrcAddrGroup1 = falseIpSrcAddrGroups.get(0); - Assert.assertEquals("192.168.1.8", falseIpSrcAddrGroup1.getKey()); - Assert.assertEquals(1, falseIpSrcAddrGroup1.getTotal()); - Assert.assertNull(falseIpSrcAddrGroup1.getGroupedBy()); - Assert.assertNull(falseIpSrcAddrGroup1.getGroupResults()); - - // isAlert == false && ip_src_addr == 192.168.1.7 group - GroupResult falseIpSrcAddrGroup2 = falseIpSrcAddrGroups.get(1); - Assert.assertEquals("192.168.1.7", falseIpSrcAddrGroup2.getKey()); - Assert.assertEquals(1, falseIpSrcAddrGroup2.getTotal()); - Assert.assertNull(falseIpSrcAddrGroup2.getGroupedBy()); - Assert.assertNull(falseIpSrcAddrGroup2.getGroupResults()); - - // isAlert == false && ip_src_addr == 192.168.1.6 group - GroupResult falseIpSrcAddrGroup3 = falseIpSrcAddrGroups.get(2); - Assert.assertEquals("192.168.1.6", falseIpSrcAddrGroup3.getKey()); - Assert.assertEquals(1, falseIpSrcAddrGroup3.getTotal()); - Assert.assertNull(falseIpSrcAddrGroup3.getGroupedBy()); - Assert.assertNull(falseIpSrcAddrGroup3.getGroupResults()); - - // isAlert == false && ip_src_addr == 192.168.1.2 group - GroupResult falseIpSrcAddrGroup4 = falseIpSrcAddrGroups.get(3); - Assert.assertEquals("192.168.1.2", falseIpSrcAddrGroup4.getKey()); - Assert.assertEquals(1, falseIpSrcAddrGroup4.getTotal()); - Assert.assertNull(falseIpSrcAddrGroup4.getGroupedBy()); - Assert.assertNull(falseIpSrcAddrGroup4.getGroupResults()); - - // isAlert == false group - GroupResult trueGroup = isAlertGroups.get(1); - Assert.assertEquals(6, trueGroup.getTotal()); - Assert.assertEquals("ip_src_addr", trueGroup.getGroupedBy()); - List<GroupResult> trueIpSrcAddrGroups = trueGroup.getGroupResults(); - Assert.assertEquals(4, trueIpSrcAddrGroups.size()); - - // isAlert == false && ip_src_addr == 192.168.1.5 group - GroupResult trueIpSrcAddrGroup1 = trueIpSrcAddrGroups.get(0); - Assert.assertEquals("192.168.1.5", trueIpSrcAddrGroup1.getKey()); - Assert.assertEquals(1, trueIpSrcAddrGroup1.getTotal()); - Assert.assertNull(trueIpSrcAddrGroup1.getGroupedBy()); - Assert.assertNull(trueIpSrcAddrGroup1.getGroupResults()); - - // isAlert == false && ip_src_addr == 192.168.1.4 group - GroupResult trueIpSrcAddrGroup2 = trueIpSrcAddrGroups.get(1); - Assert.assertEquals("192.168.1.4", trueIpSrcAddrGroup2.getKey()); - Assert.assertEquals(1, trueIpSrcAddrGroup2.getTotal()); - Assert.assertNull(trueIpSrcAddrGroup2.getGroupedBy()); - Assert.assertNull(trueIpSrcAddrGroup2.getGroupResults()); - - // isAlert == false && ip_src_addr == 192.168.1.3 group - GroupResult trueIpSrcAddrGroup3 = trueIpSrcAddrGroups.get(2); - Assert.assertEquals("192.168.1.3", trueIpSrcAddrGroup3.getKey()); - Assert.assertEquals(1, trueIpSrcAddrGroup3.getTotal()); - Assert.assertNull(trueIpSrcAddrGroup3.getGroupedBy()); - Assert.assertNull(trueIpSrcAddrGroup3.getGroupResults()); - - // isAlert == false && ip_src_addr == 192.168.1.1 group - GroupResult trueIpSrcAddrGroup4 = trueIpSrcAddrGroups.get(3); - Assert.assertEquals("192.168.1.1", trueIpSrcAddrGroup4.getKey()); - Assert.assertEquals(3, trueIpSrcAddrGroup4.getTotal()); - Assert.assertNull(trueIpSrcAddrGroup4.getGroupedBy()); - Assert.assertNull(trueIpSrcAddrGroup4.getGroupResults()); - } - //Bad group query - { - GroupRequest request = JSONUtils.INSTANCE.load(badGroupQuery, GroupRequest.class); - try { - dao.group(request); - Assert.fail("Exception expected, but did not come."); - } - catch(InvalidSearchException ise) { - // success - } - } - //Group by IP query - { - { - GroupRequest request = JSONUtils.INSTANCE.load(groupByIpQuery, GroupRequest.class); - GroupResponse response = dao.group(request); + } + + + @Test + public void group_by_ip_query() throws Exception { + GroupRequest request = JSONUtils.INSTANCE.load(groupByIpQuery, GroupRequest.class); + GroupResponse response = dao.group(request); + + // expect only 1 group for 'ip_src_addr' + Assert.assertEquals("ip_src_addr", response.getGroupedBy()); + + // there are 8 different 'ip_src_addr' values + List<GroupResult> groups = response.getGroupResults(); + Assert.assertEquals(8, groups.size()); + + // expect dotted-decimal notation in descending order + Assert.assertEquals("192.168.1.8", groups.get(0).getKey()); + Assert.assertEquals("192.168.1.7", groups.get(1).getKey()); + Assert.assertEquals("192.168.1.6", groups.get(2).getKey()); + Assert.assertEquals("192.168.1.5", groups.get(3).getKey()); + Assert.assertEquals("192.168.1.4", groups.get(4).getKey()); + Assert.assertEquals("192.168.1.3", groups.get(5).getKey()); + Assert.assertEquals("192.168.1.2", groups.get(6).getKey()); + Assert.assertEquals("192.168.1.1", groups.get(7).getKey()); + } - // expect only 1 group for 'ip_src_addr' - Assert.assertEquals("ip_src_addr", response.getGroupedBy()); + @Test + public void no_results_returned_when_query_does_not_match() throws Exception { + SearchRequest request = JSONUtils.INSTANCE.load(noResultsFieldsQuery, SearchRequest.class); + SearchResponse response = dao.search(request); + Assert.assertEquals(0, response.getTotal()); + } + + @Test + public void group_by_returns_results_in_groups() throws Exception { + // Group by test case, default order is count descending + GroupRequest request = JSONUtils.INSTANCE.load(groupByQuery, GroupRequest.class); + GroupResponse response = dao.group(request); + Assert.assertEquals("is_alert", response.getGroupedBy()); + List<GroupResult> isAlertGroups = response.getGroupResults(); + Assert.assertEquals(2, isAlertGroups.size()); + + // isAlert == true group + GroupResult trueGroup = isAlertGroups.get(0); + Assert.assertEquals("true", trueGroup.getKey()); + Assert.assertEquals(6, trueGroup.getTotal()); + Assert.assertEquals("latitude", trueGroup.getGroupedBy()); + Assert.assertEquals(198.0, trueGroup.getScore(), 0.00001); + List<GroupResult> trueLatitudeGroups = trueGroup.getGroupResults(); + Assert.assertEquals(2, trueLatitudeGroups.size()); + + + // isAlert == true && latitude == 48.5839 group + GroupResult trueLatitudeGroup2 = trueLatitudeGroups.get(0); + Assert.assertEquals(48.5839, Double.parseDouble(trueLatitudeGroup2.getKey()), 0.00001); + Assert.assertEquals(5, trueLatitudeGroup2.getTotal()); + Assert.assertEquals(148.0, trueLatitudeGroup2.getScore(), 0.00001); + + // isAlert == true && latitude == 48.0001 group + GroupResult trueLatitudeGroup1 = trueLatitudeGroups.get(1); + Assert.assertEquals(48.0001, Double.parseDouble(trueLatitudeGroup1.getKey()), 0.00001); + Assert.assertEquals(1, trueLatitudeGroup1.getTotal()); + Assert.assertEquals(50.0, trueLatitudeGroup1.getScore(), 0.00001); + + // isAlert == false group + GroupResult falseGroup = isAlertGroups.get(1); + Assert.assertEquals("false", falseGroup.getKey()); + Assert.assertEquals("latitude", falseGroup.getGroupedBy()); + Assert.assertEquals(130.0, falseGroup.getScore(), 0.00001); + List<GroupResult> falseLatitudeGroups = falseGroup.getGroupResults(); + Assert.assertEquals(2, falseLatitudeGroups.size()); + + // isAlert == false && latitude == 48.5839 group + GroupResult falseLatitudeGroup2 = falseLatitudeGroups.get(0); + Assert.assertEquals(48.5839, Double.parseDouble(falseLatitudeGroup2.getKey()), 0.00001); + Assert.assertEquals(3, falseLatitudeGroup2.getTotal()); + Assert.assertEquals(80.0, falseLatitudeGroup2.getScore(), 0.00001); + + // isAlert == false && latitude == 48.0001 group + GroupResult falseLatitudeGroup1 = falseLatitudeGroups.get(1); + Assert.assertEquals(48.0001, Double.parseDouble(falseLatitudeGroup1.getKey()), 0.00001); + Assert.assertEquals(1, falseLatitudeGroup1.getTotal()); + Assert.assertEquals(50.0, falseLatitudeGroup1.getScore(), 0.00001); + } - // there are 8 different 'ip_src_addr' values - List<GroupResult> groups = response.getGroupResults(); - Assert.assertEquals(8, groups.size()); + @Test + public void group_by_returns_results_in_sorted_groups() throws Exception { + // Group by with sorting test case where is_alert is sorted by count ascending and ip_src_addr is sorted by term descending + GroupRequest request = JSONUtils.INSTANCE.load(sortedGroupByQuery, GroupRequest.class); + GroupResponse response = dao.group(request); + Assert.assertEquals("is_alert", response.getGroupedBy()); + List<GroupResult> isAlertGroups = response.getGroupResults(); + Assert.assertEquals(2, isAlertGroups.size()); + + // isAlert == false group + GroupResult falseGroup = isAlertGroups.get(0); + Assert.assertEquals(4, falseGroup.getTotal()); + Assert.assertEquals("ip_src_addr", falseGroup.getGroupedBy()); + List<GroupResult> falseIpSrcAddrGroups = falseGroup.getGroupResults(); + Assert.assertEquals(4, falseIpSrcAddrGroups.size()); + + // isAlert == false && ip_src_addr == 192.168.1.8 group + GroupResult falseIpSrcAddrGroup1 = falseIpSrcAddrGroups.get(0); + Assert.assertEquals("192.168.1.8", falseIpSrcAddrGroup1.getKey()); + Assert.assertEquals(1, falseIpSrcAddrGroup1.getTotal()); + Assert.assertNull(falseIpSrcAddrGroup1.getGroupedBy()); + Assert.assertNull(falseIpSrcAddrGroup1.getGroupResults()); + + // isAlert == false && ip_src_addr == 192.168.1.7 group + GroupResult falseIpSrcAddrGroup2 = falseIpSrcAddrGroups.get(1); + Assert.assertEquals("192.168.1.7", falseIpSrcAddrGroup2.getKey()); + Assert.assertEquals(1, falseIpSrcAddrGroup2.getTotal()); + Assert.assertNull(falseIpSrcAddrGroup2.getGroupedBy()); + Assert.assertNull(falseIpSrcAddrGroup2.getGroupResults()); + + // isAlert == false && ip_src_addr == 192.168.1.6 group + GroupResult falseIpSrcAddrGroup3 = falseIpSrcAddrGroups.get(2); + Assert.assertEquals("192.168.1.6", falseIpSrcAddrGroup3.getKey()); + Assert.assertEquals(1, falseIpSrcAddrGroup3.getTotal()); + Assert.assertNull(falseIpSrcAddrGroup3.getGroupedBy()); + Assert.assertNull(falseIpSrcAddrGroup3.getGroupResults()); + + // isAlert == false && ip_src_addr == 192.168.1.2 group + GroupResult falseIpSrcAddrGroup4 = falseIpSrcAddrGroups.get(3); + Assert.assertEquals("192.168.1.2", falseIpSrcAddrGroup4.getKey()); + Assert.assertEquals(1, falseIpSrcAddrGroup4.getTotal()); + Assert.assertNull(falseIpSrcAddrGroup4.getGroupedBy()); + Assert.assertNull(falseIpSrcAddrGroup4.getGroupResults()); + + // isAlert == false group + GroupResult trueGroup = isAlertGroups.get(1); + Assert.assertEquals(6, trueGroup.getTotal()); + Assert.assertEquals("ip_src_addr", trueGroup.getGroupedBy()); + List<GroupResult> trueIpSrcAddrGroups = trueGroup.getGroupResults(); + Assert.assertEquals(4, trueIpSrcAddrGroups.size()); + + // isAlert == false && ip_src_addr == 192.168.1.5 group + GroupResult trueIpSrcAddrGroup1 = trueIpSrcAddrGroups.get(0); + Assert.assertEquals("192.168.1.5", trueIpSrcAddrGroup1.getKey()); + Assert.assertEquals(1, trueIpSrcAddrGroup1.getTotal()); + Assert.assertNull(trueIpSrcAddrGroup1.getGroupedBy()); + Assert.assertNull(trueIpSrcAddrGroup1.getGroupResults()); + + // isAlert == false && ip_src_addr == 192.168.1.4 group + GroupResult trueIpSrcAddrGroup2 = trueIpSrcAddrGroups.get(1); + Assert.assertEquals("192.168.1.4", trueIpSrcAddrGroup2.getKey()); + Assert.assertEquals(1, trueIpSrcAddrGroup2.getTotal()); + Assert.assertNull(trueIpSrcAddrGroup2.getGroupedBy()); + Assert.assertNull(trueIpSrcAddrGroup2.getGroupResults()); + + // isAlert == false && ip_src_addr == 192.168.1.3 group + GroupResult trueIpSrcAddrGroup3 = trueIpSrcAddrGroups.get(2); + Assert.assertEquals("192.168.1.3", trueIpSrcAddrGroup3.getKey()); + Assert.assertEquals(1, trueIpSrcAddrGroup3.getTotal()); + Assert.assertNull(trueIpSrcAddrGroup3.getGroupedBy()); + Assert.assertNull(trueIpSrcAddrGroup3.getGroupResults()); + + // isAlert == false && ip_src_addr == 192.168.1.1 group + GroupResult trueIpSrcAddrGroup4 = trueIpSrcAddrGroups.get(3); + Assert.assertEquals("192.168.1.1", trueIpSrcAddrGroup4.getKey()); + Assert.assertEquals(3, trueIpSrcAddrGroup4.getTotal()); + Assert.assertNull(trueIpSrcAddrGroup4.getGroupedBy()); + Assert.assertNull(trueIpSrcAddrGroup4.getGroupResults()); + } - // expect dotted-decimal notation in descending order - Assert.assertEquals("192.168.1.8", groups.get(0).getKey()); - Assert.assertEquals("192.168.1.7", groups.get(1).getKey()); - Assert.assertEquals("192.168.1.6", groups.get(2).getKey()); - Assert.assertEquals("192.168.1.5", groups.get(3).getKey()); - Assert.assertEquals("192.168.1.4", groups.get(4).getKey()); - Assert.assertEquals("192.168.1.3", groups.get(5).getKey()); - Assert.assertEquals("192.168.1.2", groups.get(6).getKey()); - Assert.assertEquals("192.168.1.1", groups.get(7).getKey()); - } + @Test + public void throws_exception_on_aggregation_queries_on_non_string_non_numeric_fields() + throws Exception { + thrown.expect(InvalidSearchException.class); + thrown.expectMessage("Failed to execute search"); + GroupRequest request = JSONUtils.INSTANCE.load(badGroupQuery, GroupRequest.class); + dao.group(request); + } + @Test + public void queries_fields() throws Exception { + SearchRequest request = JSONUtils.INSTANCE.load(fieldsQuery, SearchRequest.class); + SearchResponse response = dao.search(request); + Assert.assertEquals(10, response.getTotal()); + List<SearchResult> results = response.getResults(); + for (int i = 0; i < 5; ++i) { + Map<String, Object> source = results.get(i).getSource(); + Assert.assertEquals(1, source.size()); + Assert.assertNotNull(source.get("ip_src_addr")); + } + for (int i = 5; i < 10; ++i) { + Map<String, Object> source = results.get(i).getSource(); + Assert.assertEquals(1, source.size()); + Assert.assertNotNull(source.get("ip_src_addr")); + } + } + @Test + public void sort_by_guid() throws Exception { + SearchRequest request = JSONUtils.INSTANCE.load(sortByGuidQuery, SearchRequest.class); + SearchResponse response = dao.search(request); + Assert.assertEquals(5, response.getTotal()); + List<SearchResult> results = response.getResults(); + for (int i = 0; i < 5; ++i) { + Map<String, Object> source = results.get(i).getSource(); + Assert.assertEquals(1, source.size()); + Assert.assertEquals(source.get("guid"), "bro_" + (i + 1)); } }
