Repository: metron Updated Branches: refs/heads/feature/METRON-1416-upgrade-solr f715d6dba -> 7ef4b7703
METRON-1503 Alerts are not getting populated in alerts UI when search engine is Solr (merrimanr) closes apache/metron#975 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/7ef4b770 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/7ef4b770 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/7ef4b770 Branch: refs/heads/feature/METRON-1416-upgrade-solr Commit: 7ef4b7703e75538f96764cb3efafe8db5afba126 Parents: f715d6d Author: merrimanr <merrim...@gmail.com> Authored: Thu Apr 12 09:51:04 2018 -0500 Committer: merrimanr <merrim...@gmail.com> Committed: Thu Apr 12 09:51:04 2018 -0500 ---------------------------------------------------------------------- .../indexing/dao/search/SearchResponse.java | 23 + .../indexing/dao/SearchIntegrationTest.java | 2 +- .../metron/solr/dao/SolrColumnMetadataDao.java | 14 +- .../org/apache/metron/solr/dao/SolrDao.java | 12 +- .../apache/metron/solr/dao/SolrSearchDao.java | 81 ++-- .../metron/solr/dao/SolrColumnMetadataTest.java | 150 ++++++ .../org/apache/metron/solr/dao/SolrDaoTest.java | 133 ++++++ .../metron/solr/dao/SolrSearchDaoTest.java | 478 +++++++++++++++++++ .../metron/solr/dao/SolrUpdateDaoTest.java | 135 ++++++ .../matcher/ModifiableSolrParamsMatcher.java | 55 +++ .../matcher/SolrInputDocumentListMatcher.java | 60 +++ .../solr/matcher/SolrInputDocumentMatcher.java | 50 ++ .../metron/solr/matcher/SolrQueryMatcher.java | 56 +++ metron-platform/pom.xml | 4 +- pom.xml | 1 + 15 files changed, 1207 insertions(+), 47 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/7ef4b770/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SearchResponse.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SearchResponse.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SearchResponse.java index aad489a..5b0b006 100644 --- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SearchResponse.java +++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SearchResponse.java @@ -61,4 +61,27 @@ public class SearchResponse { public void setFacetCounts(Map<String, Map<String, Long>> facetCounts) { this.facetCounts = facetCounts; } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + SearchResponse that = (SearchResponse) o; + + return getTotal() == that.getTotal() && + (getResults() != null ? getResults().equals(that.getResults()) : that.getResults() != null) && + (getFacetCounts() != null ? getFacetCounts().equals(that.getFacetCounts()) : that.getFacetCounts() != null); + } + + @Override + public int hashCode() { + int result = 31 * (int) getTotal() + (getResults() != null ? getResults().hashCode() : 0); + result = 31 * result + (getFacetCounts() != null ? getFacetCounts().hashCode() : 0); + return result; + } } http://git-wip-us.apache.org/repos/asf/metron/blob/7ef4b770/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java index 16d4d7b..83046b8 100644 --- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java +++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java @@ -71,7 +71,7 @@ public abstract class SearchIntegrationTest { /** * { - * "indices": ["bro", "snort"], + * "indices": ["bro", "snort", "some_collection"], * "query": "*", * "from": 0, * "size": 10, http://git-wip-us.apache.org/repos/asf/metron/blob/7ef4b770/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrColumnMetadataDao.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrColumnMetadataDao.java b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrColumnMetadataDao.java index f645e93..61f0209 100644 --- a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrColumnMetadataDao.java +++ b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrColumnMetadataDao.java @@ -65,12 +65,8 @@ public class SolrColumnMetadataDao implements ColumnMetadataDao { Set<String> fieldBlackList = Sets.newHashSet(SolrDao.ROOT_FIELD, SolrDao.VERSION_FIELD); for (String index : indices) { - CloudSolrClient client = new CloudSolrClient.Builder().withZkHost(zkHost).build(); - client.setDefaultCollection(index); try { - SchemaRepresentation schemaRepresentation = new SchemaRequest().process(client) - .getSchemaRepresentation(); - schemaRepresentation.getFields().stream().forEach(field -> { + getIndexFields(index).forEach(field -> { String name = (String) field.get("name"); if (!fieldBlackList.contains(name)) { FieldType type = toFieldType((String) field.get("type")); @@ -108,6 +104,14 @@ public class SolrColumnMetadataDao implements ColumnMetadataDao { return indexColumnMetadata; } + protected List<Map<String, Object>> getIndexFields(String index) throws IOException, SolrServerException { + CloudSolrClient client = new CloudSolrClient.Builder().withZkHost(zkHost).build(); + client.setDefaultCollection(index); + SchemaRepresentation schemaRepresentation = new SchemaRequest().process(client) + .getSchemaRepresentation(); + return schemaRepresentation.getFields(); + } + /** * Converts a string type to the corresponding FieldType. * http://git-wip-us.apache.org/repos/asf/metron/blob/7ef4b770/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrDao.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrDao.java b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrDao.java index 46c483f..b53ae20 100644 --- a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrDao.java +++ b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrDao.java @@ -72,12 +72,12 @@ public class SolrDao implements IndexDao { @Override public void init(AccessConfig config) { if (config.getKerberosEnabled()) { - HttpClientUtil.addConfigurer(new Krb5HttpClientConfigurer()); + enableKerberos(); } if (this.client == null) { Map<String, Object> globalConfig = config.getGlobalConfigSupplier().get(); String zkHost = (String) globalConfig.get("solr.zookeeper"); - this.client = new CloudSolrClient.Builder().withZkHost((String) globalConfig.get("solr.zookeeper")).build(); + this.client = getSolrClient(zkHost); this.accessConfig = config; this.solrSearchDao = new SolrSearchDao(this.client, this.accessConfig); this.solrUpdateDao = new SolrUpdateDao(this.client); @@ -119,4 +119,12 @@ public class SolrDao implements IndexDao { public Map<String, FieldType> getColumnMetadata(List<String> indices) throws IOException { return this.solrColumnMetadataDao.getColumnMetadata(indices); } + + protected SolrClient getSolrClient(String zkHost) { + return new CloudSolrClient.Builder().withZkHost(zkHost).build(); + } + + protected void enableKerberos() { + HttpClientUtil.addConfigurer(new Krb5HttpClientConfigurer()); + } } http://git-wip-us.apache.org/repos/asf/metron/blob/7ef4b770/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrSearchDao.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrSearchDao.java b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrSearchDao.java index 2acf3c9..e336037 100644 --- a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrSearchDao.java +++ b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrSearchDao.java @@ -18,17 +18,6 @@ package org.apache.metron.solr.dao; import com.fasterxml.jackson.core.JsonProcessingException; -import java.io.IOException; -import java.lang.invoke.MethodHandles; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.stream.Collectors; - import org.apache.metron.common.Constants; import org.apache.metron.common.utils.JSONUtils; import org.apache.metron.indexing.dao.AccessConfig; @@ -51,6 +40,7 @@ import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.SolrQuery.ORDER; import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.client.solrj.response.FacetField; import org.apache.solr.client.solrj.response.FacetField.Count; import org.apache.solr.client.solrj.response.PivotField; @@ -60,6 +50,19 @@ import org.apache.solr.common.SolrDocumentList; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; + +import static org.apache.metron.common.Constants.SENSOR_TYPE; + public class SolrSearchDao implements SearchDao { private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); @@ -84,8 +87,8 @@ public class SolrSearchDao implements SearchDao { throw new InvalidSearchException( "Search result size must be less than " + accessConfig.getMaxSearchResults()); } - SolrQuery query = buildSearchRequest(searchRequest); try { + SolrQuery query = buildSearchRequest(searchRequest); QueryResponse response = client.query(query); return buildSearchResponse(searchRequest, response); } catch (IOException | SolrServerException e) { @@ -97,21 +100,21 @@ public class SolrSearchDao implements SearchDao { @Override public GroupResponse group(GroupRequest groupRequest) throws InvalidSearchException { - String groupNames = groupRequest.getGroups().stream().map(Group::getField).collect( - Collectors.joining(",")); - SolrQuery query = new SolrQuery() - .setStart(0) - .setRows(0) - .setQuery(groupRequest.getQuery()); - query.set("collection", "bro,snort"); - Optional<String> scoreField = groupRequest.getScoreField(); - if (scoreField.isPresent()) { - query.set("stats", true); - query.set("stats.field", String.format("{!tag=piv1 sum=true}%s", scoreField.get())); - } - query.set("facet", true); - query.set("facet.pivot", String.format("{!stats=piv1}%s", groupNames)); try { + String groupNames = groupRequest.getGroups().stream().map(Group::getField).collect( + Collectors.joining(",")); + SolrQuery query = new SolrQuery() + .setStart(0) + .setRows(0) + .setQuery(groupRequest.getQuery()); + query.set("collection", getCollections(groupRequest.getIndices())); + Optional<String> scoreField = groupRequest.getScoreField(); + if (scoreField.isPresent()) { + query.set("stats", true); + query.set("stats.field", String.format("{!tag=piv1 sum=true}%s", scoreField.get())); + } + query.set("facet", true); + query.set("facet.pivot", String.format("{!stats=piv1}%s", groupNames)); QueryResponse response = client.query(query); return buildGroupResponse(groupRequest, response); } catch (IOException | SolrServerException e) { @@ -152,8 +155,8 @@ public class SolrSearchDao implements SearchDao { } } - private SolrQuery buildSearchRequest( - SearchRequest searchRequest) { + protected SolrQuery buildSearchRequest( + SearchRequest searchRequest) throws IOException, SolrServerException { SolrQuery query = new SolrQuery() .setStart(searchRequest.getFrom()) .setRows(searchRequest.getSize()) @@ -176,19 +179,23 @@ public class SolrSearchDao implements SearchDao { facetFields.get().forEach(query::addFacetField); } - String collections = searchRequest.getIndices().stream().collect(Collectors.joining(",")); - query.set("collection", collections); + query.set("collection", getCollections(searchRequest.getIndices())); return query; } + private String getCollections(List<String> indices) throws IOException, SolrServerException { + List<String> existingCollections = CollectionAdminRequest.listCollections(client); + return indices.stream().filter(existingCollections::contains).collect(Collectors.joining(",")); + } + private SolrQuery.ORDER getSolrSortOrder( SortOrder sortOrder) { return sortOrder == SortOrder.DESC ? ORDER.desc : ORDER.asc; } - private SearchResponse buildSearchResponse( + protected SearchResponse buildSearchResponse( SearchRequest searchRequest, QueryResponse solrResponse) { @@ -220,7 +227,7 @@ public class SolrSearchDao implements SearchDao { return searchResponse; } - private SearchResult getSearchResult(SolrDocument solrDocument, Optional<List<String>> fields) { + protected SearchResult getSearchResult(SolrDocument solrDocument, Optional<List<String>> fields) { SearchResult searchResult = new SearchResult(); searchResult.setId((String) solrDocument.getFieldValue(Constants.GUID)); final Map<String, Object> source = new HashMap<>(); @@ -233,7 +240,7 @@ public class SolrSearchDao implements SearchDao { return searchResult; } - private Map<String, Map<String, Long>> getFacetCounts(List<String> fields, + protected Map<String, Map<String, Long>> getFacetCounts(List<String> fields, QueryResponse solrResponse) { Map<String, Map<String, Long>> fieldCounts = new HashMap<>(); for (String field : fields) { @@ -253,7 +260,7 @@ public class SolrSearchDao implements SearchDao { * @param response The search response. * @return A group response. */ - private GroupResponse buildGroupResponse( + protected GroupResponse buildGroupResponse( GroupRequest groupRequest, QueryResponse response) { String groupNames = groupRequest.getGroups().stream().map(Group::getField).collect( @@ -265,7 +272,7 @@ public class SolrSearchDao implements SearchDao { return groupResponse; } - private List<GroupResult> getGroupResults(GroupRequest groupRequest, int index, List<PivotField> pivotFields) { + protected List<GroupResult> getGroupResults(GroupRequest groupRequest, int index, List<PivotField> pivotFields) { List<Group> groups = groupRequest.getGroups(); List<GroupResult> searchResultGroups = new ArrayList<>(); final GroupOrder groupOrder = groups.get(index).getOrder(); @@ -298,13 +305,13 @@ public class SolrSearchDao implements SearchDao { return searchResultGroups; } - private Document toDocument(SolrDocument solrDocument) { + protected Document toDocument(SolrDocument solrDocument) { Map<String, Object> document = new HashMap<>(); solrDocument.getFieldNames().stream() .filter(name -> !name.equals(SolrDao.VERSION_FIELD)) .forEach(name -> document.put(name, solrDocument.getFieldValue(name))); return new Document(document, (String) solrDocument.getFieldValue(Constants.GUID), - (String) solrDocument.getFieldValue("source:type"), 0L); + (String) solrDocument.getFieldValue(SENSOR_TYPE), 0L); } } http://git-wip-us.apache.org/repos/asf/metron/blob/7ef4b770/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrColumnMetadataTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrColumnMetadataTest.java b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrColumnMetadataTest.java new file mode 100644 index 0000000..6f98809 --- /dev/null +++ b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrColumnMetadataTest.java @@ -0,0 +1,150 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.solr.dao; + +import org.apache.metron.indexing.dao.search.FieldType; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.common.SolrException; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.spy; + +public class SolrColumnMetadataTest { + + @Rule + public final ExpectedException exception = ExpectedException.none(); + + private static final String zkHost = "zookeeper:2181"; + + private SolrColumnMetadataDao solrColumnMetadataDao; + + @SuppressWarnings("unchecked") + @Before + public void setUp() throws Exception { + solrColumnMetadataDao = new SolrColumnMetadataDao(zkHost); + } + + @Test + public void getColumnMetadataShouldProperlyReturnColumnMetadata() throws Exception { + List<Map<String, Object>> broFields = new ArrayList<>(); + broFields.add(new HashMap<String, Object>(){{ + put("name", "string"); + put("type", "string"); + }}); + broFields.add(new HashMap<String, Object>(){{ + put("name", "int"); + put("type", "pint"); + }}); + broFields.add(new HashMap<String, Object>(){{ + put("name", "float"); + put("type", "pfloat"); + }}); + broFields.add(new HashMap<String, Object>(){{ + put("name", "double"); + put("type", "pdouble"); + }}); + broFields.add(new HashMap<String, Object>(){{ + put("name", "boolean"); + put("type", "boolean"); + }}); + broFields.add(new HashMap<String, Object>(){{ + put("name", "broField"); + put("type", "string"); + }}); + broFields.add(new HashMap<String, Object>(){{ + put("name", "conflict"); + put("type", "string"); + }}); + + + List<Map<String, Object>> snortFields = new ArrayList<>(); + snortFields.add(new HashMap<String, Object>(){{ + put("name", "long"); + put("type", "plong"); + }}); + snortFields.add(new HashMap<String, Object>(){{ + put("name", "snortField"); + put("type", "plong"); + }}); + snortFields.add(new HashMap<String, Object>(){{ + put("name", "unknown"); + put("type", "unknown"); + }}); + broFields.add(new HashMap<String, Object>(){{ + put("name", "conflict"); + put("type", "plong"); + }}); + + solrColumnMetadataDao = spy(new SolrColumnMetadataDao(zkHost)); + doReturn(broFields).when(solrColumnMetadataDao).getIndexFields("bro"); + doReturn(snortFields).when(solrColumnMetadataDao).getIndexFields("snort"); + + Map<String, FieldType> columnMetadata = solrColumnMetadataDao.getColumnMetadata(Arrays.asList("bro", "snort")); + + assertEquals(FieldType.BOOLEAN, columnMetadata.get("boolean")); + assertEquals(FieldType.TEXT, columnMetadata.get("string")); + assertEquals(FieldType.TEXT, columnMetadata.get("broField")); + assertEquals(FieldType.DOUBLE, columnMetadata.get("double")); + assertEquals(FieldType.LONG, columnMetadata.get("long")); + assertEquals(FieldType.FLOAT, columnMetadata.get("float")); + assertEquals(FieldType.INTEGER, columnMetadata.get("int")); + assertEquals(FieldType.LONG, columnMetadata.get("snortField")); + assertEquals(FieldType.OTHER, columnMetadata.get("conflict")); + assertEquals(FieldType.OTHER, columnMetadata.get("unknown")); + + } + + @Test + public void getColumnMetadataShouldThrowSolrException() throws Exception { + exception.expect(IOException.class); + exception.expectMessage("solr exception"); + + solrColumnMetadataDao = spy(new SolrColumnMetadataDao(zkHost)); + doThrow(new SolrServerException("solr exception")).when(solrColumnMetadataDao).getIndexFields("bro"); + + solrColumnMetadataDao.getColumnMetadata(Arrays.asList("bro", "snort")); + } + + @Test + public void getColumnMetadataShouldHandle400Exception() throws Exception { + solrColumnMetadataDao = spy(new SolrColumnMetadataDao(zkHost)); + SolrException solrException = new SolrException(SolrException.ErrorCode.BAD_REQUEST, "solr exception"); + + doThrow(solrException).when(solrColumnMetadataDao).getIndexFields("bro"); + + Map<String, FieldType> columnMetadata = solrColumnMetadataDao.getColumnMetadata(Collections.singletonList("bro")); + + assertNotNull(columnMetadata); + } + +} http://git-wip-us.apache.org/repos/asf/metron/blob/7ef4b770/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrDaoTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrDaoTest.java b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrDaoTest.java new file mode 100644 index 0000000..56e363f --- /dev/null +++ b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrDaoTest.java @@ -0,0 +1,133 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.solr.dao; + +import org.apache.metron.indexing.dao.AccessConfig; +import org.apache.metron.indexing.dao.search.GetRequest; +import org.apache.metron.indexing.dao.search.GroupRequest; +import org.apache.metron.indexing.dao.search.SearchRequest; +import org.apache.metron.indexing.dao.update.Document; +import org.apache.solr.client.solrj.SolrClient; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.runner.RunWith; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; + +import static org.mockito.Mockito.verify; +import static org.mockito.internal.verification.VerificationModeFactory.times; +import static org.powermock.api.mockito.PowerMockito.doNothing; +import static org.powermock.api.mockito.PowerMockito.doReturn; +import static org.powermock.api.mockito.PowerMockito.mock; +import static org.powermock.api.mockito.PowerMockito.spy; +import static org.powermock.api.mockito.PowerMockito.whenNew; + +@RunWith(PowerMockRunner.class) +@PrepareForTest({SolrDao.class}) +public class SolrDaoTest { + + @Rule + public final ExpectedException exception = ExpectedException.none(); + + private SolrClient client; + private SolrSearchDao solrSearchDao; + private SolrUpdateDao solrUpdateDao; + private SolrColumnMetadataDao solrColumnMetadataDao; + private SolrDao solrDao; + + @SuppressWarnings("unchecked") + @Before + public void setUp() throws Exception { + client = mock(SolrClient.class); + solrSearchDao = mock(SolrSearchDao.class); + solrUpdateDao = mock(SolrUpdateDao.class); + solrColumnMetadataDao = mock(SolrColumnMetadataDao.class); + } + + @Test + public void initShouldEnableKerberos() throws Exception { + AccessConfig accessConfig = new AccessConfig(); + + solrDao = spy(new SolrDao(client, accessConfig, solrSearchDao, solrUpdateDao, solrColumnMetadataDao)); + doNothing().when(solrDao).enableKerberos(); + + solrDao.init(accessConfig); + + verify(solrDao, times(0)).enableKerberos(); + + accessConfig.setKerberosEnabled(true); + + solrDao.init(accessConfig); + verify(solrDao).enableKerberos(); + } + + @Test + public void initShouldCreateDaos() throws Exception { + AccessConfig accessConfig = new AccessConfig(); + accessConfig.setGlobalConfigSupplier( () -> + new HashMap<String, Object>() {{ + put("solr.zookeeper", "zookeeper:2181"); + }} + ); + + solrDao = spy(new SolrDao()); + doReturn(client).when(solrDao).getSolrClient("zookeeper:2181"); + whenNew(SolrSearchDao.class).withArguments(client, accessConfig).thenReturn(solrSearchDao); + whenNew(SolrUpdateDao.class).withArguments(client).thenReturn(solrUpdateDao); + whenNew(SolrColumnMetadataDao.class).withArguments("zookeeper:2181").thenReturn(solrColumnMetadataDao); + + solrDao.init(accessConfig); + + SearchRequest searchRequest = mock(SearchRequest.class); + solrDao.search(searchRequest); + verify(solrSearchDao).search(searchRequest); + + GroupRequest groupRequest = mock(GroupRequest.class); + solrDao.group(groupRequest); + verify(solrSearchDao).group(groupRequest); + + solrDao.getLatest("guid", "collection"); + verify(solrSearchDao).getLatest("guid", "collection"); + + GetRequest getRequest1 = mock(GetRequest.class); + GetRequest getRequest2 = mock(GetRequest.class); + solrDao.getAllLatest(Arrays.asList(getRequest1, getRequest2)); + verify(solrSearchDao).getAllLatest(Arrays.asList(getRequest1, getRequest2)); + + Document document = mock(Document.class); + solrDao.update(document, Optional.of("bro")); + verify(solrUpdateDao).update(document, Optional.of("bro")); + + Map<Document, Optional<String>> updates = new HashMap<Document, Optional<String>>(){{ + put(document, Optional.of("bro")); + }}; + solrDao.batchUpdate(updates); + verify(solrUpdateDao).batchUpdate(updates); + + solrDao.getColumnMetadata(Arrays.asList("bro", "snort")); + verify(solrColumnMetadataDao).getColumnMetadata(Arrays.asList("bro", "snort")); + } + +} http://git-wip-us.apache.org/repos/asf/metron/blob/7ef4b770/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrSearchDaoTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrSearchDaoTest.java b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrSearchDaoTest.java new file mode 100644 index 0000000..762a272 --- /dev/null +++ b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrSearchDaoTest.java @@ -0,0 +1,478 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.solr.dao; + +import org.apache.metron.common.Constants; +import org.apache.metron.indexing.dao.AccessConfig; +import org.apache.metron.indexing.dao.search.GetRequest; +import org.apache.metron.indexing.dao.search.Group; +import org.apache.metron.indexing.dao.search.GroupOrder; +import org.apache.metron.indexing.dao.search.GroupRequest; +import org.apache.metron.indexing.dao.search.GroupResponse; +import org.apache.metron.indexing.dao.search.GroupResult; +import org.apache.metron.indexing.dao.search.InvalidSearchException; +import org.apache.metron.indexing.dao.search.SearchRequest; +import org.apache.metron.indexing.dao.search.SearchResponse; +import org.apache.metron.indexing.dao.search.SearchResult; +import org.apache.metron.indexing.dao.search.SortField; +import org.apache.metron.indexing.dao.update.Document; +import org.apache.metron.solr.matcher.ModifiableSolrParamsMatcher; +import org.apache.metron.solr.matcher.SolrQueryMatcher; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.client.solrj.response.FacetField; +import org.apache.solr.client.solrj.response.FieldStatsInfo; +import org.apache.solr.client.solrj.response.PivotField; +import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.common.SolrDocument; +import org.apache.solr.common.SolrDocumentList; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.common.util.NamedList; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.runner.RunWith; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.core.IsCollectionContaining.hasItems; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.argThat; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; +import static org.powermock.api.mockito.PowerMockito.mockStatic; + +@RunWith(PowerMockRunner.class) +@PrepareForTest({CollectionAdminRequest.class}) +public class SolrSearchDaoTest { + + @Rule + public final ExpectedException exception = ExpectedException.none(); + + private SolrClient client; + private AccessConfig accessConfig; + private SolrSearchDao solrSearchDao; + + @SuppressWarnings("unchecked") + @Before + public void setUp() throws Exception { + client = mock(SolrClient.class); + accessConfig = mock(AccessConfig.class); + solrSearchDao = new SolrSearchDao(client, accessConfig); + mockStatic(CollectionAdminRequest.class); + when(CollectionAdminRequest.listCollections(client)).thenReturn(Arrays.asList("bro", "snort")); + } + + @Test + public void searchShouldProperlyReturnSearchResponse() throws Exception { + SearchRequest searchRequest = mock(SearchRequest.class); + SearchResponse searchResponse = mock(SearchResponse.class); + SolrQuery solrQuery = mock(SolrQuery.class); + QueryResponse queryResponse = mock(QueryResponse.class); + + solrSearchDao = spy(new SolrSearchDao(client, accessConfig)); + when(searchRequest.getQuery()).thenReturn("query"); + doReturn(solrQuery).when(solrSearchDao).buildSearchRequest(searchRequest); + when(client.query(solrQuery)).thenReturn(queryResponse); + doReturn(searchResponse).when(solrSearchDao).buildSearchResponse(searchRequest, queryResponse); + + assertEquals(searchResponse, solrSearchDao.search(searchRequest)); + verify(solrSearchDao).buildSearchRequest(searchRequest); + verify(client).query(solrQuery); + verify(solrSearchDao).buildSearchResponse(searchRequest, queryResponse); + verifyNoMoreInteractions(client); + } + + @Test + public void searchShouldThrowInvalidSearchExceptionOnEmptyQuery() throws Exception { + exception.expect(InvalidSearchException.class); + exception.expectMessage("Search query is invalid: null"); + + solrSearchDao.search(new SearchRequest()); + } + + @Test + public void searchShouldThrowInvalidSearchExceptionOnEmptyClient() throws Exception { + exception.expect(InvalidSearchException.class); + exception.expectMessage("Uninitialized Dao! You must call init() prior to use."); + + SearchRequest searchRequest = new SearchRequest(); + searchRequest.setQuery("query"); + new SolrSearchDao(null, accessConfig).search(searchRequest); + } + + @Test + public void searchShouldThrowSearchResultSizeException() throws Exception { + exception.expect(InvalidSearchException.class); + exception.expectMessage("Search result size must be less than 100"); + + when(accessConfig.getMaxSearchResults()).thenReturn(100); + SearchRequest searchRequest = new SearchRequest(); + searchRequest.setQuery("query"); + searchRequest.setSize(200); + solrSearchDao.search(searchRequest); + } + + @Test + public void groupShouldProperlyReturnGroupResponse() throws Exception { + GroupRequest groupRequest = mock(GroupRequest.class); + QueryResponse queryResponse = mock(QueryResponse.class); + GroupResponse groupResponse = mock(GroupResponse.class); + + solrSearchDao = spy(new SolrSearchDao(client, accessConfig)); + Group group1 = new Group(); + group1.setField("field1"); + Group group2 = new Group(); + group2.setField("field2"); + when(groupRequest.getQuery()).thenReturn("query"); + when(groupRequest.getGroups()).thenReturn(Arrays.asList(group1, group2)); + when(groupRequest.getScoreField()).thenReturn(Optional.of("scoreField")); + when(groupRequest.getIndices()).thenReturn(Arrays.asList("bro", "snort")); + when(client.query(any())).thenReturn(queryResponse); + doReturn(groupResponse).when(solrSearchDao).buildGroupResponse(groupRequest, queryResponse); + SolrQuery expectedSolrQuery = new SolrQuery() + .setStart(0) + .setRows(0) + .setQuery("query"); + expectedSolrQuery.set("collection", "bro,snort"); + expectedSolrQuery.set("stats", true); + expectedSolrQuery.set("stats.field", "{!tag=piv1 sum=true}scoreField"); + expectedSolrQuery.set("facet", true); + expectedSolrQuery.set("facet.pivot", "{!stats=piv1}field1,field2"); + + assertEquals(groupResponse, solrSearchDao.group(groupRequest)); + verify(client).query(argThat(new SolrQueryMatcher(expectedSolrQuery))); + verify(solrSearchDao).buildGroupResponse(groupRequest, queryResponse); + + verifyNoMoreInteractions(client); + } + + @Test + public void getLatestShouldProperlyReturnDocument() throws Exception { + SolrDocument solrDocument = mock(SolrDocument.class); + Document document = mock(Document.class); + + solrSearchDao = spy(new SolrSearchDao(client, accessConfig)); + when(client.getById("collection", "guid")).thenReturn(solrDocument); + doReturn(document).when(solrSearchDao).toDocument(solrDocument); + + assertEquals(document, solrSearchDao.getLatest("guid", "collection")); + + verify(client).getById("collection", "guid"); + verify(solrSearchDao).toDocument(solrDocument); + verifyNoMoreInteractions(client); + } + + @Test + public void getAllLatestShouldProperlyReturnDocuments() throws Exception { + GetRequest broRequest1 = new GetRequest("bro-1", "bro"); + GetRequest broRequest2 = new GetRequest("bro-2", "bro"); + GetRequest snortRequest1 = new GetRequest("snort-1", "snort"); + GetRequest snortRequest2 = new GetRequest("snort-2", "snort"); + SolrDocument broSolrDoc1 = mock(SolrDocument.class); + SolrDocument broSolrDoc2 = mock(SolrDocument.class); + SolrDocument snortSolrDoc1 = mock(SolrDocument.class); + SolrDocument snortSolrDoc2 = mock(SolrDocument.class); + Document broDoc1 = mock(Document.class); + Document broDoc2 = mock(Document.class); + Document snortDoc1 = mock(Document.class); + Document snortDoc2 = mock(Document.class); + + solrSearchDao = spy(new SolrSearchDao(client, accessConfig)); + doReturn(broDoc1).when(solrSearchDao).toDocument(broSolrDoc1); + doReturn(broDoc2).when(solrSearchDao).toDocument(broSolrDoc2); + doReturn(snortDoc1).when(solrSearchDao).toDocument(snortSolrDoc1); + doReturn(snortDoc2).when(solrSearchDao).toDocument(snortSolrDoc2); + SolrDocumentList broList = new SolrDocumentList(); + broList.add(broSolrDoc1); + broList.add(broSolrDoc2); + SolrDocumentList snortList = new SolrDocumentList(); + snortList.add(snortSolrDoc1); + snortList.add(snortSolrDoc2); + when(client.getById((Collection<String>) argThat(hasItems("bro-1", "bro-2")), + argThat(new ModifiableSolrParamsMatcher(new ModifiableSolrParams().set("collection", "bro"))))).thenReturn(broList); + when(client.getById((Collection<String>) argThat(hasItems("snort-1", "snort-2")), + argThat(new ModifiableSolrParamsMatcher(new ModifiableSolrParams().set("collection", "snort"))))).thenReturn(snortList); + assertEquals(Arrays.asList(broDoc1, broDoc2, snortDoc1, snortDoc2), solrSearchDao.getAllLatest(Arrays.asList(broRequest1, broRequest2, snortRequest1, snortRequest2))); + } + + @Test + public void buildSearchRequestShouldReturnSolrQuery() throws Exception { + SearchRequest searchRequest = new SearchRequest(); + searchRequest.setIndices(Arrays.asList("bro", "snort")); + searchRequest.setSize(5); + searchRequest.setFrom(10); + searchRequest.setQuery("query"); + SortField sortField = new SortField(); + sortField.setField("sortField"); + sortField.setSortOrder("ASC"); + searchRequest.setSort(Collections.singletonList(sortField)); + searchRequest.setFields(Arrays.asList("field1", "field2")); + searchRequest.setFacetFields(Arrays.asList("facetField1", "facetField2")); + + SolrQuery exceptedSolrQuery = new SolrQuery() + .setStart(10) + .setRows(5) + .setQuery("query") + .addSort("sortField", SolrQuery.ORDER.asc) + .addField("field1").addField("field2") + .addFacetField("facetField1", "facetField2"); + exceptedSolrQuery.set("collection", "bro,snort"); + + SolrQuery solrQuery = solrSearchDao.buildSearchRequest(searchRequest); + assertThat(solrQuery, new SolrQueryMatcher(exceptedSolrQuery)); + } + + @Test + public void buildSearchResponseShouldReturnSearchResponse() throws Exception { + SearchRequest searchRequest = new SearchRequest(); + searchRequest.setFields(Collections.singletonList("id")); + searchRequest.setFacetFields(Collections.singletonList("facetField")); + QueryResponse queryResponse = mock(QueryResponse.class); + SolrDocument solrDocument1 = mock(SolrDocument.class); + SolrDocument solrDocument2 = mock(SolrDocument.class); + + solrSearchDao = spy(new SolrSearchDao(client, accessConfig)); + SolrDocumentList solrDocumentList = new SolrDocumentList(); + solrDocumentList.add(solrDocument1); + solrDocumentList.add(solrDocument2); + solrDocumentList.setNumFound(100); + when(queryResponse.getResults()).thenReturn(solrDocumentList); + SearchResult searchResult1 = new SearchResult(); + searchResult1.setId("id1"); + SearchResult searchResult2 = new SearchResult(); + searchResult2.setId("id2"); + doReturn(searchResult1).when(solrSearchDao).getSearchResult(solrDocument1, + Optional.of(Collections.singletonList("id"))); + doReturn(searchResult2).when(solrSearchDao).getSearchResult(solrDocument2, + Optional.of(Collections.singletonList("id"))); + Map<String, Map<String, Long>> facetCounts = new HashMap<String, Map<String, Long>>() {{ + put("id", new HashMap<String, Long>() {{ + put("id1", 1L); + put("id2", 1L); + }}); + }}; + doReturn(facetCounts).when(solrSearchDao).getFacetCounts(Collections.singletonList("facetField"), queryResponse); + SearchResponse expectedSearchResponse = new SearchResponse(); + SearchResult expectedSearchResult1 = new SearchResult(); + expectedSearchResult1.setId("id1"); + SearchResult expectedSearchResult2 = new SearchResult(); + expectedSearchResult2.setId("id2"); + expectedSearchResponse.setResults(Arrays.asList(expectedSearchResult1, expectedSearchResult2)); + expectedSearchResponse.setTotal(100); + expectedSearchResponse.setFacetCounts(facetCounts); + + assertEquals(expectedSearchResponse, solrSearchDao.buildSearchResponse(searchRequest, queryResponse)); + } + + @Test + public void getSearchResultShouldProperlyReturnResults() throws Exception { + SolrDocument solrDocument = mock(SolrDocument.class); + + when(solrDocument.getFieldValue(Constants.GUID)).thenReturn("guid"); + when(solrDocument.getFieldValue("field1")).thenReturn("value1"); + when(solrDocument.getFieldValue("field2")).thenReturn("value2"); + when(solrDocument.getFieldNames()).thenReturn(Arrays.asList("field1", "field2")); + + SearchResult expectedSearchResult = new SearchResult(); + expectedSearchResult.setId("guid"); + expectedSearchResult.setSource(new HashMap<String, Object>() {{ + put("field1", "value1"); + }}); + + assertEquals(expectedSearchResult, solrSearchDao.getSearchResult(solrDocument, + Optional.of(Collections.singletonList("field1")))); + + SearchResult expectedSearchResultAllFields = new SearchResult(); + expectedSearchResultAllFields.setId("guid"); + expectedSearchResultAllFields.setSource(new HashMap<String, Object>() {{ + put("field1", "value1"); + put("field2", "value2"); + }}); + + assertEquals(expectedSearchResultAllFields, solrSearchDao.getSearchResult(solrDocument, Optional.empty())); + } + + @Test + public void getFacetCountsShouldProperlyReturnFacetCounts() throws Exception { + QueryResponse queryResponse = mock(QueryResponse.class); + + FacetField facetField1 = new FacetField("field1"); + facetField1.add("value1", 1); + facetField1.add("value2", 2); + FacetField facetField2 = new FacetField("field2"); + facetField2.add("value3", 3); + facetField2.add("value4", 4); + when(queryResponse.getFacetField("field1")).thenReturn(facetField1); + when(queryResponse.getFacetField("field2")).thenReturn(facetField2); + + Map<String, Map<String, Long>> expectedFacetCounts = new HashMap<String, Map<String, Long>>() {{ + put("field1", new HashMap<String, Long>() {{ + put("value1", 1L); + put("value2", 2L); + }}); + put("field2", new HashMap<String, Long>() {{ + put("value3", 3L); + put("value4", 4L); + }}); + }}; + + assertEquals(expectedFacetCounts, solrSearchDao.getFacetCounts(Arrays.asList("field1", "field2"), queryResponse)); + } + + @Test + public void buildGroupResponseShouldProperlyReturnGroupReponse() throws Exception { + GroupRequest groupRequest = mock(GroupRequest.class); + QueryResponse queryResponse = mock(QueryResponse.class); + NamedList namedList = mock(NamedList.class); + List pivotFields = mock(List.class); + List groupResults = mock(List.class); + + solrSearchDao = spy(new SolrSearchDao(client, accessConfig)); + Group group1 = new Group(); + group1.setField("field1"); + Group group2 = new Group(); + group2.setField("field2"); + when(groupRequest.getGroups()).thenReturn(Arrays.asList(group1, group2)); + when(queryResponse.getFacetPivot()).thenReturn(namedList); + when(namedList.get("field1,field2")).thenReturn(pivotFields); + doReturn(groupResults).when(solrSearchDao).getGroupResults(groupRequest, 0, pivotFields); + + GroupResponse groupResponse = solrSearchDao.buildGroupResponse(groupRequest, queryResponse); + assertEquals("field1", groupResponse.getGroupedBy()); + verify(namedList).get("field1,field2"); + verify(solrSearchDao).getGroupResults(groupRequest, 0, pivotFields); + + } + + @Test + public void getGroupResultsShouldProperlyReturnGroupResults() throws Exception { + GroupRequest groupRequest = new GroupRequest(); + Group group1 = new Group(); + group1.setField("field1"); + GroupOrder groupOrder1 = new GroupOrder(); + groupOrder1.setSortOrder("ASC"); + groupOrder1.setGroupOrderType("TERM"); + group1.setOrder(groupOrder1); + Group group2 = new Group(); + group2.setField("field2"); + GroupOrder groupOrder2 = new GroupOrder(); + groupOrder2.setSortOrder("DESC"); + groupOrder2.setGroupOrderType("COUNT"); + group2.setOrder(groupOrder2); + groupRequest.setGroups(Arrays.asList(group1, group2)); + groupRequest.setScoreField("scoreField"); + + PivotField level1Pivot1 = mock(PivotField.class); + PivotField level1Pivot2 = mock(PivotField.class); + PivotField level2Pivot1 = mock(PivotField.class); + PivotField level2Pivot2 = mock(PivotField.class); + FieldStatsInfo level1Pivot1FieldStatsInfo = mock(FieldStatsInfo.class); + FieldStatsInfo level1Pivot2FieldStatsInfo = mock(FieldStatsInfo.class); + FieldStatsInfo level2Pivot1FieldStatsInfo = mock(FieldStatsInfo.class); + FieldStatsInfo level2Pivot2FieldStatsInfo = mock(FieldStatsInfo.class); + List<PivotField> level1Pivots = Arrays.asList(level1Pivot1, level1Pivot2); + List<PivotField> level2Pivots = Arrays.asList(level2Pivot1, level2Pivot2); + + when(level1Pivot1.getValue()).thenReturn("field1value1"); + when(level1Pivot1.getCount()).thenReturn(1); + when(level1Pivot1FieldStatsInfo.getSum()).thenReturn(1.0); + when(level1Pivot1.getFieldStatsInfo()).thenReturn(new HashMap<String, FieldStatsInfo>(){{ + put("score", level1Pivot1FieldStatsInfo); + }}); + when(level1Pivot2.getValue()).thenReturn("field1value2"); + when(level1Pivot2.getCount()).thenReturn(2); + when(level1Pivot2FieldStatsInfo.getSum()).thenReturn(2.0); + when(level1Pivot2.getFieldStatsInfo()).thenReturn(new HashMap<String, FieldStatsInfo>(){{ + put("score", level1Pivot2FieldStatsInfo); + }}); + when(level2Pivot1.getValue()).thenReturn("field2value1"); + when(level2Pivot1.getCount()).thenReturn(3); + when(level2Pivot1FieldStatsInfo.getSum()).thenReturn(3.0); + when(level2Pivot1.getFieldStatsInfo()).thenReturn(new HashMap<String, FieldStatsInfo>(){{ + put("score", level2Pivot1FieldStatsInfo); + }}); + when(level2Pivot2.getValue()).thenReturn("field2value2"); + when(level2Pivot2.getCount()).thenReturn(4); + when(level2Pivot2FieldStatsInfo.getSum()).thenReturn(4.0); + when(level2Pivot2.getFieldStatsInfo()).thenReturn(new HashMap<String, FieldStatsInfo>(){{ + put("score", level2Pivot2FieldStatsInfo); + }}); + when(level1Pivot1.getPivot()).thenReturn(level2Pivots); + + List<GroupResult> level1GroupResults = solrSearchDao.getGroupResults(groupRequest, 0, level1Pivots); + + assertEquals("field1value1", level1GroupResults.get(0).getKey()); + assertEquals(1, level1GroupResults.get(0).getTotal()); + assertEquals(1.0, level1GroupResults.get(0).getScore(), 0.00001); + assertEquals("field2", level1GroupResults.get(0).getGroupedBy()); + assertEquals("field1value2", level1GroupResults.get(1).getKey()); + assertEquals(2, level1GroupResults.get(1).getTotal()); + assertEquals(2.0, level1GroupResults.get(1).getScore(), 0.00001); + assertEquals("field2", level1GroupResults.get(1).getGroupedBy()); + assertEquals(0, level1GroupResults.get(1).getGroupResults().size()); + + List<GroupResult> level2GroupResults = level1GroupResults.get(0).getGroupResults(); + assertEquals("field2value2", level2GroupResults.get(0).getKey()); + assertEquals(4, level2GroupResults.get(0).getTotal()); + assertEquals(4.0, level2GroupResults.get(0).getScore(), 0.00001); + assertNull(level2GroupResults.get(0).getGroupedBy()); + assertNull(level2GroupResults.get(0).getGroupResults()); + assertEquals("field2value1", level2GroupResults.get(1).getKey()); + assertEquals(3, level2GroupResults.get(1).getTotal()); + assertEquals(3.0, level2GroupResults.get(1).getScore(), 0.00001); + assertNull(level2GroupResults.get(1).getGroupedBy()); + assertNull(level2GroupResults.get(1).getGroupResults()); + } + + @Test + public void toDocumentShouldProperlyReturnDocument() throws Exception { + SolrDocument solrDocument = new SolrDocument(); + solrDocument.addField(SolrDao.VERSION_FIELD, 1.0); + solrDocument.addField(Constants.GUID, "guid"); + solrDocument.addField(Constants.SENSOR_TYPE, "bro"); + solrDocument.addField("field", "value"); + + Document expectedDocument = new Document(new HashMap<String, Object>(){{ + put("field", "value"); + put(Constants.GUID, "guid"); + put(Constants.SENSOR_TYPE, "bro"); + }}, "guid", "bro", 0L); + + Document actualDocument = solrSearchDao.toDocument(solrDocument); + assertEquals(expectedDocument, actualDocument); + } + +} http://git-wip-us.apache.org/repos/asf/metron/blob/7ef4b770/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrUpdateDaoTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrUpdateDaoTest.java b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrUpdateDaoTest.java new file mode 100644 index 0000000..5315302 --- /dev/null +++ b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrUpdateDaoTest.java @@ -0,0 +1,135 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.solr.dao; + +import org.apache.metron.indexing.dao.update.Document; +import org.apache.metron.solr.matcher.SolrInputDocumentListMatcher; +import org.apache.metron.solr.matcher.SolrInputDocumentMatcher; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.common.SolrInputDocument; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.runner.RunWith; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; + +import static org.mockito.Matchers.argThat; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; + +@RunWith(PowerMockRunner.class) +@PrepareForTest({CollectionAdminRequest.class}) +public class SolrUpdateDaoTest { + + @Rule + public final ExpectedException exception = ExpectedException.none(); + + private SolrClient client; + private SolrUpdateDao solrUpdateDao; + + @SuppressWarnings("unchecked") + @Before + public void setUp() throws Exception { + client = mock(SolrClient.class); + solrUpdateDao = new SolrUpdateDao(client); + } + + @Test + public void updateShouldProperlyUpdateDocument() throws Exception { + Document document = new Document(new HashMap<String, Object>(){{ + put("field", "value"); + }}, "guid", "bro", 0L); + + SolrInputDocument solrInputDocument = new SolrInputDocument(); + solrInputDocument.addField("field", "value"); + + solrUpdateDao.update(document, Optional.empty()); + + verify(client).add(argThat(new SolrInputDocumentMatcher(solrInputDocument))); + + solrUpdateDao.update(document, Optional.of("bro")); + + verify(client).add(eq("bro"), argThat(new SolrInputDocumentMatcher(solrInputDocument))); + } + + @Test + public void batchUpdateShouldProperlyUpdateDocuments() throws Exception { + Document broDocument1 = new Document(new HashMap<String, Object>(){{ + put("broField1", "value"); + put("guid", "broGuid1"); + }}, "broGuid1", "bro", 0L); + Document broDocument2 = new Document(new HashMap<String, Object>(){{ + put("broField2", "value"); + put("guid", "broGuid2"); + }}, "broGuid2", "bro", 0L); + + Map<Document, Optional<String>> updates = new HashMap<Document, Optional<String>>(){{ + put(broDocument1, Optional.of("bro")); + put(broDocument2, Optional.of("bro")); + }}; + + SolrInputDocument broSolrInputDocument1 = new SolrInputDocument(); + broSolrInputDocument1.addField("broField1", "value"); + broSolrInputDocument1.addField("guid", "broGuid1"); + SolrInputDocument broSolrInputDocument2 = new SolrInputDocument(); + broSolrInputDocument2.addField("broField2", "value"); + broSolrInputDocument2.addField("guid", "broGuid2"); + + solrUpdateDao.batchUpdate(updates); + + verify(client).add(eq("bro"), argThat(new SolrInputDocumentListMatcher(Arrays.asList(broSolrInputDocument1, broSolrInputDocument2)))); + } + + @Test + public void batchUpdateShouldProperlyUpdateDocumentsWithoutIndex() throws Exception { + Document snortDocument1 = new Document(new HashMap<String, Object>(){{ + put("snortField1", "value"); + put("guid", "snortGuid1"); + }}, "snortGuid1", "snort", 0L); + Document snortDocument2 = new Document(new HashMap<String, Object>(){{ + put("snortField2", "value"); + put("guid", "snortGuid2"); + }}, "snortGuid2", "snort", 0L); + + Map<Document, Optional<String>> updates = new HashMap<Document, Optional<String>>(){{ + put(snortDocument1, Optional.empty()); + put(snortDocument2, Optional.empty()); + }}; + + SolrInputDocument snortSolrInputDocument1 = new SolrInputDocument(); + snortSolrInputDocument1.addField("snortField1", "value"); + snortSolrInputDocument1.addField("guid", "snortGuid1"); + SolrInputDocument snortSolrInputDocument2 = new SolrInputDocument(); + snortSolrInputDocument2.addField("snortField2", "value"); + snortSolrInputDocument2.addField("guid", "snortGuid2"); + + solrUpdateDao.batchUpdate(updates); + + verify(client).add(argThat(new SolrInputDocumentListMatcher(Arrays.asList(snortSolrInputDocument1, snortSolrInputDocument2)))); + } + +} http://git-wip-us.apache.org/repos/asf/metron/blob/7ef4b770/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/matcher/ModifiableSolrParamsMatcher.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/matcher/ModifiableSolrParamsMatcher.java b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/matcher/ModifiableSolrParamsMatcher.java new file mode 100644 index 0000000..cd68be9 --- /dev/null +++ b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/matcher/ModifiableSolrParamsMatcher.java @@ -0,0 +1,55 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.solr.matcher; + +import org.apache.solr.common.params.ModifiableSolrParams; +import org.hamcrest.Description; +import org.mockito.ArgumentMatcher; + +public class ModifiableSolrParamsMatcher extends ArgumentMatcher<ModifiableSolrParams> { + + private ModifiableSolrParams expectedModifiableSolrParams; + + public ModifiableSolrParamsMatcher(ModifiableSolrParams modifiableSolrParams) { + this.expectedModifiableSolrParams = modifiableSolrParams; + } + + @Override + public boolean matches(Object o) { + ModifiableSolrParams modifiableSolrParams = (ModifiableSolrParams) o; + for(String name: expectedModifiableSolrParams.getParameterNames()) { + String expectedValue = expectedModifiableSolrParams.get(name); + String value = modifiableSolrParams.get(name); + if(expectedValue == null) { + if (value != null) { + return false; + } + } else { + if (!expectedValue.equals(value)) { + return false; + } + } + } + return true; + } + + @Override + public void describeTo(Description description) { + description.appendValue(expectedModifiableSolrParams); + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/7ef4b770/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/matcher/SolrInputDocumentListMatcher.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/matcher/SolrInputDocumentListMatcher.java b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/matcher/SolrInputDocumentListMatcher.java new file mode 100644 index 0000000..6c4ab20 --- /dev/null +++ b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/matcher/SolrInputDocumentListMatcher.java @@ -0,0 +1,60 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.solr.matcher; + +import org.apache.solr.common.SolrInputDocument; +import org.hamcrest.Description; +import org.mockito.ArgumentMatcher; + +import java.util.List; + +public class SolrInputDocumentListMatcher extends ArgumentMatcher<List<SolrInputDocument>> { + + private List<SolrInputDocument> expectedSolrInputDocuments; + + public SolrInputDocumentListMatcher(List<SolrInputDocument> solrInputDocuments) { + this.expectedSolrInputDocuments = solrInputDocuments; + } + + @Override + public boolean matches(Object o) { + List<SolrInputDocument> solrInputDocuments = (List<SolrInputDocument>) o; + for(int i = 0; i < solrInputDocuments.size(); i++) { + SolrInputDocument solrInputDocument = solrInputDocuments.get(i); + for (int j = 0; j < expectedSolrInputDocuments.size(); j++) { + SolrInputDocument expectedSolrInputDocument = expectedSolrInputDocuments.get(j); + if (solrInputDocument.get("guid").equals(expectedSolrInputDocument.get("guid"))) { + for(String field: solrInputDocument.getFieldNames()) { + Object expectedValue = expectedSolrInputDocument.getField(field).getValue(); + Object value = solrInputDocument.getField(field).getValue(); + boolean matches = expectedValue != null ? expectedValue.equals(value) : value == null; + if (!matches) { + return false; + } + } + } + } + } + return true; + } + + @Override + public void describeTo(Description description) { + description.appendValue(expectedSolrInputDocuments); + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/7ef4b770/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/matcher/SolrInputDocumentMatcher.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/matcher/SolrInputDocumentMatcher.java b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/matcher/SolrInputDocumentMatcher.java new file mode 100644 index 0000000..b64c9f2 --- /dev/null +++ b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/matcher/SolrInputDocumentMatcher.java @@ -0,0 +1,50 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.solr.matcher; + +import org.apache.solr.common.SolrInputDocument; +import org.hamcrest.Description; +import org.mockito.ArgumentMatcher; + +public class SolrInputDocumentMatcher extends ArgumentMatcher<SolrInputDocument> { + + private SolrInputDocument expectedSolrInputDocument; + + public SolrInputDocumentMatcher(SolrInputDocument solrInputDocument) { + this.expectedSolrInputDocument = solrInputDocument; + } + + @Override + public boolean matches(Object o) { + SolrInputDocument solrInputDocument = (SolrInputDocument) o; + for(String field: solrInputDocument.getFieldNames()) { + Object expectedValue = expectedSolrInputDocument.getField(field).getValue(); + Object value = solrInputDocument.getField(field).getValue(); + boolean matches = expectedValue != null ? expectedValue.equals(value) : value == null; + if (!matches) { + return false; + } + } + return true; + } + + @Override + public void describeTo(Description description) { + description.appendValue(expectedSolrInputDocument); + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/7ef4b770/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/matcher/SolrQueryMatcher.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/matcher/SolrQueryMatcher.java b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/matcher/SolrQueryMatcher.java new file mode 100644 index 0000000..45bf85d --- /dev/null +++ b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/matcher/SolrQueryMatcher.java @@ -0,0 +1,56 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.solr.matcher; + +import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.hamcrest.Description; +import org.mockito.ArgumentMatcher; + +import java.util.Arrays; +import java.util.Objects; + +public class SolrQueryMatcher extends ArgumentMatcher<ModifiableSolrParams> { + + private SolrQuery expectedSolrQuery; + + public SolrQueryMatcher(SolrQuery solrQuery) { + this.expectedSolrQuery = solrQuery; + } + + @Override + public boolean matches(Object o) { + SolrQuery solrQuery = (SolrQuery) o; + return Objects.equals(solrQuery.getStart(), expectedSolrQuery.getStart()) && + Objects.equals(solrQuery.getRows(), expectedSolrQuery.getRows()) && + Objects.equals(solrQuery.getQuery(), expectedSolrQuery.getQuery()) && + Objects.equals(solrQuery.getSorts(), expectedSolrQuery.getSorts()) && + Objects.equals(solrQuery.getFields(), expectedSolrQuery.getFields()) && + Arrays.equals(solrQuery.getFacetFields(), expectedSolrQuery.getFacetFields()) && + Objects.equals(solrQuery.get("collection"), expectedSolrQuery.get("collection")) && + Objects.equals(solrQuery.get("stats"), expectedSolrQuery.get("stats")) && + Objects.equals(solrQuery.get("stats.field"), expectedSolrQuery.get("stats.field")) && + Objects.equals(solrQuery.get("facet"), expectedSolrQuery.get("facet")) && + Objects.equals(solrQuery.get("facet.pivot"), expectedSolrQuery.get("facet.pivot")); + } + + @Override + public void describeTo(Description description) { + description.appendValue(expectedSolrQuery); + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/7ef4b770/metron-platform/pom.xml ---------------------------------------------------------------------- diff --git a/metron-platform/pom.xml b/metron-platform/pom.xml index 3899e9d..594cb46 100644 --- a/metron-platform/pom.xml +++ b/metron-platform/pom.xml @@ -82,13 +82,13 @@ <dependency> <groupId>org.powermock</groupId> <artifactId>powermock-module-junit4</artifactId> - <version>1.6.6</version> + <version>${global_powermock_version}</version> <scope>test</scope> </dependency> <dependency> <groupId>org.powermock</groupId> <artifactId>powermock-api-mockito</artifactId> - <version>1.6.6</version> + <version>${global_powermock_version}</version> <scope>test</scope> </dependency> <dependency> http://git-wip-us.apache.org/repos/asf/metron/blob/7ef4b770/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 38738b5..b80785f 100644 --- a/pom.xml +++ b/pom.xml @@ -109,6 +109,7 @@ <global_java_version>1.8</global_java_version> <global_solr_version>6.6.2</global_solr_version> <global_mockito_version>1.10.19</global_mockito_version> + <global_powermock_version>1.7.0</global_powermock_version> <global_shade_version>2.4.3</global_shade_version> <global_jackson_version>2.7.4</global_jackson_version> <global_errorprone_core_version>2.0.14</global_errorprone_core_version>
