METRON-1585 SolrRetrieveLatestDao does not use the collection lookup (justinleet via merrimanr) closes apache/metron#1050
Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/aed4ffcb Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/aed4ffcb Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/aed4ffcb Branch: refs/heads/feature/METRON-1554-pcap-query-panel Commit: aed4ffcb24cf9a95c1347cccd5dd79fce7c77a6b Parents: 95e6528 Author: justinleet <justinjl...@gmail.com> Authored: Fri Jun 8 12:16:59 2018 -0500 Committer: merrimanr <merrim...@gmail.com> Committed: Fri Jun 8 12:16:59 2018 -0500 ---------------------------------------------------------------------- .../org/apache/metron/solr/dao/SolrDao.java | 10 +- .../metron/solr/dao/SolrRetrieveLatestDao.java | 37 +++- .../apache/metron/solr/dao/SolrSearchDao.java | 17 -- .../apache/metron/solr/dao/SolrUpdateDao.java | 18 +- .../apache/metron/solr/dao/SolrUtilities.java | 18 ++ .../org/apache/metron/solr/dao/SolrDaoTest.java | 4 +- .../metron/solr/dao/SolrSearchDaoTest.java | 2 +- .../metron/solr/dao/SolrUpdateDaoTest.java | 4 +- .../SolrRetrieveLatestIntegrationTest.java | 207 +++++++++++++++++++ .../integration/SolrUpdateIntegrationTest.java | 1 + 10 files changed, 266 insertions(+), 52 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/aed4ffcb/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrDao.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrDao.java b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrDao.java index 7db0ab5..a840bb4 100644 --- a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrDao.java +++ b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrDao.java @@ -22,16 +22,13 @@ import static org.apache.metron.solr.SolrConstants.SOLR_ZOOKEEPER; import com.google.common.base.Splitter; import java.io.IOException; import java.lang.invoke.MethodHandles; -import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.function.Function; import org.apache.metron.indexing.dao.AccessConfig; import org.apache.metron.indexing.dao.ColumnMetadataDao; import org.apache.metron.indexing.dao.IndexDao; import org.apache.metron.indexing.dao.RetrieveLatestDao; -import org.apache.metron.indexing.dao.search.AlertComment; import org.apache.metron.indexing.dao.search.FieldType; import org.apache.metron.indexing.dao.search.GetRequest; import org.apache.metron.indexing.dao.search.GroupRequest; @@ -43,7 +40,6 @@ import org.apache.metron.indexing.dao.update.CommentAddRemoveRequest; import org.apache.metron.indexing.dao.update.Document; import org.apache.metron.indexing.dao.update.OriginalNotFoundException; import org.apache.metron.indexing.dao.update.PatchRequest; -import org.apache.metron.solr.SolrConstants; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.impl.CloudSolrClient; import org.apache.solr.client.solrj.impl.HttpClientUtil; @@ -92,7 +88,7 @@ public class SolrDao implements IndexDao { this.accessConfig = config; this.client = getSolrClient(getZkHosts()); this.solrSearchDao = new SolrSearchDao(this.client, this.accessConfig); - this.solrRetrieveLatestDao = new SolrRetrieveLatestDao(this.client); + this.solrRetrieveLatestDao = new SolrRetrieveLatestDao(this.client, this.accessConfig); this.solrUpdateDao = new SolrUpdateDao(this.client, this.solrRetrieveLatestDao, this.accessConfig); this.solrColumnMetadataDao = new SolrColumnMetadataDao(this.client); } @@ -118,8 +114,8 @@ public class SolrDao implements IndexDao { } @Override - public Document getLatest(String guid, String collection) throws IOException { - return this.solrRetrieveLatestDao.getLatest(guid, collection); + public Document getLatest(String guid, String sensorType) throws IOException { + return this.solrRetrieveLatestDao.getLatest(guid, sensorType); } @Override http://git-wip-us.apache.org/repos/asf/metron/blob/aed4ffcb/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrRetrieveLatestDao.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrRetrieveLatestDao.java b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrRetrieveLatestDao.java index b3bc564..40262ba 100644 --- a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrRetrieveLatestDao.java +++ b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrRetrieveLatestDao.java @@ -19,13 +19,16 @@ package org.apache.metron.solr.dao; import java.io.IOException; +import java.lang.invoke.MethodHandles; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.stream.Collectors; +import org.apache.metron.indexing.dao.AccessConfig; import org.apache.metron.indexing.dao.RetrieveLatestDao; import org.apache.metron.indexing.dao.search.GetRequest; import org.apache.metron.indexing.dao.update.Document; @@ -34,21 +37,34 @@ import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrDocumentList; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class SolrRetrieveLatestDao implements RetrieveLatestDao { + private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + private transient SolrClient client; + private AccessConfig config; - public SolrRetrieveLatestDao(SolrClient client) { + public SolrRetrieveLatestDao(SolrClient client, AccessConfig config) { this.client = client; + this.config = config; } @Override - public Document getLatest(String guid, String collection) throws IOException { - + public Document getLatest(String guid, String sensorType) throws IOException { try { - SolrDocument solrDocument = client.getById(collection, guid); + Optional<String> index = SolrUtilities + .getIndex(config.getIndexSupplier(), sensorType, Optional.empty()); + if (!index.isPresent()) { + LOG.debug("Unable to find index for sensorType {}", sensorType); + return null; + } + + SolrDocument solrDocument = client.getById(index.get(), guid); if (solrDocument == null) { + LOG.debug("Unable to find document for sensorType {} and guid {}", sensorType, guid); return null; } return SolrUtilities.toDocument(solrDocument); @@ -61,10 +77,15 @@ public class SolrRetrieveLatestDao implements RetrieveLatestDao { public Iterable<Document> getAllLatest(List<GetRequest> getRequests) throws IOException { Map<String, Collection<String>> collectionIdMap = new HashMap<>(); for (GetRequest getRequest : getRequests) { - Collection<String> ids = collectionIdMap - .getOrDefault(getRequest.getSensorType(), new HashSet<>()); - ids.add(getRequest.getGuid()); - collectionIdMap.put(getRequest.getSensorType(), ids); + Optional<String> index = SolrUtilities + .getIndex(config.getIndexSupplier(), getRequest.getSensorType(), getRequest.getIndex()); + if (index.isPresent()) { + Collection<String> ids = collectionIdMap.getOrDefault(index.get(), new HashSet<>()); + ids.add(getRequest.getGuid()); + collectionIdMap.put(index.get(), ids); + } else { + LOG.debug("Unable to find index for sensorType {}", getRequest.getSensorType()); + } } try { List<Document> documents = new ArrayList<>(); http://git-wip-us.apache.org/repos/asf/metron/blob/aed4ffcb/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrSearchDao.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrSearchDao.java b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrSearchDao.java index d978ec9..4a8d482 100644 --- a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrSearchDao.java +++ b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrSearchDao.java @@ -17,24 +17,10 @@ */ package org.apache.metron.solr.dao; -import static org.apache.metron.common.Constants.SENSOR_TYPE; -import static org.apache.metron.indexing.dao.IndexDao.COMMENTS_FIELD; - import com.fasterxml.jackson.core.JsonProcessingException; import java.io.IOException; import java.lang.invoke.MethodHandles; import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.stream.Collectors; -import org.apache.metron.common.Constants; -import java.io.IOException; -import java.lang.invoke.MethodHandles; -import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -43,8 +29,6 @@ import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; import org.apache.metron.common.utils.JSONUtils; import org.apache.metron.indexing.dao.AccessConfig; -import org.apache.metron.indexing.dao.search.AlertComment; -import org.apache.metron.indexing.dao.search.GetRequest; import org.apache.metron.indexing.dao.search.Group; import org.apache.metron.indexing.dao.search.GroupOrder; import org.apache.metron.indexing.dao.search.GroupOrderType; @@ -69,7 +53,6 @@ import org.apache.solr.client.solrj.response.PivotField; import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.common.SolrDocumentList; import org.apache.solr.common.SolrException; -import org.json.simple.parser.ParseException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; http://git-wip-us.apache.org/repos/asf/metron/blob/aed4ffcb/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUpdateDao.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUpdateDao.java b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUpdateDao.java index 7f5a4ed..2f83921 100644 --- a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUpdateDao.java +++ b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUpdateDao.java @@ -30,7 +30,6 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Optional; import java.util.Set; -import java.util.function.Function; import java.util.stream.Collectors; import org.apache.metron.indexing.dao.AccessConfig; import org.apache.metron.indexing.dao.search.AlertComment; @@ -48,23 +47,12 @@ public class SolrUpdateDao implements UpdateDao { private transient SolrClient client; private AccessConfig config; - private Function<String, String> indexSupplier; private transient SolrRetrieveLatestDao retrieveLatestDao; public SolrUpdateDao(SolrClient client, SolrRetrieveLatestDao retrieveLatestDao, AccessConfig config) { this.client = client; this.retrieveLatestDao = retrieveLatestDao; this.config = config; - this.indexSupplier = config.getIndexSupplier(); - } - - private Optional<String> getIndex(String sensorName, Optional<String> index) { - if (index.isPresent()) { - return index; - } else { - String realIndex = indexSupplier.apply(sensorName); - return Optional.ofNullable(realIndex); - } } @Override @@ -80,7 +68,8 @@ public class SolrUpdateDao implements UpdateDao { } try { SolrInputDocument solrInputDocument = SolrUtilities.toSolrInputDocument(newVersion); - Optional<String> index = getIndex(update.getSensorType(), rawIndex); + Optional<String> index = SolrUtilities + .getIndex(config.getIndexSupplier(), newVersion.getSensorType(), rawIndex); if (index.isPresent()) { this.client.add(index.get(), solrInputDocument); this.client.commit(index.get()); @@ -100,7 +89,8 @@ public class SolrUpdateDao implements UpdateDao { for (Entry<Document, Optional<String>> entry : updates.entrySet()) { SolrInputDocument solrInputDocument = SolrUtilities.toSolrInputDocument(entry.getKey()); - Optional<String> index = getIndex(entry.getKey().getSensorType(), entry.getValue()); + Optional<String> index = SolrUtilities + .getIndex(config.getIndexSupplier(), entry.getKey().getSensorType(), entry.getValue()); if (index.isPresent()) { Collection<SolrInputDocument> solrInputDocuments = solrCollectionUpdates .getOrDefault(index.get(), new ArrayList<>()); http://git-wip-us.apache.org/repos/asf/metron/blob/aed4ffcb/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUtilities.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUtilities.java b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUtilities.java index 88146b0..d41b7e4 100644 --- a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUtilities.java +++ b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUtilities.java @@ -24,6 +24,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.function.Function; import java.util.stream.Collectors; @@ -122,4 +123,21 @@ public class SolrUtilities { } return solrInputDocument; } + + /** + * Gets the actual collection for the given sensor type + * @param indexSupplier The function to employ in the lookup + * @param sensorName The sensor type to be looked up + * @param index An index to use, if present. + * @return An Optional containing the actual collection + */ + public static Optional<String> getIndex(Function<String, String> indexSupplier, String sensorName, + Optional<String> index) { + if (index.isPresent()) { + return index; + } else { + String realIndex = indexSupplier.apply(sensorName); + return Optional.ofNullable(realIndex); + } + } } http://git-wip-us.apache.org/repos/asf/metron/blob/aed4ffcb/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrDaoTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrDaoTest.java b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrDaoTest.java index 85523b2..9d84669 100644 --- a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrDaoTest.java +++ b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrDaoTest.java @@ -35,8 +35,6 @@ import java.util.List; import java.util.Map; import java.util.Optional; import org.apache.metron.indexing.dao.AccessConfig; -import org.apache.metron.indexing.dao.IndexDao; -import org.apache.metron.indexing.dao.search.AlertComment; import org.apache.metron.indexing.dao.search.GetRequest; import org.apache.metron.indexing.dao.search.GroupRequest; import org.apache.metron.indexing.dao.search.SearchRequest; @@ -109,7 +107,7 @@ public class SolrDaoTest { solrDao = spy(new SolrDao()); doReturn(client).when(solrDao).getSolrClient(Collections.singletonList("zookeeper:2181")); whenNew(SolrSearchDao.class).withArguments(client, accessConfig).thenReturn(solrSearchDao); - whenNew(SolrRetrieveLatestDao.class).withArguments(client) + whenNew(SolrRetrieveLatestDao.class).withArguments(client, accessConfig) .thenReturn(solrRetrieveLatestDao); whenNew(SolrUpdateDao.class).withArguments(client, solrRetrieveLatestDao, accessConfig) .thenReturn(solrUpdateDao); http://git-wip-us.apache.org/repos/asf/metron/blob/aed4ffcb/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrSearchDaoTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrSearchDaoTest.java b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrSearchDaoTest.java index d17bf72..fe27a55 100644 --- a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrSearchDaoTest.java +++ b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrSearchDaoTest.java @@ -92,7 +92,7 @@ public class SolrSearchDaoTest { accessConfig = mock(AccessConfig.class); when(accessConfig.getIndexSupplier()).thenReturn(sensorType -> sensorType); solrSearchDao = new SolrSearchDao(client, accessConfig); - solrRetrieveLatestDao = new SolrRetrieveLatestDao(client); + solrRetrieveLatestDao = new SolrRetrieveLatestDao(client, accessConfig); mockStatic(CollectionAdminRequest.class); when(CollectionAdminRequest.listCollections(client)).thenReturn(Arrays.asList("bro", "snort")); } http://git-wip-us.apache.org/repos/asf/metron/blob/aed4ffcb/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrUpdateDaoTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrUpdateDaoTest.java b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrUpdateDaoTest.java index 7de02c1..bed43ae 100644 --- a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrUpdateDaoTest.java +++ b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrUpdateDaoTest.java @@ -95,7 +95,7 @@ public class SolrUpdateDaoTest { @Before public void setUp() throws Exception { client = mock(SolrClient.class); - solrRetrieveLatestDao = new SolrRetrieveLatestDao(client); + solrRetrieveLatestDao = new SolrRetrieveLatestDao(client, accessConfig); solrUpdateDao = new SolrUpdateDao(client, solrRetrieveLatestDao, accessConfig); } @@ -217,7 +217,7 @@ public class SolrUpdateDaoTest { latestDoc.put(COMMENTS_FIELD, comments); Document latest = new Document(latestDoc, "guid", "bro", 0L); - SolrRetrieveLatestDao retrieveLatestDao = spy(new SolrRetrieveLatestDao(null)); + SolrRetrieveLatestDao retrieveLatestDao = spy(new SolrRetrieveLatestDao(null, accessConfig)); doReturn(latest).when(retrieveLatestDao).getLatest("guid", "bro"); // Create the patch http://git-wip-us.apache.org/repos/asf/metron/blob/aed4ffcb/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrRetrieveLatestIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrRetrieveLatestIntegrationTest.java b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrRetrieveLatestIntegrationTest.java new file mode 100644 index 0000000..f7c2e86 --- /dev/null +++ b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrRetrieveLatestIntegrationTest.java @@ -0,0 +1,207 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.solr.integration; + +import static org.apache.metron.solr.SolrConstants.SOLR_ZOOKEEPER; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import com.google.common.collect.Iterables; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.metron.common.Constants; +import org.apache.metron.indexing.dao.AccessConfig; +import org.apache.metron.indexing.dao.IndexDao; +import org.apache.metron.indexing.dao.search.GetRequest; +import org.apache.metron.indexing.dao.update.Document; +import org.apache.metron.solr.dao.SolrDao; +import org.apache.metron.solr.integration.components.SolrComponent; +import org.apache.solr.client.solrj.SolrServerException; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +public class SolrRetrieveLatestIntegrationTest { + + private static SolrComponent solrComponent; + + protected static final String TEST_COLLECTION = "test"; + protected static final String TEST_SENSOR = "test_sensor"; + protected static final String BRO_SENSOR = "bro"; + + private static IndexDao dao; + + @BeforeClass + public static void setupBeforeClass() throws Exception { + solrComponent = new SolrComponent.Builder().build(); + solrComponent.start(); + } + + @Before + public void setup() throws Exception { + solrComponent + .addCollection(TEST_COLLECTION, "../metron-solr/src/test/resources/config/test/conf"); + solrComponent.addCollection(BRO_SENSOR, "../metron-solr/src/main/config/schema/bro"); + + AccessConfig accessConfig = new AccessConfig(); + Map<String, Object> globalConfig = new HashMap<>(); + globalConfig.put(SOLR_ZOOKEEPER, solrComponent.getZookeeperUrl()); + accessConfig.setGlobalConfigSupplier(() -> globalConfig); + // Map the sensor name to the collection name for test. + accessConfig.setIndexSupplier(s -> s.equals(TEST_SENSOR) ? TEST_COLLECTION : s); + + dao = new SolrDao(); + dao.init(accessConfig); + addData(BRO_SENSOR, BRO_SENSOR); + addData(TEST_COLLECTION, TEST_SENSOR); + } + + @After + public void reset() { + solrComponent.reset(); + } + + @AfterClass + public static void teardown() { + solrComponent.stop(); + } + + @Test + public void testGetLatest() throws IOException { + Document actual = dao.getLatest("message_1_bro", BRO_SENSOR); + assertEquals(buildExpectedDocument(BRO_SENSOR, 1), actual); + } + + @Test + public void testGetMissing() throws IOException { + Document actual = dao.getLatest("message_1_bro", TEST_SENSOR); + assertNull(actual); + } + + @Test + public void testGetBrokenMapping() throws IOException { + AccessConfig accessConfig = new AccessConfig(); + Map<String, Object> globalConfig = new HashMap<>(); + globalConfig.put(SOLR_ZOOKEEPER, solrComponent.getZookeeperUrl()); + accessConfig.setGlobalConfigSupplier(() -> globalConfig); + // Map the sensor name to the collection name for test. + accessConfig.setIndexSupplier(s -> null); + + dao = new SolrDao(); + dao.init(accessConfig); + + Document actual = dao.getLatest("message_1_bro", TEST_SENSOR); + assertNull(actual); + } + + @Test + public void testGetLatestCollectionSensorDiffer() throws IOException { + Document actual = dao.getLatest("message_1_test_sensor", TEST_SENSOR); + assertEquals(buildExpectedDocument(TEST_SENSOR, 1), actual); + } + + @Test + public void testGetAllLatest() throws IOException { + List<GetRequest> requests = new ArrayList<>(); + requests.add(buildGetRequest(BRO_SENSOR, 1)); + requests.add(buildGetRequest(BRO_SENSOR, 2)); + + Iterable<Document> actual = dao.getAllLatest(requests); + assertTrue(Iterables.contains(actual, buildExpectedDocument(BRO_SENSOR, 1))); + assertTrue(Iterables.contains(actual, buildExpectedDocument(BRO_SENSOR, 2))); + assertEquals(2, Iterables.size(actual)); + } + + @Test + public void testGetAllLatestCollectionExplicitIndex() throws IOException { + List<GetRequest> requests = new ArrayList<>(); + GetRequest getRequestOne = buildGetRequest(TEST_SENSOR, 1); + // Explicitly use the incorrect index. This forces it to prefer the explicit index over the + // implicit one. + getRequestOne.setIndex(BRO_SENSOR); + requests.add(getRequestOne); + + Iterable<Document> actual = dao.getAllLatest(requests); + // Expect 0 because the explicit index was incorrect. + assertEquals(0, Iterables.size(actual)); + } + + @Test + public void testGetAllLatestCollectionSensorMixed() throws IOException { + List<GetRequest> requests = new ArrayList<>(); + requests.add(buildGetRequest(TEST_SENSOR, 1)); + requests.add(buildGetRequest(BRO_SENSOR, 2)); + + Iterable<Document> actual = dao.getAllLatest(requests); + assertTrue(Iterables.contains(actual, buildExpectedDocument(TEST_SENSOR, 1))); + assertTrue(Iterables.contains(actual, buildExpectedDocument(BRO_SENSOR, 2))); + assertEquals(2, Iterables.size(actual)); + } + + @Test + public void testGetAllLatestCollectionOneMissing() throws IOException { + List<GetRequest> requests = new ArrayList<>(); + requests.add(buildGetRequest(TEST_SENSOR, 1)); + GetRequest brokenRequest= new GetRequest(); + brokenRequest.setGuid(buildGuid(BRO_SENSOR, 2)); + brokenRequest.setSensorType(TEST_SENSOR); + requests.add(brokenRequest); + + Iterable<Document> actual = dao.getAllLatest(requests); + assertTrue(Iterables.contains(actual, buildExpectedDocument(TEST_SENSOR, 1))); + assertEquals(1, Iterables.size(actual)); + } + + protected Document buildExpectedDocument(String sensor, int i) { + Map<String, Object> expectedMapOne = new HashMap<>(); + expectedMapOne.put("source.type", sensor); + expectedMapOne.put(Constants.GUID, buildGuid(sensor, i)); + return new Document(expectedMapOne, buildGuid(sensor, i), sensor, 0L); + } + + protected GetRequest buildGetRequest(String sensor, int i) { + GetRequest requestOne = new GetRequest(); + requestOne.setGuid(buildGuid(sensor, i)); + requestOne.setSensorType(sensor); + return requestOne; + } + + protected static void addData(String collection, String sensorName) + throws IOException, SolrServerException { + List<Map<String, Object>> inputData = new ArrayList<>(); + for (int i = 0; i < 3; ++i) { + final String name = buildGuid(sensorName, i); + HashMap<String, Object> inputMap = new HashMap<>(); + inputMap.put("source.type", sensorName); + inputMap.put(Constants.GUID, name); + inputData.add(inputMap); + } + solrComponent.addDocs(collection, inputData); + } + + protected static String buildGuid(String sensorName, int i) { + return "message_" + i + "_" + sensorName; + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/aed4ffcb/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrUpdateIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrUpdateIntegrationTest.java b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrUpdateIntegrationTest.java index c0697b8..5b96559 100644 --- a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrUpdateIntegrationTest.java +++ b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrUpdateIntegrationTest.java @@ -84,6 +84,7 @@ public class SolrUpdateIntegrationTest extends UpdateIntegrationTest { globalConfig.put(HBaseDao.HBASE_TABLE, TABLE_NAME); globalConfig.put(HBaseDao.HBASE_CF, CF); accessConfig.setGlobalConfigSupplier(() -> globalConfig); + accessConfig.setIndexSupplier(s -> s); CuratorFramework client = ConfigurationsUtils .getClient(solrComponent.getZookeeperUrl());