This is an automated email from the ASF dual-hosted git repository. nickallen pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/metron.git
The following commit(s) were added to refs/heads/master by this push: new ed63eac METRON-2172 Solr Updates Not Tested in Integration Test (nickwallen) closes apache/metron#1465 ed63eac is described below commit ed63eac260e442f8c6f17dd93f6d87fc946cae0d Author: nickwallen <n...@nickallen.org> AuthorDate: Fri Jul 19 13:00:38 2019 -0400 METRON-2172 Solr Updates Not Tested in Integration Test (nickwallen) closes apache/metron#1465 --- .../ElasticsearchUpdateIntegrationTest.java | 5 -- .../metron/indexing/dao/UpdateIntegrationTest.java | 84 +++++++++++++++++----- .../integration/HBaseDaoIntegrationTest.java | 5 -- .../org/apache/metron/solr/dao/SolrUtilities.java | 5 +- .../apache/metron/solr/dao/SolrSearchDaoTest.java | 22 ++++-- .../apache/metron/solr/dao/SolrUtilitiesTest.java | 5 +- .../SolrRetrieveLatestIntegrationTest.java | 22 +++--- .../integration/SolrUpdateIntegrationTest.java | 54 +++++--------- 8 files changed, 122 insertions(+), 80 deletions(-) diff --git a/metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchUpdateIntegrationTest.java b/metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchUpdateIntegrationTest.java index 16760d2..97afd41 100644 --- a/metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchUpdateIntegrationTest.java +++ b/metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchUpdateIntegrationTest.java @@ -160,11 +160,6 @@ public class ElasticsearchUpdateIntegrationTest extends UpdateIntegrationTest { return es.getAllIndexedDocs(index, SENSOR_NAME + "_doc"); } - @Override - protected MockHTable getMockHTable() { - return table; - } - /** * Install the index template to ensure that "guid" is of type "keyword". The * {@link org.apache.metron.elasticsearch.dao.ElasticsearchRetrieveLatestDao} cannot find diff --git a/metron-platform/metron-indexing/metron-indexing-common/src/test/java/org/apache/metron/indexing/dao/UpdateIntegrationTest.java b/metron-platform/metron-indexing/metron-indexing-common/src/test/java/org/apache/metron/indexing/dao/UpdateIntegrationTest.java index 6333d32..ab41c08 100644 --- a/metron-platform/metron-indexing/metron-indexing-common/src/test/java/org/apache/metron/indexing/dao/UpdateIntegrationTest.java +++ b/metron-platform/metron-indexing/metron-indexing-common/src/test/java/org/apache/metron/indexing/dao/UpdateIntegrationTest.java @@ -17,12 +17,12 @@ package org.apache.metron.indexing.dao; import org.adrianwalker.multilinestring.Multiline; import org.apache.commons.collections.MapUtils; import org.apache.metron.common.Constants; -import org.apache.metron.hbase.mock.MockHTable; import org.apache.metron.indexing.dao.search.AlertComment; import org.apache.metron.indexing.dao.update.CommentAddRemoveRequest; import org.apache.metron.indexing.dao.update.Document; import org.apache.metron.indexing.dao.update.OriginalNotFoundException; import org.apache.metron.indexing.dao.update.PatchRequest; +import org.json.simple.parser.ParseException; import org.junit.Assert; import org.junit.Test; @@ -35,9 +35,10 @@ import java.util.Optional; import java.util.UUID; import java.util.stream.Collectors; -import static org.apache.metron.indexing.dao.IndexDao.COMMENTS_FIELD; import static org.hamcrest.CoreMatchers.hasItem; +import static org.apache.metron.indexing.dao.IndexDao.COMMENTS_FIELD; + public abstract class UpdateIntegrationTest { /** @@ -272,29 +273,81 @@ public abstract class UpdateIntegrationTest { return new Document(message1, guid, SENSOR_NAME, timestamp); } - private List<AlertComment> getComments(Document withComment) { - List<Map<String, Object>> commentsField = List.class.cast(withComment.getDocument().get(COMMENTS_FIELD)); + private static List<AlertComment> getComments(Document withComment) throws ParseException { + return getComments(withComment.getDocument()); + } + + private static List<AlertComment> getComments(Map<String, Object> fields) throws ParseException { List<AlertComment> comments = new ArrayList<>(); - if(commentsField != null) { - comments = commentsField - .stream() - .map(map -> new AlertComment(map)) - .collect(Collectors.toList()); + boolean hasComments = fields.containsKey(COMMENTS_FIELD); + if(hasComments) { + List<Object> commentsField = List.class.cast(fields.get(COMMENTS_FIELD)); + for (Object commentObject: commentsField) { + if (commentObject instanceof Map) { + // comments are stored as maps in Elasticsearch + Map<String, Object> commentAsMap = (Map<String, Object>) commentObject; + comments.add(new AlertComment(commentAsMap)); + + } else if (commentObject instanceof String) { + // comments are stored as json strings in Solr + String commentAsString = (String) commentObject; + comments.add(new AlertComment(commentAsString)); + + } else { + throw new IllegalArgumentException(String.format("Unexpected comment value; %s", commentObject)); + } + } } return comments; } - protected Document findUpdatedDoc(Map<String, Object> message0, String guid, String sensorType) + /** + * Normalizes the format of stored comments. + * + * <p>Comments are serialized differently when stored in Elasticsearch and Solr. Comments + * are stored as maps in Elasticsearch and JSON strings in Solr. This reformats all comments + * as maps, so they look the same when validation occurs in the integration tests. + * @param fields The fields of a document that may contain comments. + */ + protected static void normalizeCommentsAsMap(Map<String, Object> fields) { + @SuppressWarnings("unchecked") + List<Object> commentValues = (List<Object>) fields.get(COMMENTS_FIELD); + if (commentValues != null) { + try { + List<AlertComment> comments = getComments(fields); + if(comments.size() > 0) { + // overwrite the comments field + List<Map<String, Object>> serializedComments = comments + .stream() + .map(AlertComment::asMap) + .collect(Collectors.toList()); + fields.put(COMMENTS_FIELD, serializedComments); + + } else { + // there are no longer any comments + fields.remove(COMMENTS_FIELD); + } + + } catch (ParseException e) { + throw new IllegalStateException("Unable to parse comment", e); + } + } + } + + protected Document findUpdatedDoc(Map<String, Object> expected, String guid, String sensorType) throws InterruptedException, IOException, OriginalNotFoundException { + // comments are stored differently in Solr and Elasticsearch + normalizeCommentsAsMap(expected); + for (int t = 0; t < MAX_RETRIES; ++t, Thread.sleep(SLEEP_MS)) { - Document doc = getDao().getLatest(guid, sensorType); - if (doc != null && message0.equals(doc.getDocument())) { - return doc; + Document found = getDao().getLatest(guid, sensorType); + if (found != null && expected.equals(found.getDocument())) { + return found; } if (t == MAX_RETRIES -1) { - MapUtils.debugPrint(System.out, "Expected", message0); - MapUtils.debugPrint(System.out, "actual", doc.getDocument()); + MapUtils.debugPrint(System.out, "Expected", expected); + MapUtils.debugPrint(System.out, "Actual", found.getDocument()); } } throw new OriginalNotFoundException("Count not find " + guid + " after " + MAX_RETRIES + " tries"); @@ -309,7 +362,6 @@ public abstract class UpdateIntegrationTest { } protected abstract String getIndexName(); - protected abstract MockHTable getMockHTable(); protected abstract void addTestData(String indexName, String sensorType, List<Map<String,Object>> docs) throws Exception; protected abstract List<Map<String,Object>> getIndexedTestData(String indexName, String sensorType) throws Exception; } diff --git a/metron-platform/metron-indexing/metron-indexing-common/src/test/java/org/apache/metron/indexing/integration/HBaseDaoIntegrationTest.java b/metron-platform/metron-indexing/metron-indexing-common/src/test/java/org/apache/metron/indexing/integration/HBaseDaoIntegrationTest.java index 8148b69..75e88f4 100644 --- a/metron-platform/metron-indexing/metron-indexing-common/src/test/java/org/apache/metron/indexing/integration/HBaseDaoIntegrationTest.java +++ b/metron-platform/metron-indexing/metron-indexing-common/src/test/java/org/apache/metron/indexing/integration/HBaseDaoIntegrationTest.java @@ -231,11 +231,6 @@ public class HBaseDaoIntegrationTest extends UpdateIntegrationTest { } @Override - protected MockHTable getMockHTable() { - return null; - } - - @Override protected void addTestData(String indexName, String sensorType, List<Map<String, Object>> docs) { } diff --git a/metron-platform/metron-solr/metron-solr-common/src/main/java/org/apache/metron/solr/dao/SolrUtilities.java b/metron-platform/metron-solr/metron-solr-common/src/main/java/org/apache/metron/solr/dao/SolrUtilities.java index d41b7e4..eb2ff06 100644 --- a/metron-platform/metron-solr/metron-solr-common/src/main/java/org/apache/metron/solr/dao/SolrUtilities.java +++ b/metron-platform/metron-solr/metron-solr-common/src/main/java/org/apache/metron/solr/dao/SolrUtilities.java @@ -64,8 +64,9 @@ public class SolrUtilities { insertChildAlerts(solrDocument, document); return new Document(document, - (String) solrDocument.getFieldValue(Constants.GUID), - (String) solrDocument.getFieldValue(Constants.SENSOR_TYPE), 0L); + (String) solrDocument.getFieldValue(Constants.GUID), + (String) solrDocument.getFieldValue(Constants.SENSOR_TYPE), + (Long) solrDocument.getFieldValue(Constants.Fields.TIMESTAMP.getName())); } protected static void reformatComments(SolrDocument solrDocument, Map<String, Object> document) { diff --git a/metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/dao/SolrSearchDaoTest.java b/metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/dao/SolrSearchDaoTest.java index bf9458f..c02a612 100644 --- a/metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/dao/SolrSearchDaoTest.java +++ b/metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/dao/SolrSearchDaoTest.java @@ -38,6 +38,8 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.UUID; + import org.apache.metron.common.Constants; import org.apache.metron.indexing.dao.AccessConfig; import org.apache.metron.indexing.dao.search.GetRequest; @@ -219,7 +221,7 @@ public class SolrSearchDaoTest { @Test public void getLatestShouldProperlyReturnDocument() throws Exception { - SolrDocument solrDocument = mock(SolrDocument.class); + SolrDocument solrDocument = createSolrDocument("bro", 123456789L); solrSearchDao = spy(new SolrSearchDao(client, accessConfig)); when(client.getById("collection", "guid")).thenReturn(solrDocument); @@ -237,10 +239,10 @@ public class SolrSearchDaoTest { GetRequest broRequest2 = new GetRequest("bro-2", "bro"); GetRequest snortRequest1 = new GetRequest("snort-1", "snort"); GetRequest snortRequest2 = new GetRequest("snort-2", "snort"); - SolrDocument broSolrDoc1 = mock(SolrDocument.class); - SolrDocument broSolrDoc2 = mock(SolrDocument.class); - SolrDocument snortSolrDoc1 = mock(SolrDocument.class); - SolrDocument snortSolrDoc2 = mock(SolrDocument.class); + SolrDocument broSolrDoc1 = createSolrDocument("bro", 12345L); + SolrDocument broSolrDoc2 = createSolrDocument("bro", 34567L); + SolrDocument snortSolrDoc1 = createSolrDocument("snort", 12345L); + SolrDocument snortSolrDoc2 = createSolrDocument("snort", 67890L); Document broDoc1 = SolrUtilities.toDocument(broSolrDoc1); Document broDoc2 = SolrUtilities.toDocument(broSolrDoc2); Document snortDoc1 = SolrUtilities.toDocument(snortSolrDoc1); @@ -510,5 +512,13 @@ public class SolrSearchDaoTest { assertNull(level2GroupResults.get(1).getGroupResults()); } - + private SolrDocument createSolrDocument(String sensorType, Long timestamp) { + SolrDocument solrDocument = new SolrDocument(); + solrDocument.addField(SolrDao.VERSION_FIELD, 1.0); + solrDocument.addField(Constants.GUID, UUID.randomUUID().toString()); + solrDocument.addField(Constants.SENSOR_TYPE, sensorType); + solrDocument.addField(Constants.Fields.TIMESTAMP.getName(), timestamp); + solrDocument.addField(Constants.Fields.SRC_ADDR.getName(), "192.168.1.1"); + return solrDocument; + } } diff --git a/metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/dao/SolrUtilitiesTest.java b/metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/dao/SolrUtilitiesTest.java index f284f25..89441c0 100644 --- a/metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/dao/SolrUtilitiesTest.java +++ b/metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/dao/SolrUtilitiesTest.java @@ -30,17 +30,20 @@ public class SolrUtilitiesTest { @Test public void toDocumentShouldProperlyReturnDocument() throws Exception { + long expectedTimestamp = System.currentTimeMillis(); SolrDocument solrDocument = new SolrDocument(); solrDocument.addField(SolrDao.VERSION_FIELD, 1.0); solrDocument.addField(Constants.GUID, "guid"); solrDocument.addField(Constants.SENSOR_TYPE, "bro"); + solrDocument.addField(Constants.Fields.TIMESTAMP.getName(), expectedTimestamp); solrDocument.addField("field", "value"); Document expectedDocument = new Document(new HashMap<String, Object>() {{ put("field", "value"); put(Constants.GUID, "guid"); put(Constants.SENSOR_TYPE, "bro"); - }}, "guid", "bro", 0L); + put(Constants.Fields.TIMESTAMP.getName(), expectedTimestamp); + }}, "guid", "bro", expectedTimestamp); Document actualDocument = SolrUtilities.toDocument(solrDocument); assertEquals(expectedDocument, actualDocument); diff --git a/metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/integration/SolrRetrieveLatestIntegrationTest.java b/metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/integration/SolrRetrieveLatestIntegrationTest.java index 8c18981..d147202 100644 --- a/metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/integration/SolrRetrieveLatestIntegrationTest.java +++ b/metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/integration/SolrRetrieveLatestIntegrationTest.java @@ -51,7 +51,7 @@ public class SolrRetrieveLatestIntegrationTest { protected static final String TEST_COLLECTION = "test"; protected static final String TEST_SENSOR = "test_sensor"; protected static final String BRO_SENSOR = "bro"; - + protected final long expectedTimestamp = 123456789L; private static IndexDao dao; @BeforeClass @@ -62,8 +62,7 @@ public class SolrRetrieveLatestIntegrationTest { @Before public void setup() throws Exception { - solrComponent - .addCollection(TEST_COLLECTION, "./src/test/resources/config/test/conf"); + solrComponent.addCollection(TEST_COLLECTION, "./src/test/resources/config/test/conf"); solrComponent.addCollection(BRO_SENSOR, "./src/main/config/schema/bro"); AccessConfig accessConfig = new AccessConfig(); @@ -75,8 +74,8 @@ public class SolrRetrieveLatestIntegrationTest { dao = new SolrDao(); dao.init(accessConfig); - addData(BRO_SENSOR, BRO_SENSOR); - addData(TEST_COLLECTION, TEST_SENSOR); + addData(BRO_SENSOR, BRO_SENSOR, expectedTimestamp); + addData(TEST_COLLECTION, TEST_SENSOR, expectedTimestamp); } @After @@ -131,8 +130,11 @@ public class SolrRetrieveLatestIntegrationTest { requests.add(buildGetRequest(BRO_SENSOR, 2)); Iterable<Document> actual = dao.getAllLatest(requests); - assertTrue(Iterables.contains(actual, buildExpectedDocument(BRO_SENSOR, 1))); - assertTrue(Iterables.contains(actual, buildExpectedDocument(BRO_SENSOR, 2))); + Document expected1 = buildExpectedDocument(BRO_SENSOR, 1); + assertTrue(Iterables.contains(actual, expected1)); + + Document expected2 = buildExpectedDocument(BRO_SENSOR, 2); + assertTrue(Iterables.contains(actual, expected2)); assertEquals(2, Iterables.size(actual)); } @@ -179,8 +181,9 @@ public class SolrRetrieveLatestIntegrationTest { protected Document buildExpectedDocument(String sensor, int i) { Map<String, Object> expectedMapOne = new HashMap<>(); expectedMapOne.put("source.type", sensor); + expectedMapOne.put(Constants.Fields.TIMESTAMP.getName(), expectedTimestamp); expectedMapOne.put(Constants.GUID, buildGuid(sensor, i)); - return new Document(expectedMapOne, buildGuid(sensor, i), sensor, 0L); + return new Document(expectedMapOne, buildGuid(sensor, i), sensor, expectedTimestamp); } protected GetRequest buildGetRequest(String sensor, int i) { @@ -190,7 +193,7 @@ public class SolrRetrieveLatestIntegrationTest { return requestOne; } - protected static void addData(String collection, String sensorName) + protected static void addData(String collection, String sensorName, Long timestamp) throws IOException, SolrServerException { List<Map<String, Object>> inputData = new ArrayList<>(); for (int i = 0; i < 3; ++i) { @@ -198,6 +201,7 @@ public class SolrRetrieveLatestIntegrationTest { HashMap<String, Object> inputMap = new HashMap<>(); inputMap.put("source.type", sensorName); inputMap.put(Constants.GUID, name); + inputMap.put(Constants.Fields.TIMESTAMP.getName(), timestamp); inputData.add(inputMap); } solrComponent.addDocs(collection, inputData); diff --git a/metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/integration/SolrUpdateIntegrationTest.java b/metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/integration/SolrUpdateIntegrationTest.java index d49f4b4..852ca10 100644 --- a/metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/integration/SolrUpdateIntegrationTest.java +++ b/metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/integration/SolrUpdateIntegrationTest.java @@ -17,32 +17,19 @@ */ package org.apache.metron.solr.integration; -import static org.apache.metron.solr.SolrConstants.SOLR_ZOOKEEPER; -import static org.junit.Assert.assertEquals; - -import java.io.IOException; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; import org.apache.commons.lang3.StringUtils; import org.apache.curator.framework.CuratorFramework; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.metron.common.configuration.ConfigurationsUtils; import org.apache.metron.common.zookeeper.ZKConfigurationsCache; -import org.apache.metron.hbase.mock.MockHBaseTableProvider; -import org.apache.metron.hbase.mock.MockHTable; import org.apache.metron.indexing.dao.AccessConfig; import org.apache.metron.indexing.dao.HBaseDao; -import org.apache.metron.indexing.dao.IndexDao; -import org.apache.metron.indexing.dao.MultiIndexDao; import org.apache.metron.indexing.dao.UpdateIntegrationTest; import org.apache.metron.indexing.dao.update.Document; import org.apache.metron.indexing.util.IndexingCacheUtil; import org.apache.metron.solr.client.SolrClientFactory; import org.apache.metron.solr.dao.SolrDao; import org.apache.metron.solr.integration.components.SolrComponent; +import org.apache.solr.common.SolrException; import org.junit.After; import org.junit.AfterClass; import org.junit.Before; @@ -51,6 +38,15 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static org.apache.metron.solr.SolrConstants.SOLR_ZOOKEEPER; +import static org.junit.Assert.assertEquals; + public class SolrUpdateIntegrationTest extends UpdateIntegrationTest { @Rule public final ExpectedException exception = ExpectedException.none(); @@ -59,8 +55,6 @@ public class SolrUpdateIntegrationTest extends UpdateIntegrationTest { private static final String TABLE_NAME = "modifications"; private static final String CF = "p"; - private static MockHTable table; - private static IndexDao hbaseDao; @BeforeClass public static void setupBeforeClass() throws Exception { @@ -73,28 +67,21 @@ public class SolrUpdateIntegrationTest extends UpdateIntegrationTest { solrComponent.addCollection(SENSOR_NAME, "./src/test/resources/config/test/conf"); solrComponent.addCollection("error", "./src/main/config/schema/error"); - Configuration config = HBaseConfiguration.create(); - MockHBaseTableProvider tableProvider = new MockHBaseTableProvider(); - MockHBaseTableProvider.addToCache(TABLE_NAME, CF); - table = (MockHTable) tableProvider.getTable(config, TABLE_NAME); - - hbaseDao = new HBaseDao(); - AccessConfig accessConfig = new AccessConfig(); - accessConfig.setTableProvider(tableProvider); Map<String, Object> globalConfig = createGlobalConfig(); globalConfig.put(HBaseDao.HBASE_TABLE, TABLE_NAME); globalConfig.put(HBaseDao.HBASE_CF, CF); - accessConfig.setGlobalConfigSupplier(() -> globalConfig); - accessConfig.setIndexSupplier(s -> s); - CuratorFramework client = ConfigurationsUtils - .getClient(solrComponent.getZookeeperUrl()); + CuratorFramework client = ConfigurationsUtils.getClient(solrComponent.getZookeeperUrl()); client.start(); ZKConfigurationsCache cache = new ZKConfigurationsCache(client); cache.start(); + + AccessConfig accessConfig = new AccessConfig(); + accessConfig.setGlobalConfigSupplier(() -> globalConfig); + accessConfig.setIndexSupplier(s -> s); accessConfig.setIndexSupplier(IndexingCacheUtil.getIndexLookupFunction(cache, "solr")); - MultiIndexDao dao = new MultiIndexDao(hbaseDao, new SolrDao()); + SolrDao dao = new SolrDao(); dao.init(accessConfig); setDao(dao); } @@ -102,7 +89,6 @@ public class SolrUpdateIntegrationTest extends UpdateIntegrationTest { @After public void reset() { solrComponent.reset(); - table.clear(); } @AfterClass @@ -116,11 +102,6 @@ public class SolrUpdateIntegrationTest extends UpdateIntegrationTest { return SENSOR_NAME; } - @Override - protected MockHTable getMockHTable() { - return table; - } - private static Map<String, Object> createGlobalConfig() { return new HashMap<String, Object>() {{ put(SOLR_ZOOKEEPER, solrComponent.getZookeeperUrl()); @@ -184,8 +165,9 @@ public class SolrUpdateIntegrationTest extends UpdateIntegrationTest { documentMap.put("error_hash", hugeString); errorDoc = new Document(documentMap, "error", "error", 0L); - exception.expect(IOException.class); + exception.expect(SolrException.class); exception.expectMessage("Document contains at least one immense term in field=\"error_hash\""); + getDao().update(errorDoc, Optional.of("error")); } }