METRON-1567 Large error message can't be written in Solr (justinleet) closes apache/metron#1020
Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/bc16e682 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/bc16e682 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/bc16e682 Branch: refs/heads/feature/METRON-1554-pcap-query-panel Commit: bc16e68252de98846bba58aafb2d7d445caac9c1 Parents: 832296f Author: justinleet <justinjl...@gmail.com> Authored: Fri May 18 11:34:30 2018 -0400 Committer: leet <l...@apache.org> Committed: Fri May 18 11:34:30 2018 -0400 ---------------------------------------------------------------------- .../src/main/config/schema/error/schema.xml | 4 +- .../integration/SolrUpdateIntegrationTest.java | 49 ++++++++++++++++++-- 2 files changed, 48 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/bc16e682/metron-platform/metron-solr/src/main/config/schema/error/schema.xml ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/main/config/schema/error/schema.xml b/metron-platform/metron-solr/src/main/config/schema/error/schema.xml index 5f9f9a3..4aa80ef 100644 --- a/metron-platform/metron-solr/src/main/config/schema/error/schema.xml +++ b/metron-platform/metron-solr/src/main/config/schema/error/schema.xml @@ -29,7 +29,6 @@ <!-- The message in string form (this may be garbage if the message is a bytes rather than text --> <field name="message" type="string" indexed="true" stored="true" /> - <field name="raw_message" type="string" indexed="false" stored="true" /> <field name="raw_message_bytes" type="bytes" indexed="false" stored="true" /> <field name="error_fields" type="string" indexed="true" stored="true" /> <field name="error_hash" type="string" indexed="true" stored="true" /> @@ -40,6 +39,8 @@ <field name="guid" type="string" indexed="true" stored="true" required="true" multiValued="false" /> <uniqueKey>guid</uniqueKey> + <!-- Raw messages can be split into multiple fields --> + <dynamicField name="raw_message*" type="text_raw" indexed="false" stored="true"/> <!-- Catch all, if we don't know about it, it gets dropped. --> <dynamicField name="*" type="ignored" multiValued="false" docValues="true"/> @@ -55,4 +56,5 @@ <fieldType name="ip" stored="true" indexed="true" multiValued="false" class="solr.StrField" sortMissingLast="true" docValues="false"/> <fieldType name="timestamp" stored="true" indexed="true" multiValued="false" class="solr.TrieLongField" sortMissingLast="false" docValues="true"/> <fieldType name="ignored" stored="true" indexed="true" multiValued="true" class="solr.StrField" sortMissingLast="false" docValues="false"/> + <fieldType name="text_raw" class="solr.TextField"/> </schema> http://git-wip-us.apache.org/repos/asf/metron/blob/bc16e682/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrUpdateIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrUpdateIntegrationTest.java b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrUpdateIntegrationTest.java index 232c727..dc08967 100644 --- a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrUpdateIntegrationTest.java +++ b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrUpdateIntegrationTest.java @@ -17,11 +17,14 @@ */ package org.apache.metron.solr.integration; +import static org.junit.Assert.assertEquals; + +import java.io.IOException; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; - +import org.apache.commons.lang3.StringUtils; import org.apache.metron.indexing.dao.IndexDao; import org.apache.metron.indexing.dao.MultiIndexDao; import org.apache.metron.indexing.dao.UpdateIntegrationTest; @@ -29,10 +32,13 @@ import org.apache.metron.indexing.dao.update.Document; import org.apache.metron.integration.InMemoryComponent; import org.apache.metron.solr.dao.SolrDao; import org.apache.metron.solr.integration.components.SolrComponent; -import org.junit.Assert; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.ExpectedException; public class SolrUpdateIntegrationTest extends UpdateIntegrationTest { + @Rule + public final ExpectedException exception = ExpectedException.none(); protected static SolrComponent solrComponent; @@ -55,7 +61,9 @@ public class SolrUpdateIntegrationTest extends UpdateIntegrationTest { @Override protected InMemoryComponent startIndex() throws Exception { - solrComponent = new SolrComponent.Builder().addCollection(SENSOR_NAME, "../metron-solr/src/main/config/schema/bro").build(); + solrComponent = new SolrComponent.Builder().addCollection(SENSOR_NAME, "../metron-solr/src/main/config/schema/bro") + .addCollection("error", "../metron-solr/src/main/config/schema/error") + .build(); solrComponent.start(); return solrComponent; } @@ -98,6 +106,39 @@ public class SolrUpdateIntegrationTest extends UpdateIntegrationTest { Document indexedDocument = dao.getLatest("bro_1", SENSOR_NAME); // assert no extra expanded fields are included - Assert.assertEquals(8, indexedDocument.getDocument().size()); + assertEquals(8, indexedDocument.getDocument().size()); + } + + @Test + public void testHugeErrorFields() throws Exception { + dao = new MultiIndexDao(createDao()); + dao.init(getAccessConfig()); + + String hugeString = StringUtils.repeat("test ", 1_000_000); + String hugeStringTwo = hugeString + "-2"; + + Map<String, Object> documentMap = new HashMap<>(); + documentMap.put("guid", "error_guid"); + // Needs to be over 32kb + documentMap.put("raw_message", hugeString); + documentMap.put("raw_message_1", hugeStringTwo); + Document errorDoc = new Document(documentMap, "error", "error", 0L); + dao.update(errorDoc, Optional.of("error")); + + // Ensure that the huge string is returned when not a string field + Document latest = dao.getLatest("error_guid", "error"); + @SuppressWarnings("unchecked") + String actual = (String) latest.getDocument().get("raw_message"); + assertEquals(actual, hugeString); + String actualTwo = (String) latest.getDocument().get("raw_message_1"); + assertEquals(actualTwo, hugeStringTwo); + + // Validate that error occurs for string fields. + documentMap.put("error_hash", hugeString); + errorDoc = new Document(documentMap, "error", "error", 0L); + + exception.expect(IOException.class); + exception.expectMessage("Document contains at least one immense term in field=\"error_hash\""); + dao.update(errorDoc, Optional.of("error")); } }