Repository: metron Updated Branches: refs/heads/feature/METRON-1416-upgrade-solr 7ef4b7703 -> f8d7843e9
METRON-1526 Location field types cause DocValuesField appear more than once error (merrimanr via justinleet) closes apache/metron#995 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/f8d7843e Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/f8d7843e Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/f8d7843e Branch: refs/heads/feature/METRON-1416-upgrade-solr Commit: f8d7843e93e410147257cca40d83b2c2d4963f1b Parents: 7ef4b77 Author: merrimanr <[email protected]> Authored: Thu Apr 26 09:01:03 2018 -0400 Committer: leet <[email protected]> Committed: Thu Apr 26 09:01:03 2018 -0400 ---------------------------------------------------------------------- .../indexing/dao/UpdateIntegrationTest.java | 21 +++++++++--- metron-platform/metron-solr/README.md | 16 +++++++++ .../src/main/config/schema/bro/schema.xml | 1 + .../src/main/config/schema/snort/schema.xml | 1 + .../src/main/config/schema/yaf/schema.xml | 1 + .../integration/SolrUpdateIntegrationTest.java | 34 ++++++++++++++++++-- .../integration/components/SolrComponent.java | 7 +++- 7 files changed, 74 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/f8d7843e/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/UpdateIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/UpdateIntegrationTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/UpdateIntegrationTest.java index 369fa79..471acf6 100644 --- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/UpdateIntegrationTest.java +++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/UpdateIntegrationTest.java @@ -31,11 +31,14 @@ import org.apache.metron.hbase.mock.MockHTable; import org.apache.metron.indexing.dao.update.Document; import org.apache.metron.indexing.dao.update.ReplaceRequest; import org.apache.metron.integration.InMemoryComponent; +import org.junit.After; import org.junit.AfterClass; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import static org.junit.Assert.assertEquals; + public abstract class UpdateIntegrationTest { private static final int MAX_RETRIES = 10; @@ -46,6 +49,7 @@ public abstract class UpdateIntegrationTest { private static String index; private static MockHTable table; private static IndexDao hbaseDao; + private static AccessConfig accessConfig; protected static MultiIndexDao dao; protected static InMemoryComponent indexComponent; @@ -62,20 +66,24 @@ public abstract class UpdateIntegrationTest { table = (MockHTable)tableProvider.getTable(config, TABLE_NAME); hbaseDao = new HBaseDao(); - AccessConfig accessConfig = new AccessConfig(); + accessConfig = new AccessConfig(); accessConfig.setTableProvider(tableProvider); Map<String, Object> globalConfig = createGlobalConfig(); globalConfig.put(HBaseDao.HBASE_TABLE, TABLE_NAME); globalConfig.put(HBaseDao.HBASE_CF, CF); accessConfig.setGlobalConfigSupplier(() -> globalConfig); - - dao = new MultiIndexDao(hbaseDao, createDao()); - dao.init(accessConfig); } } + protected AccessConfig getAccessConfig() { + return accessConfig; + } + @Test public void test() throws Exception { + dao = new MultiIndexDao(hbaseDao, createDao()); + dao.init(getAccessConfig()); + List<Map<String, Object>> inputData = new ArrayList<>(); for(int i = 0; i < 10;++i) { final String name = "message" + i; @@ -181,6 +189,11 @@ public abstract class UpdateIntegrationTest { } } + @After + public void reset() throws Exception { + indexComponent.reset(); + } + @AfterClass public static void teardown() { if(indexComponent != null) { http://git-wip-us.apache.org/repos/asf/metron/blob/f8d7843e/metron-platform/metron-solr/README.md ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/README.md b/metron-platform/metron-solr/README.md index 4321114..0d525ec 100644 --- a/metron-platform/metron-solr/README.md +++ b/metron-platform/metron-solr/README.md @@ -95,3 +95,19 @@ As of now, we have mapped out the Schemas in `src/main/config/schema`. Ambari will eventually install these, but at the moment it's manual and you should refer to the Solr documentation [https://lucene.apache.org/solr/guide/6_6](here) in general and [here](https://lucene.apache.org/solr/guide/6_6/documents-fields-and-schema-design.html) if you'd like to know more about schemas in Solr. + +In Metron's Solr DAO implementation, document updates involve reading a document, applying the update and replacing the original by reindexing the whole document. +Indexing LatLonType and PointType field types stores data in internal fields that should not be returned in search results. For these fields a dynamic field type matching the suffix needs to be added to store the data points. +Solr 6+ comes with a new LatLonPointSpatialField field type that should be used instead of LatLonType if possible. Otherwise, a LatLongType field should be defined as: +``` +<dynamicField name="*.location_point" type="location" multiValued="false" docValues="false"/> +<dynamicField name="*_coordinate" type="pdouble" indexed="true" stored="false" docValues="false"/> +<fieldType name="location" class="solr.LatLonType" subFieldSuffix="_coordinate"/> +``` +A PointType field should be defined as: +``` +<dynamicField name="*.point" type="point" multiValued="false" docValues="false"/> +<dynamicField name="*_point" type="pdouble" indexed="true" stored="false" docValues="false"/> +<fieldType name="point" class="solr.PointType" subFieldSuffix="_point"/> +``` +If any copy fields are defined, stored and docValues should be set to false. \ No newline at end of file http://git-wip-us.apache.org/repos/asf/metron/blob/f8d7843e/metron-platform/metron-solr/src/main/config/schema/bro/schema.xml ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/main/config/schema/bro/schema.xml b/metron-platform/metron-solr/src/main/config/schema/bro/schema.xml index 4ce8d6b..b463366 100644 --- a/metron-platform/metron-solr/src/main/config/schema/bro/schema.xml +++ b/metron-platform/metron-solr/src/main/config/schema/bro/schema.xml @@ -659,6 +659,7 @@ <!-- Geo Enrichment Fields --> <dynamicField name="*.location_point" type="location" multiValued="false" docValues="false"/> + <dynamicField name="*_coordinate" type="pdouble" indexed="true" stored="false" docValues="false"/> <dynamicField name="*.country" type="string" multiValued="false" docValues="true"/> <dynamicField name="*.city" type="string" multiValued="false" docValues="true"/> <dynamicField name="*.locID" type="string" multiValued="false" docValues="true"/> http://git-wip-us.apache.org/repos/asf/metron/blob/f8d7843e/metron-platform/metron-solr/src/main/config/schema/snort/schema.xml ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/main/config/schema/snort/schema.xml b/metron-platform/metron-solr/src/main/config/schema/snort/schema.xml index ae13b8d..129c0f0 100644 --- a/metron-platform/metron-solr/src/main/config/schema/snort/schema.xml +++ b/metron-platform/metron-solr/src/main/config/schema/snort/schema.xml @@ -52,6 +52,7 @@ <!-- Geo Enrichment Fields --> <dynamicField name="*.location_point" type="location" multiValued="false" docValues="false"/> + <dynamicField name="*_coordinate" type="pdouble" indexed="true" stored="false" docValues="false"/> <dynamicField name="*.country" type="string" multiValued="false" docValues="true"/> <dynamicField name="*.city" type="string" multiValued="false" docValues="true"/> <dynamicField name="*.locID" type="string" multiValued="false" docValues="true"/> http://git-wip-us.apache.org/repos/asf/metron/blob/f8d7843e/metron-platform/metron-solr/src/main/config/schema/yaf/schema.xml ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/main/config/schema/yaf/schema.xml b/metron-platform/metron-solr/src/main/config/schema/yaf/schema.xml index edccd9d..f3abb14 100644 --- a/metron-platform/metron-solr/src/main/config/schema/yaf/schema.xml +++ b/metron-platform/metron-solr/src/main/config/schema/yaf/schema.xml @@ -58,6 +58,7 @@ <!-- Geo Enrichment Fields --> <dynamicField name="*.location_point" type="location" multiValued="false" docValues="false"/> + <dynamicField name="*_coordinate" type="pdouble" indexed="true" stored="false" docValues="false"/> <dynamicField name="*.country" type="string" multiValued="false" docValues="true"/> <dynamicField name="*.city" type="string" multiValued="false" docValues="true"/> <dynamicField name="*.locID" type="string" multiValued="false" docValues="true"/> http://git-wip-us.apache.org/repos/asf/metron/blob/f8d7843e/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrUpdateIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrUpdateIntegrationTest.java b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrUpdateIntegrationTest.java index faa4ec4..232c727 100644 --- a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrUpdateIntegrationTest.java +++ b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrUpdateIntegrationTest.java @@ -20,15 +20,21 @@ package org.apache.metron.solr.integration; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; + import org.apache.metron.indexing.dao.IndexDao; +import org.apache.metron.indexing.dao.MultiIndexDao; import org.apache.metron.indexing.dao.UpdateIntegrationTest; +import org.apache.metron.indexing.dao.update.Document; import org.apache.metron.integration.InMemoryComponent; import org.apache.metron.solr.dao.SolrDao; import org.apache.metron.solr.integration.components.SolrComponent; +import org.junit.Assert; +import org.junit.Test; public class SolrUpdateIntegrationTest extends UpdateIntegrationTest { - SolrComponent solrComponent; + protected static SolrComponent solrComponent; @Override protected String getIndexName() { @@ -49,7 +55,7 @@ public class SolrUpdateIntegrationTest extends UpdateIntegrationTest { @Override protected InMemoryComponent startIndex() throws Exception { - solrComponent = new SolrComponent.Builder().addCollection(SENSOR_NAME, "../metron-solr/src/test/resources/config/test/conf").build(); + solrComponent = new SolrComponent.Builder().addCollection(SENSOR_NAME, "../metron-solr/src/main/config/schema/bro").build(); solrComponent.start(); return solrComponent; } @@ -70,4 +76,28 @@ public class SolrUpdateIntegrationTest extends UpdateIntegrationTest { throws Exception { return solrComponent.getAllIndexedDocs(indexName); } + + @Test + public void suppress_expanded_fields() throws Exception { + dao = new MultiIndexDao(createDao()); + dao.init(getAccessConfig()); + + Map<String, Object> fields = new HashMap<>(); + fields.put("guid", "bro_1"); + fields.put("source.type", SENSOR_NAME); + fields.put("ip_src_port", 8010); + fields.put("long_field", 10000); + fields.put("latitude", 48.5839); + fields.put("score", 10.0); + fields.put("is_alert", true); + fields.put("field.location_point", "48.5839,7.7455"); + + Document document = new Document(fields, "bro_1", SENSOR_NAME, 0L); + dao.update(document, Optional.of(SENSOR_NAME)); + + Document indexedDocument = dao.getLatest("bro_1", SENSOR_NAME); + + // assert no extra expanded fields are included + Assert.assertEquals(8, indexedDocument.getDocument().size()); + } } http://git-wip-us.apache.org/repos/asf/metron/blob/f8d7843e/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/components/SolrComponent.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/components/SolrComponent.java b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/components/SolrComponent.java index 85d14f4..1cf8c34 100644 --- a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/components/SolrComponent.java +++ b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/components/SolrComponent.java @@ -116,7 +116,12 @@ public class SolrComponent implements InMemoryComponent { @Override public void reset() { try { - miniSolrCloudCluster.deleteCollection("metron"); + MetronSolrClient solr = getSolrClient(); + for (String collection: collections.keySet()) { + solr.setDefaultCollection(collection); + solr.deleteByQuery("*:*"); + } + solr.commit(); } catch (SolrServerException | IOException e) { // Do nothing }
