http://git-wip-us.apache.org/repos/asf/metron/blob/49f851e0/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/components/SolrComponent.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/components/SolrComponent.java b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/components/SolrComponent.java index 1cf8c34..4bc9f8a 100644 --- a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/components/SolrComponent.java +++ b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/components/SolrComponent.java @@ -19,9 +19,13 @@ package org.apache.metron.solr.integration.components; import com.google.common.base.Function; import java.util.Collection; +import java.util.Map.Entry; import java.util.stream.Collectors; +import org.apache.metron.common.Constants; +import org.apache.metron.indexing.dao.metaalert.MetaAlertConstants; import org.apache.metron.integration.InMemoryComponent; import org.apache.metron.integration.UnableToStartException; +import org.apache.metron.solr.dao.SolrUtilities; import org.apache.metron.solr.writer.MetronSolrClient; import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.SolrServerException; @@ -29,6 +33,7 @@ import org.apache.solr.client.solrj.embedded.JettyConfig; import org.apache.solr.client.solrj.impl.CloudSolrClient; import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.client.solrj.response.UpdateResponse; import org.apache.solr.cloud.MiniSolrCloudCluster; import org.apache.solr.common.SolrDocument; @@ -40,13 +45,15 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import org.apache.solr.common.SolrInputDocument; +import org.apache.zookeeper.KeeperException; public class SolrComponent implements InMemoryComponent { public static class Builder { + private int port = 8983; private String solrXmlPath = "../metron-solr/src/test/resources/solr/solr.xml"; - private Map<String, String> collections = new HashMap<>(); + private Map<String, String> initialCollections = new HashMap<>(); private Function<SolrComponent, Void> postStartCallback; public Builder withPort(int port) { @@ -59,8 +66,8 @@ public class SolrComponent implements InMemoryComponent { return this; } - public Builder addCollection(String name, String configPath) { - collections.put(name, configPath); + public Builder addInitialCollection(String name, String configPath) { + initialCollections.put(name, configPath); return this; } @@ -69,9 +76,8 @@ public class SolrComponent implements InMemoryComponent { return this; } - public SolrComponent build() throws Exception { - if (collections.isEmpty()) throw new Exception("Must add at least 1 collection"); - return new SolrComponent(port, solrXmlPath, collections, postStartCallback); + public SolrComponent build() { + return new SolrComponent(port, solrXmlPath, initialCollections, postStartCallback); } } @@ -81,7 +87,8 @@ public class SolrComponent implements InMemoryComponent { private MiniSolrCloudCluster miniSolrCloudCluster; private Function<SolrComponent, Void> postStartCallback; - private SolrComponent(int port, String solrXmlPath, Map<String, String> collections, Function<SolrComponent, Void> postStartCallback) throws Exception { + private SolrComponent(int port, String solrXmlPath, Map<String, String> collections, + Function<SolrComponent, Void> postStartCallback) { this.port = port; this.solrXmlPath = solrXmlPath; this.collections = collections; @@ -93,14 +100,17 @@ public class SolrComponent implements InMemoryComponent { try { File baseDir = Files.createTempDirectory("solrcomponent").toFile(); baseDir.deleteOnExit(); - miniSolrCloudCluster = new MiniSolrCloudCluster(1, baseDir.toPath(), JettyConfig.builder().setPort(port).build()); + miniSolrCloudCluster = new MiniSolrCloudCluster(1, baseDir.toPath(), + JettyConfig.builder().setPort(port).build()); for(String name: collections.keySet()) { String configPath = collections.get(name); miniSolrCloudCluster.uploadConfigSet(new File(configPath).toPath(), name); CollectionAdminRequest.createCollection(name, 1, 1).process(miniSolrCloudCluster.getSolrClient()); } - if (postStartCallback != null) postStartCallback.apply(this); - } catch(Exception e) { + if (postStartCallback != null) { + postStartCallback.apply(this); + } + } catch (Exception e) { throw new UnableToStartException(e.getMessage(), e); } } @@ -108,21 +118,18 @@ public class SolrComponent implements InMemoryComponent { @Override public void stop() { try { + miniSolrCloudCluster.deleteAllCollections(); miniSolrCloudCluster.shutdown(); } catch (Exception e) { + // Do nothing } } @Override public void reset() { try { - MetronSolrClient solr = getSolrClient(); - for (String collection: collections.keySet()) { - solr.setDefaultCollection(collection); - solr.deleteByQuery("*:*"); - } - solr.commit(); - } catch (SolrServerException | IOException e) { + miniSolrCloudCluster.deleteAllCollections(); + } catch (Exception e) { // Do nothing } } @@ -139,12 +146,19 @@ public class SolrComponent implements InMemoryComponent { return miniSolrCloudCluster.getZkServer().getZkAddress(); } + public void addCollection(String name, String configPath) + throws InterruptedException, IOException, KeeperException, SolrServerException { + miniSolrCloudCluster.uploadConfigSet(new File(configPath).toPath(), name); + CollectionAdminRequest.createCollection(name, 1, 1) + .process(miniSolrCloudCluster.getSolrClient()); + } + public boolean hasCollection(String collection) { MetronSolrClient solr = getSolrClient(); boolean collectionFound = false; try { collectionFound = solr.listCollections().contains(collection); - } catch(Exception e) { + } catch (Exception e) { e.printStackTrace(); } return collectionFound; @@ -155,12 +169,21 @@ public class SolrComponent implements InMemoryComponent { CloudSolrClient solr = miniSolrCloudCluster.getSolrClient(); solr.setDefaultCollection(collection); SolrQuery parameters = new SolrQuery(); - parameters.set("q", "*:*"); + + // If it's metaalert, we need to adjust the query. We want child docs with the parent, + // not separate. + if (collection.equals("metaalert")) { + parameters.setQuery("source.type:metaalert") + .setFields("*", "[child parentFilter=source.type:metaalert limit=999]"); + } else { + parameters.set("q", "*:*"); + } try { solr.commit(); QueryResponse response = solr.query(parameters); for (SolrDocument solrDocument : response.getResults()) { - docs.add(solrDocument); + // Use the utils to make sure we get child docs. + docs.add(SolrUtilities.toDocument(solrDocument).getDocument()); } } catch (SolrServerException | IOException e) { e.printStackTrace(); @@ -174,9 +197,36 @@ public class SolrComponent implements InMemoryComponent { solr.setDefaultCollection(collection); Collection<SolrInputDocument> solrInputDocuments = docs.stream().map(doc -> { SolrInputDocument solrInputDocument = new SolrInputDocument(); - doc.forEach(solrInputDocument::addField); + for (Entry<String, Object> entry : doc.entrySet()) { + // If the entry itself is a map, add it as a child document. Handle one level of nesting. + if (entry.getValue() instanceof List && !entry.getKey().equals( + MetaAlertConstants.METAALERT_FIELD)) { + for (Object entryItem : (List)entry.getValue()) { + if (entryItem instanceof Map) { + @SuppressWarnings("unchecked") + Map<String, Object> childDoc = (Map<String, Object>) entryItem; + SolrInputDocument childInputDoc = new SolrInputDocument(); + for (Entry<String, Object> childEntry : childDoc.entrySet()) { + childInputDoc.addField(childEntry.getKey(), childEntry.getValue()); + } + solrInputDocument.addChildDocument(childInputDoc); + } + } + } else { + solrInputDocument.addField(entry.getKey(), entry.getValue()); + } + } return solrInputDocument; }).collect(Collectors.toList()); - solr.add(collection, solrInputDocuments); + + checkUpdateResponse(solr.add(collection, solrInputDocuments)); + // Make sure to commit so things show up + checkUpdateResponse(solr.commit(true, true)); + } + + protected void checkUpdateResponse(UpdateResponse result) throws IOException { + if (result.getStatus() != 0) { + throw new IOException("Response error received while adding documents: " + result); + } } }
http://git-wip-us.apache.org/repos/asf/metron/blob/49f851e0/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/schema/SchemaValidationIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/schema/SchemaValidationIntegrationTest.java b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/schema/SchemaValidationIntegrationTest.java index e655428..73113f6 100644 --- a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/schema/SchemaValidationIntegrationTest.java +++ b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/schema/SchemaValidationIntegrationTest.java @@ -25,9 +25,12 @@ import org.apache.metron.common.writer.BulkWriterResponse; import org.apache.metron.solr.integration.components.SolrComponent; import org.apache.metron.solr.writer.SolrWriter; import org.apache.metron.stellar.common.utils.ConversionUtils; +import org.apache.solr.client.solrj.SolrServerException; import org.apache.storm.tuple.Tuple; +import org.apache.zookeeper.KeeperException; import org.json.simple.JSONObject; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; import org.mockito.Mock; import java.io.File; @@ -38,7 +41,6 @@ import java.util.*; import static org.mockito.Mockito.mock; public class SchemaValidationIntegrationTest { - public static Iterable<String> getData(String sensor) throws IOException { return Iterables.filter( Files.readLines(new File("src/test/resources/example_data/" + sensor), Charset.defaultCharset()), @@ -53,9 +55,7 @@ public class SchemaValidationIntegrationTest { } public static SolrComponent createSolrComponent(String sensor) throws Exception { - return new SolrComponent.Builder() - .addCollection(String.format("%s", sensor), String.format("src/main/config/schema/%s", sensor)) - .build(); + return new SolrComponent.Builder().build(); } @Test @@ -92,6 +92,7 @@ public class SchemaValidationIntegrationTest { try { component = createSolrComponent(sensorType); component.start(); + component.addCollection(String.format("%s", sensorType), String.format("src/main/config/schema/%s", sensorType)); Map<String, Object> globalConfig = getGlobalConfig(sensorType, component); List<JSONObject> inputs = new ArrayList<>(); http://git-wip-us.apache.org/repos/asf/metron/blob/49f851e0/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/writer/SolrWriterTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/writer/SolrWriterTest.java b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/writer/SolrWriterTest.java index 685c5fd..7b7d208 100644 --- a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/writer/SolrWriterTest.java +++ b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/writer/SolrWriterTest.java @@ -17,11 +17,19 @@ */ package org.apache.metron.solr.writer; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.argThat; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Iterables; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import org.apache.metron.common.Constants; -import org.apache.metron.common.configuration.EnrichmentConfigurations; import org.apache.metron.common.configuration.IndexingConfigurations; import org.apache.metron.common.configuration.writer.IndexingWriterConfiguration; import org.apache.metron.enrichment.integration.utils.SampleUtil; @@ -34,17 +42,6 @@ import org.junit.Test; import org.mockito.ArgumentMatcher; import org.mockito.Mockito; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static org.mockito.Matchers.eq; -import static org.mockito.Mockito.argThat; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; - public class SolrWriterTest { http://git-wip-us.apache.org/repos/asf/metron/blob/49f851e0/metron-platform/metron-solr/src/test/resources/config/test/conf/managed-schema ---------------------------------------------------------------------- diff --git a/metron-platform/metron-solr/src/test/resources/config/test/conf/managed-schema b/metron-platform/metron-solr/src/test/resources/config/test/conf/managed-schema index 85d6915..bb2de59 100644 --- a/metron-platform/metron-solr/src/test/resources/config/test/conf/managed-schema +++ b/metron-platform/metron-solr/src/test/resources/config/test/conf/managed-schema @@ -16,53 +16,59 @@ limitations under the License. --> <schema name="example" version="1.6"> - <field name="_version_" type="plong" indexed="false" stored="false"/> + <field name="_version_" type="plong" indexed="false" stored="false"/> - <!-- points to the root document of a block of nested documents. Required for nested - document support, may be removed otherwise - --> - <field name="_root_" type="string" indexed="true" stored="false" docValues="false" /> + <!-- points to the root document of a block of nested documents. Required for nested + document support, may be removed otherwise + --> + <field name="_root_" type="string" indexed="true" stored="false" docValues="false"/> - <!-- Only remove the "id" field if you have a very good reason to. While not strictly - required, it is highly recommended. A <uniqueKey> is present in almost all Solr - installations. See the <uniqueKey> declaration below where <uniqueKey> is set to "id". - Do NOT change the type and apply index-time analysis to the <uniqueKey> as it will likely - make routing in SolrCloud and document replacement in general fail. Limited _query_ time - analysis is possible as long as the indexing process is guaranteed to index the term - in a compatible way. Any analysis applied to the <uniqueKey> should _not_ produce multiple - tokens - --> - <field name="guid" type="string" indexed="true" stored="true" required="true" multiValued="false" /> + <!-- Only remove the "id" field if you have a very good reason to. While not strictly + required, it is highly recommended. A <uniqueKey> is present in almost all Solr + installations. See the <uniqueKey> declaration below where <uniqueKey> is set to "id". + Do NOT change the type and apply index-time analysis to the <uniqueKey> as it will likely + make routing in SolrCloud and document replacement in general fail. Limited _query_ time + analysis is possible as long as the indexing process is guaranteed to index the term + in a compatible way. Any analysis applied to the <uniqueKey> should _not_ produce multiple + tokens + --> + <field name="guid" type="string" indexed="true" stored="true" required="true" + multiValued="false"/> - <field name="source:type" type="string" indexed="true" stored="true"/> - <field name="name" type="string" indexed="true" stored="true" /> - <field name="timestamp" type="plong" indexed="true" stored="true" /> - <field name="new-field" type="string" indexed="true" stored="true" /> + <field name="source.type" type="string" indexed="true" stored="true"/> + <field name="name" type="string" indexed="true" stored="true"/> + <field name="timestamp" type="plong" indexed="true" stored="true"/> + <field name="new-field" type="string" indexed="true" stored="true"/> + <field name="metaalerts" type="string" multiValued="true" indexed="true" stored="true"/> + <field name="threat:triage:score" type="pdouble" indexed="true" stored="true"/> + <field name="score" type="pdouble" indexed="true" stored="true"/> - <dynamicField name="*" type="ignored" multiValued="false" docValues="true"/> + <dynamicField name="*" type="ignored" multiValued="false" docValues="true"/> - <!-- Field to use to determine and enforce document uniqueness. - Unless this field is marked with required="false", it will be a required field - --> - <uniqueKey>guid</uniqueKey> - - <!-- field type definitions. The "name" attribute is - just a label to be used by field definitions. The "class" - attribute and any other attributes determine the real - behavior of the fieldType. - Class names starting with "solr" refer to java classes in a - standard package such as org.apache.solr.analysis + <!-- Field to use to determine and enforce document uniqueness. + Unless this field is marked with required="false", it will be a required field --> + <uniqueKey>guid</uniqueKey> + + + <!-- field type definitions. The "name" attribute is + just a label to be used by field definitions. The "class" + attribute and any other attributes determine the real + behavior of the fieldType. + Class names starting with "solr" refer to java classes in a + standard package such as org.apache.solr.analysis + --> - <!-- The StrField type is not analyzed, but indexed/stored verbatim. --> - <fieldType name="string" class="solr.StrField" sortMissingLast="true" /> - <fieldType name="boolean" class="solr.BoolField" sortMissingLast="true"/> - <fieldType name="pint" class="solr.IntPointField" docValues="true"/> - <fieldType name="pfloat" class="solr.FloatPointField" docValues="true"/> - <fieldType name="plong" class="solr.LongPointField" docValues="true"/> - <fieldType name="pdouble" class="solr.DoublePointField" docValues="true"/> - <fieldType name="ignored" stored="false" indexed="false" multiValued="true" class="solr.StrField" /> + <!-- The StrField type is not analyzed, but indexed/stored verbatim. --> + <fieldType name="string" class="solr.StrField" sortMissingLast="true"/> + <fieldType name="boolean" class="solr.BoolField" sortMissingLast="true"/> + <fieldType name="pint" class="solr.IntPointField" docValues="true"/> + <fieldType name="pfloat" class="solr.FloatPointField" docValues="true"/> + <fieldType name="plong" class="solr.LongPointField" docValues="true"/> + <fieldType name="pdouble" class="solr.DoublePointField" docValues="true"/> + <fieldType name="ignored" stored="false" indexed="false" multiValued="true" + class="solr.StrField"/> </schema>
